X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=data%2Frbot%2Fplugins%2Furl.rb;h=8c064dccb91750d866e111fc87765ac4aaf4635c;hb=37fb9f96d582d24ffb9bbc14cc75fc9fec52ace3;hp=396c5ef219ccc7db8deb0b61782500ed1a30a4f0;hpb=7bbff3acaa1ea14fcb61dd3166ef62b01c33c01a;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git diff --git a/data/rbot/plugins/url.rb b/data/rbot/plugins/url.rb index 396c5ef2..8c064dcc 100644 --- a/data/rbot/plugins/url.rb +++ b/data/rbot/plugins/url.rb @@ -1,283 +1,29 @@ -require 'net/http' -require 'uri' -require 'cgi' +Url = Struct.new("Url", :channel, :nick, :time, :url, :info) -Url = Struct.new("Url", :channel, :nick, :time, :url) -TITLE_RE = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im - -UNESCAPE_TABLE = { - 'raquo' => '>>', - 'quot' => '"', - 'micro' => 'u', - 'copy' => '(c)', - 'trade' => '(tm)', - 'reg' => '(R)', - '#174' => '(R)', - '#8220' => '"', - '#8221' => '"', - '#8212' => '--', - '#39' => '\'', -=begin - # extras codes, for future use... - 'zwnj' => '‌', - 'aring' => '\xe5', - 'gt' => '>', - 'yen' => '\xa5', - 'ograve' => '\xf2', - 'Chi' => 'Χ', - 'bull' => '•', - 'Egrave' => '\xc8', - 'Ntilde' => '\xd1', - 'upsih' => 'ϒ', - 'Yacute' => '\xdd', - 'asymp' => '≈', - 'radic' => '√', - 'otimes' => '⊗', - 'nabla' => '∇', - 'aelig' => '\xe6', - 'oelig' => 'œ', - 'equiv' => '≡', - 'Psi' => 'Ψ', - 'auml' => '\xe4', - 'circ' => 'ˆ', - 'Acirc' => '\xc2', - 'Epsilon' => 'Ε', - 'Yuml' => 'Ÿ', - 'Eta' => 'Η', - 'lt' => '<', - 'Icirc' => '\xce', - 'Upsilon' => 'Υ', - 'ndash' => '–', - 'there4' => '∴', - 'Prime' => '″', - 'prime' => '′', - 'psi' => 'ψ', - 'Kappa' => 'Κ', - 'rsaquo' => '›', - 'Tau' => 'Τ', - 'darr' => '↓', - 'ocirc' => '\xf4', - 'lrm' => '‎', - 'zwj' => '‍', - 'cedil' => '\xb8', - 'Ecirc' => '\xca', - 'not' => '\xac', - 'amp' => '&', - 'AElig' => '\xc6', - 'oslash' => '\xf8', - 'acute' => '\xb4', - 'lceil' => '⌈', - 'laquo' => '\xab', - 'shy' => '\xad', - 'rdquo' => '”', - 'ge' => '≥', - 'Igrave' => '\xcc', - 'Ograve' => '\xd2', - 'euro' => '€', - 'dArr' => '⇓', - 'sdot' => '⋅', - 'nbsp' => '\xa0', - 'lfloor' => '⌊', - 'lArr' => '⇐', - 'Auml' => '\xc4', - 'larr' => '←', - 'Atilde' => '\xc3', - 'Otilde' => '\xd5', - 'szlig' => '\xdf', - 'clubs' => '♣', - 'diams' => '♦', - 'agrave' => '\xe0', - 'Ocirc' => '\xd4', - 'Iota' => 'Ι', - 'Theta' => 'Θ', - 'Pi' => 'Π', - 'OElig' => 'Œ', - 'Scaron' => 'Š', - 'frac14' => '\xbc', - 'egrave' => '\xe8', - 'sub' => '⊂', - 'iexcl' => '\xa1', - 'frac12' => '\xbd', - 'sbquo' => '‚', - 'ordf' => '\xaa', - 'sum' => '∑', - 'prop' => '∝', - 'Uuml' => '\xdc', - 'ntilde' => '\xf1', - 'sup' => '⊃', - 'theta' => 'θ', - 'prod' => '∏', - 'nsub' => '⊄', - 'hArr' => '⇔', - 'rlm' => '‏', - 'THORN' => '\xde', - 'infin' => '∞', - 'yuml' => '\xff', - 'Mu' => 'Μ', - 'le' => '≤', - 'Eacute' => '\xc9', - 'thinsp' => ' ', - 'ecirc' => '\xea', - 'bdquo' => '„', - 'Sigma' => 'Σ', - 'fnof' => 'ƒ', - 'Aring' => '\xc5', - 'tilde' => '˜', - 'frac34' => '\xbe', - 'emsp' => ' ', - 'mdash' => '—', - 'uarr' => '↑', - 'permil' => '‰', - 'Ugrave' => '\xd9', - 'rarr' => '→', - 'Agrave' => '\xc0', - 'chi' => 'χ', - 'forall' => '∀', - 'eth' => '\xf0', - 'rceil' => '⌉', - 'iuml' => '\xef', - 'gamma' => 'γ', - 'lambda' => 'λ', - 'harr' => '↔', - 'rang' => '〉', - 'xi' => 'ξ', - 'dagger' => '†', - 'divide' => '\xf7', - 'Ouml' => '\xd6', - 'image' => 'ℑ', - 'alefsym' => 'ℵ', - 'igrave' => '\xec', - 'otilde' => '\xf5', - 'Oacute' => '\xd3', - 'sube' => '⊆', - 'alpha' => 'α', - 'frasl' => '⁄', - 'ETH' => '\xd0', - 'lowast' => '∗', - 'Nu' => 'Ν', - 'plusmn' => '\xb1', - 'Euml' => '\xcb', - 'real' => 'ℜ', - 'sup1' => '\xb9', - 'sup2' => '\xb2', - 'sup3' => '\xb3', - 'Oslash' => '\xd8', - 'Aacute' => '\xc1', - 'cent' => '\xa2', - 'oline' => '‾', - 'Beta' => 'Β', - 'perp' => '⊥', - 'Delta' => 'Δ', - 'loz' => '◊', - 'pi' => 'π', - 'iota' => 'ι', - 'empty' => '∅', - 'euml' => '\xeb', - 'brvbar' => '\xa6', - 'iacute' => '\xed', - 'para' => '\xb6', - 'micro' => '\xb5', - 'cup' => '∪', - 'weierp' => '℘', - 'uuml' => '\xfc', - 'part' => '∂', - 'icirc' => '\xee', - 'delta' => 'δ', - 'omicron' => 'ο', - 'upsilon' => 'υ', - 'Iuml' => '\xcf', - 'Lambda' => 'Λ', - 'Xi' => 'Ξ', - 'kappa' => 'κ', - 'ccedil' => '\xe7', - 'Ucirc' => '\xdb', - 'cap' => '∩', - 'mu' => 'μ', - 'scaron' => 'š', - 'lsquo' => '‘', - 'isin' => '∈', - 'Zeta' => 'Ζ', - 'supe' => '⊇', - 'deg' => '\xb0', - 'and' => '∧', - 'tau' => 'τ', - 'pound' => '\xa3', - 'hellip' => '…', - 'curren' => '\xa4', - 'int' => '∫', - 'ucirc' => '\xfb', - 'rfloor' => '⌋', - 'ensp' => ' ', - 'crarr' => '↵', - 'ugrave' => '\xf9', - 'notin' => '∉', - 'exist' => '∃', - 'uArr' => '⇑', - 'cong' => '≅', - 'Dagger' => '‡', - 'oplus' => '⊕', - 'times' => '\xd7', - 'atilde' => '\xe3', - 'piv' => 'ϖ', - 'ni' => '∋', - 'Phi' => 'Φ', - 'lsaquo' => '‹', - 'Uacute' => '\xda', - 'Omicron' => 'Ο', - 'ang' => '∠', - 'ne' => '≠', - 'iquest' => '\xbf', - 'eta' => 'η', - 'yacute' => '\xfd', - 'Rho' => 'Ρ', - 'uacute' => '\xfa', - 'Alpha' => 'Α', - 'zeta' => 'ζ', - 'Omega' => 'Ω', - 'nu' => 'ν', - 'sim' => '∼', - 'sect' => '\xa7', - 'phi' => 'φ', - 'sigmaf' => 'ς', - 'macr' => '\xaf', - 'minus' => '−', - 'Ccedil' => '\xc7', - 'ordm' => '\xba', - 'epsilon' => 'ε', - 'beta' => 'β', - 'rArr' => '⇒', - 'rho' => 'ρ', - 'aacute' => '\xe1', - 'eacute' => '\xe9', - 'omega' => 'ω', - 'middot' => '\xb7', - 'Gamma' => 'Γ', - 'Iacute' => '\xcd', - 'lang' => '〈', - 'spades' => '♠', - 'rsquo' => '’', - 'uml' => '\xa8', - 'thorn' => '\xfe', - 'ouml' => '\xf6', - 'thetasym' => 'ϑ', - 'or' => '∨', - 'raquo' => '\xbb', - 'acirc' => '\xe2', - 'ldquo' => '“', - 'hearts' => '♥', - 'sigma' => 'σ', - 'oacute' => '\xf3', -=end -} +class ::UrlLinkError < RuntimeError +end class UrlPlugin < Plugin + TITLE_RE = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im + LINK_INFO = "[Link Info]" + BotConfig.register BotConfigIntegerValue.new('url.max_urls', :default => 100, :validate => Proc.new{|v| v > 0}, :desc => "Maximum number of urls to store. New urls replace oldest ones.") BotConfig.register BotConfigBooleanValue.new('url.display_link_info', - :default => true, + :default => false, :desc => "Get the title of any links pasted to the channel and display it (also tells if the link is broken or the site is down)") - + BotConfig.register BotConfigBooleanValue.new('url.titles_only', + :default => false, + :desc => "Only show info for links that have tags (in other words, don't display info for jpegs, mpegs, etc.)") + BotConfig.register BotConfigBooleanValue.new('url.first_par', + :default => false, + :desc => "Also try to get the first paragraph of a web page") + BotConfig.register BotConfigBooleanValue.new('url.info_on_list', + :default => false, + :desc => "Show link info when listing/searching for urls") + + def initialize super @registry.set_default(Array.new) @@ -287,78 +33,74 @@ class UrlPlugin < Plugin "urls [<max>=4] => list <max> last urls mentioned in current channel, urls search [<max>=4] <regexp> => search for matching urls. In a private message, you must specify the channel to query, eg. urls <channel> [max], urls search <channel> [max] <regexp>" end - def unescape_title(htmldata) - # first pass -- let CGI try to attack it... - htmldata = CGI::unescapeHTML htmldata - - # second pass -- destroy the remaining bits... - htmldata.gsub(/(&(.+?);)/) { - symbol = $2 - - # remove the 0-paddng from unicode integers - if symbol =~ /#(.+)/ - symbol = "##{$1.to_i.to_s}" - end - - # output the symbol's irc-translated character, or a * if it's unknown - UNESCAPE_TABLE[symbol] || '*' - } - end - def get_title_from_html(pagedata) return unless TITLE_RE.match(pagedata) - title = $1.strip.gsub(/\s*\n+\s*/, " ") - title = unescape_title title - title = title[0..255] if title.length > 255 - "[Link Info] title: #{title}" + $1.ircify_html end - def get_title_for_url(uri_str, depth=10) - # This god-awful mess is what the ruby http library has reduced me to. - # Python's HTTP lib is so much nicer. :~( - - if depth == 0 - raise "Error: Maximum redirects hit." - end - - puts "+ Getting #{uri_str}" - url = URI.parse(uri_str) + def get_title_for_url(uri_str) + + url = uri_str.kind_of?(URI) ? uri_str : URI.parse(uri_str) return if url.scheme !~ /https?/ - - puts "+ connecting to #{url.host}:#{url.port}" - http = @bot.httputil.get_proxy(url) - title = http.start do |http| - url.path = '/' if url.path == '' - head = http.request_head(url.path) - case head - when Net::HTTPRedirection then - # call self recursively if this is a redirect - redirect_to = head['location'] - puts "+ redirect location: #{redirect_to}" - url = URI.join url.to_s, redirect_to - puts "+ whee, redirecting to #{url.to_s}!" - title = get_title_for_url(url.to_s, depth-1) - when Net::HTTPSuccess then - if head['content-type'] =~ /^text\// and (not head['content-length'] or head['content-length'].to_i < 400000) - # since the content is 'text/*' and is small enough to - # be a webpage, retrieve the title from the page - puts "+ getting #{url.request_uri}" - response = http.request_get(url.request_uri) - return get_title_from_html(response.body) - else + + title = nil + extra = String.new + + begin + debug "+ getting #{url.request_uri}" + @bot.httputil.get_response(url) { |resp| + case resp + when Net::HTTPSuccess + + if resp['content-type'] =~ /^text\/|(?:x|ht)ml/ + # The page is text or HTML, so we can try finding a title and, if + # requested, the first par. + # + # We act differently depending on whether we want the first par or + # not: in the first case we download the initial part and the parse + # it; in the second case we only download as much as we need to find + # the title + # + if @bot.config['url.first_par'] + partial = resp.partial_body(@bot.config['http.info_bytes']) + title = get_title_from_html(partial) + first_par = Utils.ircify_first_html_par(partial, :strip => title) + extra << ", #{Bold}text#{Bold}: #{first_par}" unless first_par.empty? + return "#{Bold}title#{Bold}: #{title}#{extra}" if title + else + resp.partial_body(@bot.config['http.info_bytes']) { |part| + title = get_title_from_html(part) + return "#{Bold}title#{Bold}: #{title}" if title + } + end + # if nothing was found, provide more basic info, as for non-html pages + end + + debug resp.to_hash.inspect + + enc = resp['content-encoding'] + + extra << ", #{Bold}encoding#{Bold}: #{enc}" if enc + + unless @bot.config['url.titles_only'] # content doesn't have title, just display info. - size = head['content-length'].gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') - #lastmod = head['last-modified'] - return "[Link Info] type: #{head['content-type']}#{size ? ", size: #{size} bytes" : ""}" + size = resp['content-length'].gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') rescue nil + size = size ? ", #{Bold}size#{Bold}: #{size} bytes" : "" + return "#{Bold}type#{Bold}: #{resp['content-type']}#{size}#{extra}" end - when Net::HTTPClientError then - return "[Link Info] Error getting link (#{head.code} - #{head.message})" - when Net::HTTPServerError then - return "[Link Info] Error getting link (#{head.code} - #{head.message})" + else + raise UrlLinkError, "getting link (#{resp.code} - #{resp.message})" + end + } + rescue Exception => e + case e + when UrlLinkError + raise e + else + error e + raise "connecting to site/processing information (#{e.message})" end end - rescue SocketError => e - return "[Link Info] Error connecting to site (#{e.message})" end def listen(m) @@ -370,21 +112,28 @@ class UrlPlugin < Plugin urlstr = $1 list = @registry[m.target] + title = nil if @bot.config['url.display_link_info'] - debug "Getting title for #{urlstr}..." - title = get_title_for_url urlstr - if title - m.reply title - debug "Title found!" - else - debug "Title not found!" - end + Thread.start do + debug "Getting title for #{urlstr}..." + begin + title = get_title_for_url urlstr + if title + m.reply "#{LINK_INFO} #{title}", :overlong => :truncate + debug "Title found!" + else + debug "Title not found!" + end + rescue => e + m.reply "Error #{e.message}" + end + end end - + # check to see if this url is already listed return if list.find {|u| u.url == urlstr } - - url = Url.new(m.target, m.sourcenick, Time.new, urlstr) + + url = Url.new(m.target, m.sourcenick, Time.new, urlstr, title) debug "#{list.length} urls so far" if list.length > @bot.config['url.max_urls'] list.pop @@ -397,6 +146,20 @@ class UrlPlugin < Plugin end end + def reply_urls(m, list, max) + list[0..(max-1)].each do |url| + disp = "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}" + if @bot.config['url.info_on_list'] + title = url.info || get_title_for_url(url.url) rescue nil + if title and not url.info + url.info = title + end + disp << " --> #{title}" if title + end + m.reply disp, :overlong => :truncate + end + end + def urls(m, params) channel = params[:channel] ? params[:channel] : m.target max = params[:limit].to_i @@ -406,9 +169,7 @@ class UrlPlugin < Plugin if list.empty? m.reply "no urls seen yet for channel #{channel}" else - list[0..(max-1)].each do |url| - m.reply "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}" - end + reply_urls(m, list, max) end end @@ -425,12 +186,11 @@ class UrlPlugin < Plugin if list.empty? m.reply "no matches for channel #{channel}" else - list[0..(max-1)].each do |url| - m.reply "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}" - end + reply_urls(m, list, max) end end end + plugin = UrlPlugin.new plugin.map 'urls search :channel :limit :string', :action => 'search', :defaults => {:limit => 4},