X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=data%2Frbot%2Fplugins%2Furl.rb;h=821b0cc7f12dd66fa95b0585877e9be9addfc587;hb=7d71cd7b5382ae519118148fa6deed1a58c0bd2d;hp=f13398a867b2f122f33ec2fe078f25cbb3f76cbc;hpb=576d95621f715d48845472b2e29c0a0c0996020f;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git diff --git a/data/rbot/plugins/url.rb b/data/rbot/plugins/url.rb index f13398a8..821b0cc7 100644 --- a/data/rbot/plugins/url.rb +++ b/data/rbot/plugins/url.rb @@ -3,7 +3,272 @@ require 'uri' require 'cgi' Url = Struct.new("Url", :channel, :nick, :time, :url) -TITLE_RE = /<\s*title\s*>(.+)<\s*\/title\s*>/im +TITLE_RE = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im + +UNESCAPE_TABLE = { + 'raquo' => '>>', + 'quot' => '"', + 'micro' => 'u', + 'copy' => '(c)', + 'trade' => '(tm)', + 'reg' => '(R)', + '#174' => '(R)', + '#8220' => '"', + '#8221' => '"', + '#8212' => '--', + '#39' => '\'', +=begin + # extras codes, for future use... + 'zwnj' => '‌', + 'aring' => '\xe5', + 'gt' => '>', + 'yen' => '\xa5', + 'ograve' => '\xf2', + 'Chi' => 'Χ', + 'bull' => '•', + 'Egrave' => '\xc8', + 'Ntilde' => '\xd1', + 'upsih' => 'ϒ', + 'Yacute' => '\xdd', + 'asymp' => '≈', + 'radic' => '√', + 'otimes' => '⊗', + 'nabla' => '∇', + 'aelig' => '\xe6', + 'oelig' => 'œ', + 'equiv' => '≡', + 'Psi' => 'Ψ', + 'auml' => '\xe4', + 'circ' => 'ˆ', + 'Acirc' => '\xc2', + 'Epsilon' => 'Ε', + 'Yuml' => 'Ÿ', + 'Eta' => 'Η', + 'lt' => '<', + 'Icirc' => '\xce', + 'Upsilon' => 'Υ', + 'ndash' => '–', + 'there4' => '∴', + 'Prime' => '″', + 'prime' => '′', + 'psi' => 'ψ', + 'Kappa' => 'Κ', + 'rsaquo' => '›', + 'Tau' => 'Τ', + 'darr' => '↓', + 'ocirc' => '\xf4', + 'lrm' => '‎', + 'zwj' => '‍', + 'cedil' => '\xb8', + 'Ecirc' => '\xca', + 'not' => '\xac', + 'amp' => '&', + 'AElig' => '\xc6', + 'oslash' => '\xf8', + 'acute' => '\xb4', + 'lceil' => '⌈', + 'laquo' => '\xab', + 'shy' => '\xad', + 'rdquo' => '”', + 'ge' => '≥', + 'Igrave' => '\xcc', + 'Ograve' => '\xd2', + 'euro' => '€', + 'dArr' => '⇓', + 'sdot' => '⋅', + 'nbsp' => '\xa0', + 'lfloor' => '⌊', + 'lArr' => '⇐', + 'Auml' => '\xc4', + 'larr' => '←', + 'Atilde' => '\xc3', + 'Otilde' => '\xd5', + 'szlig' => '\xdf', + 'clubs' => '♣', + 'diams' => '♦', + 'agrave' => '\xe0', + 'Ocirc' => '\xd4', + 'Iota' => 'Ι', + 'Theta' => 'Θ', + 'Pi' => 'Π', + 'OElig' => 'Œ', + 'Scaron' => 'Š', + 'frac14' => '\xbc', + 'egrave' => '\xe8', + 'sub' => '⊂', + 'iexcl' => '\xa1', + 'frac12' => '\xbd', + 'sbquo' => '‚', + 'ordf' => '\xaa', + 'sum' => '∑', + 'prop' => '∝', + 'Uuml' => '\xdc', + 'ntilde' => '\xf1', + 'sup' => '⊃', + 'theta' => 'θ', + 'prod' => '∏', + 'nsub' => '⊄', + 'hArr' => '⇔', + 'rlm' => '‏', + 'THORN' => '\xde', + 'infin' => '∞', + 'yuml' => '\xff', + 'Mu' => 'Μ', + 'le' => '≤', + 'Eacute' => '\xc9', + 'thinsp' => ' ', + 'ecirc' => '\xea', + 'bdquo' => '„', + 'Sigma' => 'Σ', + 'fnof' => 'ƒ', + 'Aring' => '\xc5', + 'tilde' => '˜', + 'frac34' => '\xbe', + 'emsp' => ' ', + 'mdash' => '—', + 'uarr' => '↑', + 'permil' => '‰', + 'Ugrave' => '\xd9', + 'rarr' => '→', + 'Agrave' => '\xc0', + 'chi' => 'χ', + 'forall' => '∀', + 'eth' => '\xf0', + 'rceil' => '⌉', + 'iuml' => '\xef', + 'gamma' => 'γ', + 'lambda' => 'λ', + 'harr' => '↔', + 'rang' => '〉', + 'xi' => 'ξ', + 'dagger' => '†', + 'divide' => '\xf7', + 'Ouml' => '\xd6', + 'image' => 'ℑ', + 'alefsym' => 'ℵ', + 'igrave' => '\xec', + 'otilde' => '\xf5', + 'Oacute' => '\xd3', + 'sube' => '⊆', + 'alpha' => 'α', + 'frasl' => '⁄', + 'ETH' => '\xd0', + 'lowast' => '∗', + 'Nu' => 'Ν', + 'plusmn' => '\xb1', + 'Euml' => '\xcb', + 'real' => 'ℜ', + 'sup1' => '\xb9', + 'sup2' => '\xb2', + 'sup3' => '\xb3', + 'Oslash' => '\xd8', + 'Aacute' => '\xc1', + 'cent' => '\xa2', + 'oline' => '‾', + 'Beta' => 'Β', + 'perp' => '⊥', + 'Delta' => 'Δ', + 'loz' => '◊', + 'pi' => 'π', + 'iota' => 'ι', + 'empty' => '∅', + 'euml' => '\xeb', + 'brvbar' => '\xa6', + 'iacute' => '\xed', + 'para' => '\xb6', + 'micro' => '\xb5', + 'cup' => '∪', + 'weierp' => '℘', + 'uuml' => '\xfc', + 'part' => '∂', + 'icirc' => '\xee', + 'delta' => 'δ', + 'omicron' => 'ο', + 'upsilon' => 'υ', + 'Iuml' => '\xcf', + 'Lambda' => 'Λ', + 'Xi' => 'Ξ', + 'kappa' => 'κ', + 'ccedil' => '\xe7', + 'Ucirc' => '\xdb', + 'cap' => '∩', + 'mu' => 'μ', + 'scaron' => 'š', + 'lsquo' => '‘', + 'isin' => '∈', + 'Zeta' => 'Ζ', + 'supe' => '⊇', + 'deg' => '\xb0', + 'and' => '∧', + 'tau' => 'τ', + 'pound' => '\xa3', + 'hellip' => '…', + 'curren' => '\xa4', + 'int' => '∫', + 'ucirc' => '\xfb', + 'rfloor' => '⌋', + 'ensp' => ' ', + 'crarr' => '↵', + 'ugrave' => '\xf9', + 'notin' => '∉', + 'exist' => '∃', + 'uArr' => '⇑', + 'cong' => '≅', + 'Dagger' => '‡', + 'oplus' => '⊕', + 'times' => '\xd7', + 'atilde' => '\xe3', + 'piv' => 'ϖ', + 'ni' => '∋', + 'Phi' => 'Φ', + 'lsaquo' => '‹', + 'Uacute' => '\xda', + 'Omicron' => 'Ο', + 'ang' => '∠', + 'ne' => '≠', + 'iquest' => '\xbf', + 'eta' => 'η', + 'yacute' => '\xfd', + 'Rho' => 'Ρ', + 'uacute' => '\xfa', + 'Alpha' => 'Α', + 'zeta' => 'ζ', + 'Omega' => 'Ω', + 'nu' => 'ν', + 'sim' => '∼', + 'sect' => '\xa7', + 'phi' => 'φ', + 'sigmaf' => 'ς', + 'macr' => '\xaf', + 'minus' => '−', + 'Ccedil' => '\xc7', + 'ordm' => '\xba', + 'epsilon' => 'ε', + 'beta' => 'β', + 'rArr' => '⇒', + 'rho' => 'ρ', + 'aacute' => '\xe1', + 'eacute' => '\xe9', + 'omega' => 'ω', + 'middot' => '\xb7', + 'Gamma' => 'Γ', + 'Iacute' => '\xcd', + 'lang' => '〈', + 'spades' => '♠', + 'rsquo' => '’', + 'uml' => '\xa8', + 'thorn' => '\xfe', + 'ouml' => '\xf6', + 'thetasym' => 'ϑ', + 'or' => '∨', + 'raquo' => '\xbb', + 'acirc' => '\xe2', + 'ldquo' => '“', + 'hearts' => '♥', + 'sigma' => 'σ', + 'oacute' => '\xf3', +=end +} class UrlPlugin < Plugin BotConfig.register BotConfigIntegerValue.new('url.max_urls', @@ -22,60 +287,111 @@ class UrlPlugin < Plugin "urls [=4] => list last urls mentioned in current channel, urls search [=4] => search for matching urls. In a private message, you must specify the channel to query, eg. urls [max], urls search [max] " end + def unescape_title(htmldata) + # first pass -- let CGI try to attack it... + htmldata = CGI::unescapeHTML htmldata + + # second pass -- destroy the remaining bits... + htmldata.gsub(/(&(.+?);)/) { + symbol = $2 + + # remove the 0-paddng from unicode integers + if symbol =~ /#(.+)/ + symbol = "##{$1.to_i.to_s}" + end + + # output the symbol's irc-translated character, or a * if it's unknown + UNESCAPE_TABLE[symbol] || '*' + } + end + def get_title_from_html(pagedata) return unless TITLE_RE.match(pagedata) title = $1.strip.gsub(/\s*\n+\s*/, " ") - title = CGI::unescapeHTML title + title = unescape_title title title = title[0..255] if title.length > 255 "[Link Info] title: #{title}" end - def get_title_for_url(uri_str) + def read_data_from_response(response, amount) + + amount_read = 0 + chunks = [] + + response.read_body do |chunk| # read body now + + amount_read += chunk.length + + if amount_read > amount + amount_of_overflow = amount_read - amount + chunk = chunk[0...-amount_of_overflow] + end + + chunks << chunk + + break if amount_read >= amount + + end + + chunks.join('') + + end + + + def get_title_for_url(uri_str, depth=10) # This god-awful mess is what the ruby http library has reduced me to. # Python's HTTP lib is so much nicer. :~( + if depth == 0 + raise "Error: Maximum redirects hit." + end + puts "+ Getting #{uri_str}" url = URI.parse(uri_str) return if url.scheme !~ /https?/ + + title = nil - puts "+ connecting to #{url.host}:#{url.port}" - http = @bot.httputil.get_proxy(url) - title = http.start do |http| + puts "+ connecting to #{url.host}:#{url.port}" + http = @bot.httputil.get_proxy(url) + http.start { |http| url.path = '/' if url.path == '' - head = http.request_head(url.path) - case head - when Net::HTTPRedirection then - # call self recursively if this is a redirect - redirect_to = head['location'] - puts "+ redirect location: #{redirect_to}" - absolute_uris = URI.extract redirect_to - raise "wtf! redirect = #{redirect_to}" if absolute_uris.size > 1 - if absolute_uris.size == 1 - url = URI.parse absolute_uris[0] - else - url.path = redirect_to - end - puts "+ whee, redirect to #{url.to_s}!" - title = get_title_for_url(url.to_s) - when Net::HTTPSuccess then - if head['content-type'] =~ /^text\// - # content is 'text/*' - # retrieve the title from the page - puts "+ getting #{url.path}" - response = http.request_get(url.path) - return get_title_from_html(response.body) + + http.request_get(url.path, "User-Agent" => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)") { |response| + + case response + when Net::HTTPRedirection, Net::HTTPMovedPermanently then + # call self recursively if this is a redirect + redirect_to = response['location'] || './' + puts "+ redirect location: #{redirect_to.inspect}" + url = URI.join url.to_s, redirect_to + puts "+ whee, redirecting to #{url.to_s}!" + return get_title_for_url(url.to_s, depth-1) + when Net::HTTPSuccess then + if response['content-type'] =~ /^text\// + # since the content is 'text/*' and is small enough to + # be a webpage, retrieve the title from the page + puts "+ getting #{url.request_uri}" + data = read_data_from_response(response, 50000) + return get_title_from_html(data) + else + # content doesn't have title, just display info. + size = response['content-length'].gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') + return "[Link Info] type: #{response['content-type']}#{size ? ", size: #{size} bytes" : ""}" + end + when Net::HTTPClientError then + return "[Link Info] Error getting link (#{response.code} - #{response.message})" + when Net::HTTPServerError then + return "[Link Info] Error getting link (#{response.code} - #{response.message})" else - # content isn't 'text/*'... display info about the file. - size = head['content-length'].gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') - #lastmod = head['last-modified'] - return "[Link Info] type: #{head['content-type']}#{size ? ", size: #{size} bytes" : ""}" - end - when Net::HTTPClientError then - return "[Link Info] Error getting link (#{head.code} - #{head.message})" - when Net::HTTPServerError then - return "[Link Info] Error getting link (#{head.code} - #{head.message})" - end - end + return nil + end # end of "case response" + + } # end of request block + } # end of http start block + + return title + rescue SocketError => e return "[Link Info] Error connecting to site (#{e.message})" end @@ -89,7 +405,7 @@ class UrlPlugin < Plugin urlstr = $1 list = @registry[m.target] - if @bot.config['url.say_titles'] + if @bot.config['url.display_link_info'] debug "Getting title for #{urlstr}..." title = get_title_for_url urlstr if title @@ -137,7 +453,7 @@ class UrlPlugin < Plugin string = params[:string] max = 10 if max > 10 max = 1 if max < 1 - regex = Regexp.new(string) + regex = Regexp.new(string, Regexp::IGNORECASE) list = @registry[channel].find_all {|url| regex.match(url.url) || regex.match(url.nick) }