fix: URI.escape is not needed anymore

[user/henk/code/ruby/rbot.git] / data / rbot / plugins / url.rb

diff --git a/data/rbot/plugins/url.rb b/data/rbot/plugins/url.rb
index e3cecb6b0d5805654c4ff0fe831928dc5bba0566..e974c96b99042608da98829f82546ee4301781d5 100644 (file)
--- a/data/rbot/plugins/url.rb
+++ b/data/rbot/plugins/url.rb
@@ -5,11 +5,7 @@
 
 define_structure :Url, :channel, :nick, :time, :url, :info
 
 
 define_structure :Url, :channel, :nick, :time, :url, :info
 

-class ::UrlLinkError < RuntimeError
-end
-

 class UrlPlugin < Plugin
 class UrlPlugin < Plugin

-  TITLE_RE = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im

   LINK_INFO = "[Link Info]"
   OUR_UNSAFE = Regexp.new("[^#{URI::PATTERN::UNRESERVED}#{URI::PATTERN::RESERVED}%# ]", false, 'N')
 
   LINK_INFO = "[Link Info]"
   OUR_UNSAFE = Regexp.new("[^#{URI::PATTERN::UNRESERVED}#{URI::PATTERN::RESERVED}%# ]", false, 'N')
 


@@ -19,12 +15,24 @@ class UrlPlugin < Plugin
   Config.register Config::IntegerValue.new('url.display_link_info',
     :default => 0,
     :desc => "Get the title of links pasted to the channel and display it (also tells if the link is broken or the site is down). Do it for at most this many links per line (set to 0 to disable)")
   Config.register Config::IntegerValue.new('url.display_link_info',
     :default => 0,
     :desc => "Get the title of links pasted to the channel and display it (also tells if the link is broken or the site is down). Do it for at most this many links per line (set to 0 to disable)")

+  Config.register Config::BooleanValue.new('url.auto_shorten',
+    :default => false,
+    :desc => "Automatically spit out shortened URLs when they're seen. Check shortenurls for config options")
+  Config.register Config::IntegerValue.new('url.auto_shorten_min_length',
+    :default => 48,
+    :desc => "Minimum length of URL to auto-shorten.  Only has an effect when url.auto_shorten is true.")

   Config.register Config::BooleanValue.new('url.titles_only',
     :default => false,
     :desc => "Only show info for links that have <title> tags (in other words, don't display info for jpegs, mpegs, etc.)")
   Config.register Config::BooleanValue.new('url.first_par',
     :default => false,
     :desc => "Also try to get the first paragraph of a web page")
   Config.register Config::BooleanValue.new('url.titles_only',
     :default => false,
     :desc => "Only show info for links that have <title> tags (in other words, don't display info for jpegs, mpegs, etc.)")
   Config.register Config::BooleanValue.new('url.first_par',
     :default => false,
     :desc => "Also try to get the first paragraph of a web page")

+  Config.register Config::IntegerValue.new('url.first_par_length',
+    :default => 150,
+    :desc => "The max length of the first paragraph")
+  Config.register Config::ArrayValue.new('url.first_par_whitelist',
+    :default => ['twitter.com'],
+    :desc => "List of url patterns to show the content for.")

   Config.register Config::BooleanValue.new('url.info_on_list',
     :default => false,
     :desc => "Show link info when listing/searching for urls")
   Config.register Config::BooleanValue.new('url.info_on_list',
     :default => false,
     :desc => "Show link info when listing/searching for urls")


@@ -32,7 +40,12 @@ class UrlPlugin < Plugin
     :default => ['localhost', '^192\.168\.', '^10\.', '^127\.', '^172\.(1[6-9]|2\d|31)\.'],
     :on_change => Proc.new { |bot, v| bot.plugins['url'].reset_no_info_hosts },
     :desc => "A list of regular expressions matching hosts for which no info should be provided")
     :default => ['localhost', '^192\.168\.', '^10\.', '^127\.', '^172\.(1[6-9]|2\d|31)\.'],
     :on_change => Proc.new { |bot, v| bot.plugins['url'].reset_no_info_hosts },
     :desc => "A list of regular expressions matching hosts for which no info should be provided")

-
+  Config.register Config::ArrayValue.new('url.only_on_channels',
+    :desc => "Show link info only on these channels",
+    :default => [])
+  Config.register Config::ArrayValue.new('url.ignore',
+    :desc => "Don't show link info for urls from users represented as hostmasks on this list. Useful for ignoring other bots, for example.",
+    :default => [])

 
   def initialize
     super
 
   def initialize
     super


@@ -41,6 +54,8 @@ class UrlPlugin < Plugin
       @bot.config.items[:'url.display_link_info'].set_string(@bot.config['url.display_link_info'].to_s)
     end
     reset_no_info_hosts
       @bot.config.items[:'url.display_link_info'].set_string(@bot.config['url.display_link_info'].to_s)
     end
     reset_no_info_hosts

+    self.filter_group = :htmlinfo
+    load_filters

   end
 
   def reset_no_info_hosts
   end
 
   def reset_no_info_hosts


@@ -49,98 +64,80 @@ class UrlPlugin < Plugin
   end
 
   def help(plugin, topic="")
   end
 
   def help(plugin, topic="")

-    "urls [<max>=4] => list <max> last urls mentioned in current channel, urls search [<max>=4] <regexp> => search for matching urls. In a private message, you must specify the channel to query, eg. urls <channel> [max], urls search <channel> [max] <regexp>"
+    "url info <url> => display link info for <url> (set url.display_link_info > 0 if you want the bot to do it automatically when someone writes an url), urls [<max>=4] => list <max> last urls mentioned in current channel, urls search [<max>=4] <regexp> => search for matching urls. In a private message, you must specify the channel to query, eg. urls <channel> [max], urls search <channel> [max] <regexp>"

   end
 
   def get_title_from_html(pagedata)
   end
 
   def get_title_from_html(pagedata)

-    return unless TITLE_RE.match(pagedata)
-    $1.ircify_html
+    return pagedata.ircify_html_title

   end
 
   end
 

-  def get_title_for_url(uri_str, nick = nil, channel = nil, ircline = nil)
+  def get_title_for_url(uri_str, opts = {})

 
     url = uri_str.kind_of?(URI) ? uri_str : URI.parse(uri_str)
     return if url.scheme !~ /https?/
 
 
     url = uri_str.kind_of?(URI) ? uri_str : URI.parse(uri_str)
     return if url.scheme !~ /https?/
 

-    if url.host =~ @no_info_hosts
-      return "Sorry, info retrieval for #{url.host} is disabled"
+    # also check the ip, the canonical name and the aliases
+    begin
+      checks = TCPSocket.gethostbyname(url.host)
+      checks.delete_at(-2)
+    rescue => e
+      return "Unable to retrieve info for #{url.host}: #{e.message}"

     end
 
     end
 

-    logopts = Hash.new
-    logopts[:nick] = nick if nick
-    logopts[:channel] = channel if channel
-    logopts[:ircline] = ircline if ircline
+    checks << url.host
+    checks.flatten!
+
+    unless checks.grep(@no_info_hosts).empty?
+      return ( opts[:always_reply] ? "Sorry, info retrieval for #{url.host} (#{checks.first}) is disabled" : false )
+    end
+
+    logopts = opts.dup

 
     title = nil
 
     title = nil

-    extra = String.new
+    extra = []

 
     begin
 
     begin

-      debug "+ getting #{url.request_uri}"
-      @bot.httputil.get_response(url) { |resp|
-        case resp
-        when Net::HTTPSuccess
-
-          debug resp.to_hash
-
-          if resp['content-type'] =~ /^text\/|(?:x|ht)ml/
-            # The page is text or HTML, so we can try finding a title and, if
-            # requested, the first par.
-            #
-            # We act differently depending on whether we want the first par or
-            # not: in the first case we download the initial part and the parse
-            # it; in the second case we only download as much as we need to find
-            # the title
-            #
-            if @bot.config['url.first_par']
-              partial = resp.partial_body(@bot.config['http.info_bytes'])
-              logopts[:title] = title = get_title_from_html(partial)
-              if url.fragment and not url.fragment.empty?
-                fragreg = /.*?<a\s+[^>]*name=["']?#{url.fragment}["']?.*?>/im
-                partial.sub!(fragreg,'')
-              end
-              first_par = Utils.ircify_first_html_par(partial, :strip => title)
-              unless first_par.empty?
-                logopts[:extra] = first_par
-                extra << ", #{Bold}text#{Bold}: #{first_par}"
-              end
-              call_event(:url_added, url.to_s, logopts)
-              return "#{Bold}title#{Bold}: #{title}#{extra}" if title
-            else
-              resp.partial_body(@bot.config['http.info_bytes']) { |part|
-                logopts[:title] = title = get_title_from_html(part)
-                call_event(:url_added, url.to_s, logopts)
-                return "#{Bold}title#{Bold}: #{title}" if title
-              }
-            end
-          # if nothing was found, provide more basic info, as for non-html pages
-          else
-            resp.no_cache = true
-          end
+      debug "+ getting info for #{url.request_uri}"
+      info = @bot.filter(:htmlinfo, url)
+      logopts[:htmlinfo] = info
+      resp = info[:headers]

 
 

-          enc = resp['content-encoding']
-          logopts[:extra] = String.new
-          logopts[:extra] << "Content Type: #{resp['content-type']}"
-          if enc
-            logopts[:extra] << ", encoding: #{enc}"
-            extra << ", #{Bold}encoding#{Bold}: #{enc}"
-          end
+      logopts[:title] = title = info[:title]
+
+      if info[:content]
+        logopts[:extra] = info[:content]
+
+        max_length = @bot.config['url.first_par_length']

 
 

-          unless @bot.config['url.titles_only']
-            # content doesn't have title, just display info.
-            size = resp['content-length'].gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') rescue nil
-            if size
-              logopts[:extra] << ", size: #{size} bytes"
-              size = ", #{Bold}size#{Bold}: #{size} bytes"
+        whitelist = @bot.config['url.first_par_whitelist']
+        content = nil
+        if whitelist.length > 0
+          whitelist.each do |pattern|
+            if Regexp.new(pattern, Regexp::IGNORECASE).match(url.to_s)
+              content = info[:content][0...max_length]
+              break

             end
             end

-            call_event(:url_added, url.to_s, logopts)
-            return "#{Bold}type#{Bold}: #{resp['content-type']}#{size}#{extra}"

           end
           end

-          call_event(:url_added, url.to_s, logopts)

         else
         else

-          raise UrlLinkError, "getting link (#{resp.code} - #{resp.message})"
+          content = info[:content][0...max_length]
+        end
+
+        extra << "#{Bold}text#{Bold}: #{content}" if @bot.config['url.first_par'] and content
+      else
+        logopts[:extra] = String.new
+        logopts[:extra] << "Content Type: #{resp['content-type']}"
+        extra << "#{Bold}type#{Bold}: #{resp['content-type']}" unless title
+        if enc = resp['content-encoding']
+          logopts[:extra] << ", encoding: #{enc}"
+          extra << "#{Bold}encoding#{Bold}: #{enc}" if @bot.config['url.first_par'] or not title
+        end
+
+        size = resp['content-length'].first.gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') rescue nil
+        if size
+          logopts[:extra] << ", size: #{size} bytes"
+          extra << "#{Bold}size#{Bold}: #{size} bytes" if @bot.config['url.first_par'] or not title

         end
         end

-      }
-      return nil
+      end

     rescue Exception => e
       case e
       when UrlLinkError
     rescue Exception => e
       case e
       when UrlLinkError


@@ -150,38 +147,82 @@ class UrlPlugin < Plugin
         raise "connecting to site/processing information (#{e.message})"
       end
     end
         raise "connecting to site/processing information (#{e.message})"
       end
     end

+
+    call_event(:url_added, url.to_s, logopts)
+    if title
+      extra.unshift("#{Bold}title#{Bold}: #{title}")
+    end
+    return extra.join(", ") if title or not @bot.config['url.titles_only']

   end
 
   end
 

-  def handle_urls(m, urls, display_info=@bot.config['url.display_link_info'])
+  def handle_urls(m, params={})
+    opts = {
+      :display_info => @bot.config['url.display_link_info'],
+      :channels => @bot.config['url.only_on_channels'],
+      :ignore => @bot.config['url.ignore']
+    }.merge params
+    urls = opts[:urls]
+    display_info= opts[:display_info]
+    channels = opts[:channels]
+    ignore = opts[:ignore]
+
+    unless channels.empty?
+      return unless channels.map { |c| c.downcase }.include?(m.channel.downcase)
+    end
+
+    ignore.each { |u| return if m.source.matches?(u) }
+

     return if urls.empty?
     debug "found urls #{urls.inspect}"
     return if urls.empty?
     debug "found urls #{urls.inspect}"

-    if m.public?
-      list = @registry[m.target] 
-    else
-      list = nil
-    end
+    list = m.public? ? @registry[m.target] : nil
+    debug "display link info: #{display_info}"

     urls_displayed = 0
     urls_displayed = 0

-    urls.each { |urlstr|
+    urls.each do |urlstr|

       debug "working on #{urlstr}"
       debug "working on #{urlstr}"

-      next unless urlstr =~ /^https?:/
+      next unless urlstr =~ /^https?:\/\/./
+      if @bot.config['url.auto_shorten'] == true and
+         urlstr.length >= @bot.config['url.auto_shorten_min_length']
+        m.reply(bot.plugins['shortenurls'].shorten(nil, {:url=>urlstr, :called=>true}))
+        next
+      end

       title = nil
       title = nil

-      debug "display link info: #{display_info}"
-      if display_info > urls_displayed
-        urls_displayed += 1
-        Thread.start do
-          debug "Getting title for #{urlstr}..."
-          begin
-            title = get_title_for_url urlstr, m.source.nick, m.channel, m.message
-            if title
-              m.reply "#{LINK_INFO} #{title}", :overlong => :truncate
-              debug "Title found!"
-            else
-              debug "Title not found!"
-            end
-          rescue => e
-            m.reply "Error #{e.message}"
+      debug "Getting title for #{urlstr}..."
+      reply = nil
+      begin
+        title = get_title_for_url(urlstr,
+                                  :always_reply => m.address?,
+                                  :nick => m.source.nick,
+                                  :channel => m.channel,
+                                  :ircline => m.message)
+        debug "Title #{title ? '' : 'not '} found"
+        reply = "#{LINK_INFO} #{title}" if title
+      rescue => e
+        debug e
+        # we might get a 404 because of trailing punctuation, so we try again
+        # with the last character stripped. this might generate invalid URIs
+        # (e.g. because "some.url" gets chopped to some.url%2, so catch that too
+        if e.message =~ /\(404 - Not Found\)/i or e.kind_of?(URI::InvalidURIError)
+          # chop off last non-word character from the unescaped version of
+          # the URL, and retry if we still have enough string to look like a
+          # minimal URL
+          unescaped = URI.unescape(urlstr)
+          debug "Unescaped: #{unescaped}"
+          if unescaped.sub!(/\W$/,'') and unescaped =~ /^https?:\/\/./
+            urlstr.replace URI.escape(unescaped, OUR_UNSAFE)
+            retry
+          else
+            debug "Not retrying #{unescaped}"

           end
         end
           end
         end

+        reply = "Error #{e.message}"
+      end
+
+      if display_info > urls_displayed
+        if reply
+          m.reply reply, :overlong => :truncate, :to => :public,
+            :nick => (m.address? ? :auto : false)
+          urls_displayed += 1
+        end

       end
 
       next unless list
       end
 
       next unless list


@@ -191,29 +232,31 @@ class UrlPlugin < Plugin
 
       url = Url.new(m.target, m.sourcenick, Time.new, urlstr, title)
       debug "#{list.length} urls so far"
 
       url = Url.new(m.target, m.sourcenick, Time.new, urlstr, title)
       debug "#{list.length} urls so far"

-      if list.length > @bot.config['url.max_urls']
-        list.pop
-      end
+      list.pop if list.length > @bot.config['url.max_urls']

       debug "storing url #{url.url}"
       list.unshift url
       debug "#{list.length} urls now"
       debug "storing url #{url.url}"
       list.unshift url
       debug "#{list.length} urls now"

-    }
+    end

     @registry[m.target] = list
   end
 
   def info(m, params)
     escaped = URI.escape(params[:urls].to_s, OUR_UNSAFE)
     urls = URI.extract(escaped)
     @registry[m.target] = list
   end
 
   def info(m, params)
     escaped = URI.escape(params[:urls].to_s, OUR_UNSAFE)
     urls = URI.extract(escaped)

-    handle_urls(m, urls, params[:urls].length)
+    Thread.new do
+      handle_urls(m,
+                  :urls => urls,
+                  :display_info => params[:urls].length,
+                  :channels => [])
+    end

   end
 
   end
 

-  def listen(m)
-    return unless m.kind_of?(PrivMessage)
+  def message(m)

     return if m.address?
 
     return if m.address?
 

-    escaped = URI.escape(m.message, OUR_UNSAFE)
-    urls = URI.extract(escaped)
-    handle_urls(m, urls)
+    urls = URI.extract(m.message, ['http', 'https'])
+    return if urls.empty?
+    Thread.new { handle_urls(m, :urls => urls) }

   end
 
   def reply_urls(opts={})
   end
 
   def reply_urls(opts={})


@@ -225,7 +268,9 @@ class UrlPlugin < Plugin
     list[0..(max-1)].each do |url|
       disp = "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}"
       if @bot.config['url.info_on_list']
     list[0..(max-1)].each do |url|
       disp = "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}"
       if @bot.config['url.info_on_list']

-        title = url.info || get_title_for_url(url.url, url.nick, channel) rescue nil
+        title = url.info ||
+          get_title_for_url(url.url,
+                            :nick => url.nick, :channel => channel) rescue nil

         # If the url info was missing and we now have some, try to upgrade it
         if channel and title and not url.info
           ll = @registry[channel]
         # If the url info was missing and we now have some, try to upgrade it
         if channel and title and not url.info
           ll = @registry[channel]


@@ -275,6 +320,7 @@ end
 
 plugin = UrlPlugin.new
 plugin.map 'urls info *urls', :action => 'info'
 
 plugin = UrlPlugin.new
 plugin.map 'urls info *urls', :action => 'info'

+plugin.map 'url info *urls', :action => 'info'

 plugin.map 'urls search :channel :limit :string', :action => 'search',
                           :defaults => {:limit => 4},
                           :requirements => {:limit => /^\d+$/},
 plugin.map 'urls search :channel :limit :string', :action => 'search',
                           :defaults => {:limit => 4},
                           :requirements => {:limit => /^\d+$/},