rss plugin: overrule max lines, display all feeds

[user/henk/code/ruby/rbot.git] / data / rbot / plugins / rss.rb

diff --git a/data/rbot/plugins/rss.rb b/data/rbot/plugins/rss.rb
index 5873c496a15f5a034893fb099da88c5567125635..885ac84d44792c634f2fc2cc0c6c47abfb20d6cd 100644 (file)
--- a/data/rbot/plugins/rss.rb
+++ b/data/rbot/plugins/rss.rb
@@ -20,6 +20,16 @@ require 'rss'
 # the DublinCore code.\r
 module ::RSS\r
 \r
+  # Make an  'unique' ID for a given item, based on appropriate bot options\r
+  # Currently only suppored is bot.config['rss.show_updated']: when true, the\r
+  # description is included in the uid hashing, otherwise it's not\r
+  #\r
+  def RSS.item_uid_for_bot(item, opts={})\r
+    options = { :show_updated => true}.merge(opts)\r
+    desc = options[:show_updated] ? item.description : nil\r
+    [item.title, item.link, desc].hash\r
+  end\r
+\r
   unless defined?(SLASH_PREFIX)\r
     SLASH_PREFIX = 'slash'\r
     SLASH_URI = "http://purl.org/rss/1.0/modules/slash/"\r
@@ -231,6 +241,10 @@ class RSSFeedsPlugin < Plugin
     :default => 300, :validate => Proc.new{|v| v > 30},\r
     :desc => "How many seconds to sleep before checking RSS feeds again")\r
 \r
+  BotConfig.register BotConfigBooleanValue.new('rss.show_updated',\r
+    :default => true,\r
+    :desc => "Whether feed items for which the description was changed should be shown as new")\r
+\r
   # We used to save the Mutex with the RssBlob, which was idiotic. And\r
   # since Mutexes dumped in one version might not be resotrable in another,\r
   # we need a few tricks to be able to restore data from other versions of Ruby\r
@@ -246,6 +260,8 @@ class RSSFeedsPlugin < Plugin
     end\r
   end\r
 \r
+  attr_reader :feeds\r
+\r
   def initialize\r
     super\r
     if @registry.has_key?(:feeds)\r
@@ -397,7 +413,7 @@ class RSSFeedsPlugin < Plugin
 \r
     m.reply "lemme fetch it..."\r
     title = items = nil\r
-    fetched = fetchRss(feed, m)\r
+    fetched = fetchRss(feed, m, false)\r
     return unless fetched or feed.xml\r
     if not fetched and feed.items\r
       m.reply "using old data"\r
@@ -447,7 +463,7 @@ class RSSFeedsPlugin < Plugin
       reply = "no feeds found"\r
       reply << " matching #{wanted}" if wanted\r
     end\r
-    m.reply reply\r
+    m.reply reply, :max_lines => reply.length\r
   end\r
 \r
   def watched_rss(m, params)\r
@@ -641,16 +657,18 @@ class RSSFeedsPlugin < Plugin
     end\r
     status = Hash.new\r
     status[:failures] = 0\r
+    status[:first_run] = true\r
     @watch[feed.handle] = @bot.timer.add(0, status) {\r
       debug "watcher for #{feed} started"\r
       failures = status[:failures]\r
+      first_run = status.delete(:first_run)\r
       begin\r
         debug "fetching #{feed}"\r
         oldxml = feed.xml ? feed.xml.dup : nil\r
         unless fetchRss(feed)\r
           failures += 1\r
         else\r
-          if oldxml and oldxml == feed.xml\r
+          if first_run or (oldxml and oldxml == feed.xml)\r
             debug "xml for #{feed} didn't change"\r
             failures -= 1 if failures > 0\r
           else\r
@@ -659,16 +677,40 @@ class RSSFeedsPlugin < Plugin
               parseRss(feed)\r
               failures -= 1 if failures > 0\r
             else\r
-              otxt = feed.items.map { |item| item.to_s }\r
+              # This one is used for debugging\r
+              otxt = []\r
+\r
+              # These are used for checking new items vs old ones\r
+              uid_opts = { :show_updated => @bot.config['rss.show_updated'] }\r
+              oids = Set.new feed.items.map { |item|\r
+                uid = RSS.item_uid_for_bot(item, uid_opts)\r
+                otxt << item.to_s\r
+                debug [uid, otxt.last].inspect\r
+                uid\r
+              }\r
+\r
               unless parseRss(feed)\r
                 debug "no items in feed #{feed}"\r
                 failures += 1\r
               else\r
                 debug "Checking if new items are available for #{feed}"\r
                 failures -= 1 if failures > 0\r
+\r
                 dispItems = feed.items.reject { |item|\r
-                  otxt.include?(item.to_s)\r
+                  uid = RSS.item_uid_for_bot(item, uid_opts)\r
+                  txt = item.to_s\r
+                  if oids.include?(uid)\r
+                    debug "rejecting old #{uid} #{item.inspect}"\r
+                    debug [uid, txt].inspect\r
+                    true\r
+                  else\r
+                    debug "accepting new #{uid} #{item.inspect}"\r
+                    debug [uid, txt].inspect\r
+                    warn "same text! #{txt}" if otxt.include?(txt)\r
+                    false\r
+                  end\r
                 }\r
+\r
                 if dispItems.length > 0\r
                   debug "Found #{dispItems.length} new items in #{feed}"\r
                   # When displaying watched feeds, publish them from older to newer\r
@@ -727,9 +769,9 @@ class RSSFeedsPlugin < Plugin
       end\r
     end\r
 \r
-    title = "#{Bold}#{item.title.chomp.riphtml}#{Bold}" if item.title\r
+    title = "#{Bold}#{item.title.ircify_html}#{Bold}" if item.title\r
 \r
-    desc = item.description.gsub(/\s+/,' ').strip.riphtml if item.description\r
+    desc = item.description.ircify_html(:a_href => :link_out) if item.description\r
 \r
     link = item.link.chomp if item.link\r
 \r
@@ -772,12 +814,13 @@ class RSSFeedsPlugin < Plugin
     }\r
   end\r
 \r
-  def fetchRss(feed, m=nil)\r
+  def fetchRss(feed, m=nil, cache=true)\r
     begin\r
       # Use 60 sec timeout, cause the default is too low\r
       xml = @bot.httputil.get(feed.url,\r
                               :read_timeout => 60,\r
-                              :open_timeout => 60)\r
+                              :open_timeout => 60,\r
+                              :cache => cache)\r
     rescue URI::InvalidURIError, URI::BadURIError => e\r
       report_problem("invalid rss feed #{feed.url}", e, m)\r
       return nil\r