X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=data%2Frbot%2Fplugins%2Frss.rb;h=6b5e62be8030e2d8cf9ec658b7c5776e720e2a60;hb=386274dd8f83d4aac27e837f1dca11f0f9250ee8;hp=ff326d52b84ad44a7016d0343e8842bc41bac90e;hpb=012d590a4efc503642cbb7c264741e523db1d366;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git

diff --git a/data/rbot/plugins/rss.rb b/data/rbot/plugins/rss.rb
index ff326d52..6b5e62be 100644
--- a/data/rbot/plugins/rss.rb
+++ b/data/rbot/plugins/rss.rb
@@ -19,19 +19,18 @@ require 'rss/dublincore'
 #   warning "Unable to load RSS libraries, RSS plugin functionality crippled"
 # end
 
-class ::String
-  def riphtml
-    self.gsub(/<[^>]+>/, '').gsub(/&amp;/,'&').gsub(/&quot;/,'"').gsub(/&lt;/,'<').gsub(/&gt;/,'>').gsub(/&ellip;/,'...').gsub(/&apos;/, "'").gsub("\n",'')
-  end
-end
-
 class ::RssBlob
-  attr :url
-  attr :handle
-  attr :type
+  attr_accessor :url
+  attr_accessor :handle
+  attr_accessor :type
   attr :watchers
+  attr_accessor :refresh_rate
+  attr_accessor :xml
+  attr_accessor :title
+  attr_accessor :items
+  attr_accessor :mutex
 
-  def initialize(url,handle=nil,type=nil,watchers=[])
+  def initialize(url,handle=nil,type=nil,watchers=[], xml=nil)
     @url = url
     if handle
       @handle = handle
@@ -40,9 +39,22 @@ class ::RssBlob
     end
     @type = type
     @watchers=[]
+    @refresh_rate = nil
+    @xml = xml
+    @title = nil
+    @items = nil
+    @mutex = Mutex.new
     sanitize_watchers(watchers)
   end
 
+  def dup
+    self.class.new(@url,
+                   @handle,
+                   @type ? @type.dup : nil,
+                   @watchers.dup,
+                   @xml ? @xml.dup : nil)
+  end
+
   # Downcase all watchers, possibly turning them into Strings if they weren't
   def sanitize_watchers(list=@watchers)
     ls = list.dup
@@ -73,7 +85,7 @@ class ::RssBlob
   end
 
   def to_a
-    [@handle,@url,@type,@watchers]
+    [@handle,@url,@type,@refresh_rate,@watchers]
   end
 
   def to_s(watchers=false)
@@ -108,7 +120,9 @@ class RSSFeedsPlugin < Plugin
         @feeds.delete(k)
       }
       @feeds.each { |k, f|
+        f.mutex = Mutex.new unless f.mutex
         f.sanitize_watchers
+        parseRss(f) if f.xml
       }
     else
       @feeds = Hash.new
@@ -130,7 +144,13 @@ class RSSFeedsPlugin < Plugin
   end
 
   def save
-    @registry[:feeds] = @feeds
+    unparsed = Hash.new()
+    @feeds.each { |k, f|
+      f.mutex.synchronize do
+        unparsed[k] = f.dup
+      end
+    }
+    @registry[:feeds] = unparsed
   end
 
   def stop_watch(handle)
@@ -158,9 +178,13 @@ class RSSFeedsPlugin < Plugin
     when "list"
       "rss list [#{Bold}handle#{Bold}] : list all rss feeds (matching #{Bold}handle#{Bold})"
     when "watched"
-      "rss watched [#{Bold}handle#{Bold}] : list all watched rss feeds (matching #{Bold}handle#{Bold})"
+      "rss watched [#{Bold}handle#{Bold}] [in #{Bold}chan#{Bold}]: list all watched rss feeds (matching #{Bold}handle#{Bold}) (in channel #{Bold}chan#{Bold})"
+    when "who", "watches", "who watches"
+      "rss who watches [#{Bold}handle#{Bold}]]: list all watchers for rss feeds (matching #{Bold}handle#{Bold})"
     when "add"
       "rss add #{Bold}handle#{Bold} #{Bold}url#{Bold} [#{Bold}type#{Bold}] : add a new rss called #{Bold}handle#{Bold} from url #{Bold}url#{Bold} (of type #{Bold}type#{Bold})"
+    when "change"
+      "rss change #{Bold}what#{Bold} of #{Bold}handle#{Bold} to #{Bold}new#{Bold} : change the #{Underline}handle#{Underline}, #{Underline}url#{Underline}, #{Underline}type#{Underline} or #{Underline}refresh#{Underline} rate of rss called #{Bold}handle#{Bold} to value #{Bold}new#{Bold}"
     when /^(del(ete)?|rm)$/
       "rss del(ete)|rm #{Bold}handle#{Bold} : delete rss feed #{Bold}handle#{Bold}"
     when "replace"
@@ -168,13 +192,13 @@ class RSSFeedsPlugin < Plugin
     when "forcereplace"
       "rss forcereplace #{Bold}handle#{Bold} #{Bold}url#{Bold} [#{Bold}type#{Bold}] : replace the url of rss called #{Bold}handle#{Bold} with #{Bold}url#{Bold} (of type #{Bold}type#{Bold})"
     when "watch"
-      "rss watch #{Bold}handle#{Bold} [#{Bold}url#{Bold} [#{Bold}type#{Bold}]] : watch rss #{Bold}handle#{Bold} for changes; when the other parameters are present, it will be created if it doesn't exist yet"
+      "rss watch #{Bold}handle#{Bold} [#{Bold}url#{Bold} [#{Bold}type#{Bold}]]  [in #{Bold}chan#{Bold}]: watch rss #{Bold}handle#{Bold} for changes (in channel #{Bold}chan#{Bold}); when the other parameters are present, the feed will be created if it doesn't exist yet"
     when /(un|rm)watch/
-      "rss unwatch|rmwatch #{Bold}handle#{Bold} : stop watching rss #{Bold}handle#{Bold} for changes"
+      "rss unwatch|rmwatch #{Bold}handle#{Bold} [in #{Bold}chan#{Bold}]: stop watching rss #{Bold}handle#{Bold} (in channel #{Bold}chan#{Bold}) for changes"
     when "rewatch"
       "rss rewatch : restart threads that watch for changes in watched rss"
     else
-      "manage RSS feeds: rss show|list|watched|add|del(ete)|rm|(force)replace|watch|unwatch|rmwatch|rewatch"
+      "manage RSS feeds: rss show|list|watched|add|change|del(ete)|rm|(force)replace|watch|unwatch|rmwatch|rewatch"
     end
   end
 
@@ -212,8 +236,17 @@ class RSSFeedsPlugin < Plugin
 
     m.reply "lemme fetch it..."
     title = items = nil
-    title, items = fetchRss(feed, m)
-    return unless items
+    fetched = fetchRss(feed, m)
+    return unless fetched or feed.xml
+    if not fetched and feed.items
+      m.reply "using old data"
+    else
+      parsed = parseRss(feed, m)
+      m.reply "using old data" unless parsed
+    end
+    return unless feed.items
+    title = feed.title
+    items = feed.items
 
     # We sort the feeds in freshness order (newer ones first)
     items = freshness_sort(items)
@@ -245,6 +278,7 @@ class RSSFeedsPlugin < Plugin
     @feeds.each { |handle, feed|
       next if wanted and !handle.match(/#{wanted}/i)
       reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})"
+      (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate
       (reply << " (watched)") if feed.watched_by?(m.replyto)
       reply << "\n"
     }
@@ -256,12 +290,32 @@ class RSSFeedsPlugin < Plugin
   end
 
   def watched_rss(m, params)
+    wanted = params[:handle]
+    chan = params[:chan] || m.replyto
+    reply = String.new
+    watchlist.each { |handle, feed|
+      next if wanted and !handle.match(/#{wanted}/i)
+      next unless feed.watched_by?(chan)
+      reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})"
+      (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate
+      reply << "\n"
+    }
+    if reply.empty?
+      reply = "no watched feeds"
+      reply << " matching #{wanted}" if wanted
+    end
+    m.reply reply
+  end
+
+  def who_watches(m, params)
     wanted = params[:handle]
     reply = String.new
     watchlist.each { |handle, feed|
       next if wanted and !handle.match(/#{wanted}/i)
-      next unless feed.watched_by?(m.replyto)
-      reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})\n"
+      reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})"
+      (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate
+      reply << ": watched by #{feed.watchers.join(', ')}"
+      reply << "\n"
     }
     if reply.empty?
       reply = "no watched feeds"
@@ -295,6 +349,52 @@ class RSSFeedsPlugin < Plugin
     return handle
   end
 
+  def change_rss(m, params)
+    handle = params[:handle].downcase
+    feed = @feeds.fetch(handle, nil)
+    unless feed
+      m.reply "No such feed with handle #{handle}"
+      return
+    end
+    case params[:what].intern
+    when :handle
+      new = params[:new].downcase
+      if @feeds.key?(new) and @feeds[new]
+        m.reply "There already is a feed with handle #{new}"
+        return
+      else
+        feed.mutex.synchronize do
+          @feeds[new] = feed
+          @feeds.delete(handle)
+          feed.handle = new
+        end
+        handle = new
+      end
+    when :url
+      new = params[:new]
+      feed.mutex.synchronize do
+        feed.url = new
+      end
+    when :format, :type
+      new = params[:new]
+      new = nil if new == 'default'
+      feed.mutex.synchronize do
+        feed.type = new
+      end
+    when :refresh
+      new = params[:new].to_i
+      new = nil if new == 0
+      feed.mutex.synchronize do
+        feed.refresh_rate = new
+      end
+    else
+      m.reply "Don't know how to change #{params[:what]} for feeds"
+      return
+    end
+    m.reply "Feed changed:"
+    list_rss(m, {:handle => handle})
+  end
+
   def del_rss(m, params, pass=false)
     feed = unwatch_rss(m, params, true)
     if feed.watched?
@@ -324,6 +424,7 @@ class RSSFeedsPlugin < Plugin
 
   def watch_rss(m, params)
     handle = params[:handle]
+    chan = params[:chan] || m.replyto
     url = params[:url]
     type = params[:type]
     if url
@@ -331,11 +432,11 @@ class RSSFeedsPlugin < Plugin
     end
     feed = @feeds.fetch(handle.downcase, nil)
     if feed
-      if feed.add_watch(m.replyto)
+      if feed.add_watch(chan)
         watchRss(feed, m)
         m.okay
       else
-        m.reply "Already watching #{feed.handle}"
+        m.reply "Already watching #{feed.handle} in #{chan}"
       end
     else
       m.reply "Couldn't watch feed #{handle} (no such feed found)"
@@ -344,15 +445,16 @@ class RSSFeedsPlugin < Plugin
 
   def unwatch_rss(m, params, pass=false)
     handle = params[:handle].downcase
+    chan = params[:chan] || m.replyto
     unless @feeds.has_key?(handle)
       m.reply("dunno that feed")
       return
     end
     feed = @feeds[handle]
-    if feed.rm_watch(m.replyto)
-      m.reply "#{m.replyto} has been removed from the watchlist for #{feed.handle}"
+    if feed.rm_watch(chan)
+      m.reply "#{chan} has been removed from the watchlist for #{feed.handle}"
     else
-      m.reply("#{m.replyto} wasn't watching #{feed.handle}") unless pass
+      m.reply("#{chan} wasn't watching #{feed.handle}") unless pass
     end
     if !feed.watched?
       stop_watch(handle)
@@ -377,42 +479,47 @@ class RSSFeedsPlugin < Plugin
       return
     end
     status = Hash.new
-    status[:oldItems] = []
-    status[:firstRun] = true
     status[:failures] = 0
     @watch[feed.handle] = @bot.timer.add(0, status) {
       debug "watcher for #{feed} started"
-      oldItems = status[:oldItems]
-      firstRun = status[:firstRun]
       failures = status[:failures]
       begin
         debug "fetching #{feed}"
-        title = newItems = nil
-        title, newItems = fetchRss(feed)
-        unless newItems
-          debug "no items in feed #{feed}"
-          failures +=1
+        oldxml = feed.xml ? feed.xml.dup : nil
+        unless fetchRss(feed)
+          failures += 1
         else
-          debug "Checking if new items are available for #{feed}"
-          if firstRun
-            debug "First run, we'll see next time"
-            firstRun = false
+          if oldxml and oldxml == feed.xml
+            debug "xml for #{feed} didn't change"
+            failures -= 1 if failures > 0
           else
-            otxt = oldItems.map { |item| item.to_s }
-            dispItems = newItems.reject { |item|
-              otxt.include?(item.to_s)
-            }
-            if dispItems.length > 0
-              debug "Found #{dispItems.length} new items in #{feed}"
-              # When displaying watched feeds, publish them from older to newer
-              dispItems.reverse.each { |item|
-                printFormattedRss(feed, item)
-              }
+            if not feed.items
+              debug "no previous items in feed #{feed}"
+              parseRss(feed)
+              failures -= 1 if failures > 0
             else
-              debug "No new items found in #{feed}"
+              otxt = feed.items.map { |item| item.to_s }
+              unless parseRss(feed)
+                debug "no items in feed #{feed}"
+                failures += 1
+              else
+                debug "Checking if new items are available for #{feed}"
+                failures -= 1 if failures > 0
+                dispItems = feed.items.reject { |item|
+                  otxt.include?(item.to_s)
+                }
+                if dispItems.length > 0
+                  debug "Found #{dispItems.length} new items in #{feed}"
+                  # When displaying watched feeds, publish them from older to newer
+                  dispItems.reverse.each { |item|
+                    printFormattedRss(feed, item)
+                  }
+                else
+                  debug "No new items found in #{feed}"
+                end
+              end
             end
           end
-          oldItems = newItems.dup
         end
       rescue Exception => e
         error "Error watching #{feed}: #{e.inspect}"
@@ -420,14 +527,14 @@ class RSSFeedsPlugin < Plugin
         failures += 1
       end
 
-      status[:oldItems] = oldItems
-      status[:firstRun] = firstRun
       status[:failures] = failures
 
-      seconds = @bot.config['rss.thread_sleep'] * (failures + 1)
-      seconds += seconds * (rand(100)-50)/100
-      debug "watcher for #{feed} going to sleep #{seconds} seconds.."
-      @bot.timer.reschedule(@watch[feed.handle], seconds)
+      feed.mutex.synchronize do
+        seconds = (feed.refresh_rate || @bot.config['rss.thread_sleep']) * (failures + 1)
+        seconds += seconds * (rand(100)-50)/100
+        debug "watcher for #{feed} going to sleep #{seconds} seconds.."
+        @bot.timer.reschedule(@watch[feed.handle], seconds)
+      end
     }
     debug "watcher for #{feed} added"
   end
@@ -494,59 +601,71 @@ class RSSFeedsPlugin < Plugin
       xml = @bot.httputil.get_cached(feed.url, 60, 60)
     rescue URI::InvalidURIError, URI::BadURIError => e
       report_problem("invalid rss feed #{feed.url}", e, m)
-      return
+      return nil
     rescue => e
       report_problem("error getting #{feed.url}", e, m)
-      return
+      return nil
     end
     debug "fetched #{feed}"
     unless xml
       report_problem("reading feed #{feed} failed", nil, m)
-      return
+      return nil
+    end
+    feed.mutex.synchronize do
+      feed.xml = xml
     end
+    return true
+  end
 
-    begin
-      ## do validate parse
-      rss = RSS::Parser.parse(xml)
-      debug "parsed #{feed}"
-    rescue RSS::InvalidRSSError
-      ## do non validate parse for invalid RSS 1.0
+  def parseRss(feed, m=nil)
+    return nil unless feed.xml
+    feed.mutex.synchronize do
+      xml = feed.xml
       begin
-        rss = RSS::Parser.parse(xml, false)
+        ## do validate parse
+        rss = RSS::Parser.parse(xml)
+        debug "parsed #{feed}"
+      rescue RSS::InvalidRSSError
+        ## do non validate parse for invalid RSS 1.0
+        begin
+          rss = RSS::Parser.parse(xml, false)
+        rescue RSS::Error => e
+          report_problem("parsing rss stream failed, whoops =(", e, m)
+          return nil
+        end
       rescue RSS::Error => e
-        report_problem("parsing rss stream failed, whoops =(", e, m)
-        return
-      end
-    rescue RSS::Error => e
-      report_problem("parsing rss stream failed, oioi", e, m)
-      return
-    rescue => e
-      report_problem("processing error occured, sorry =(", e, m)
-      return
-    end
-    items = []
-    if rss.nil?
-      report_problem("#{feed} does not include RSS 1.0 or 0.9x/2.0", nil, m)
-    else
-      begin
-        rss.output_encoding = 'UTF-8'
-      rescue RSS::UnknownConvertMethod => e
-        report_problem("bah! something went wrong =(", e, m)
-        return
+        report_problem("parsing rss stream failed, oioi", e, m)
+        return nil
+      rescue => e
+        report_problem("processing error occured, sorry =(", e, m)
+        return nil
       end
-      rss.channel.title ||= "Unknown"
-      title = rss.channel.title
-      rss.items.each do |item|
-        item.title ||= "Unknown"
-        items << item
+      items = []
+      if rss.nil?
+        report_problem("#{feed} does not include RSS 1.0 or 0.9x/2.0", nil, m)
+      else
+        begin
+          rss.output_encoding = 'UTF-8'
+        rescue RSS::UnknownConvertMethod => e
+          report_problem("bah! something went wrong =(", e, m)
+          return nil
+        end
+        rss.channel.title ||= "Unknown"
+        title = rss.channel.title
+        rss.items.each do |item|
+          item.title ||= "Unknown"
+          items << item
+        end
       end
-    end
 
-    if items.empty?
-      report_problem("no items found in the feed, maybe try weed?", e, m)
-      return
+      if items.empty?
+        report_problem("no items found in the feed, maybe try weed?", e, m)
+        return nil
+      end
+      feed.title = title
+      feed.items = items
+      return true
     end
-    return [title, items]
   end
 end
 
@@ -558,13 +677,22 @@ plugin.map 'rss show :handle :limit',
   :defaults => {:limit => 5}
 plugin.map 'rss list :handle',
   :action => 'list_rss',
-  :defaults =>  {:handle => nil}
-plugin.map 'rss watched :handle',
+  :defaults => {:handle => nil}
+plugin.map 'rss watched :handle [in :chan]',
   :action => 'watched_rss',
-  :defaults =>  {:handle => nil}
+  :defaults => {:handle => nil}
+plugin.map 'rss who watches :handle',
+  :action => 'who_watches',
+  :defaults => {:handle => nil}
 plugin.map 'rss add :handle :url :type',
   :action => 'add_rss',
   :defaults => {:type => nil}
+plugin.map 'rss change :what of :handle to :new',
+  :action => 'change_rss',
+  :requirements => { :what => /handle|url|format|type|refresh/ }
+plugin.map 'rss change :what for :handle to :new',
+  :action => 'change_rss',
+  :requirements => { :what => /handle|url|format|type|refesh/ }
 plugin.map 'rss del :handle',
   :action => 'del_rss'
 plugin.map 'rss delete :handle',
@@ -577,12 +705,12 @@ plugin.map 'rss replace :handle :url :type',
 plugin.map 'rss forcereplace :handle :url :type',
   :action => 'forcereplace_rss',
   :defaults => {:type => nil}
-plugin.map 'rss watch :handle :url :type',
+plugin.map 'rss watch :handle :url :type [in :chan]',
   :action => 'watch_rss',
   :defaults => {:url => nil, :type => nil}
-plugin.map 'rss unwatch :handle',
+plugin.map 'rss unwatch :handle [in :chan]',
   :action => 'unwatch_rss'
-plugin.map 'rss rmwatch :handle',
+plugin.map 'rss rmwatch :handle [in :chan]',
   :action => 'unwatch_rss'
 plugin.map 'rss rewatch',
   :action => 'rewatch_rss'