X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=data%2Frbot%2Fplugins%2Frss.rb;h=6b5e62be8030e2d8cf9ec658b7c5776e720e2a60;hb=386274dd8f83d4aac27e837f1dca11f0f9250ee8;hp=745dad0d87fb4de81cce115ea08c5ef2da105e2c;hpb=c7905d630c15e9f71b4585088a8a37706f599b84;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git diff --git a/data/rbot/plugins/rss.rb b/data/rbot/plugins/rss.rb index 745dad0d..6b5e62be 100644 --- a/data/rbot/plugins/rss.rb +++ b/data/rbot/plugins/rss.rb @@ -19,27 +19,18 @@ require 'rss/dublincore' # warning "Unable to load RSS libraries, RSS plugin functionality crippled" # end -class ::String - def shorten(limit) - if self.length > limit - self+". " =~ /^(.{#{limit}}[^.!;?]*[.!;?])/mi - return $1 - end - self - end - - def riphtml - self.gsub(/<[^>]+>/, '').gsub(/&/,'&').gsub(/"/,'"').gsub(/</,'<').gsub(/>/,'>').gsub(/&ellip;/,'...').gsub(/'/, "'").gsub("\n",'') - end -end - class ::RssBlob - attr :url - attr :handle - attr :type + attr_accessor :url + attr_accessor :handle + attr_accessor :type attr :watchers + attr_accessor :refresh_rate + attr_accessor :xml + attr_accessor :title + attr_accessor :items + attr_accessor :mutex - def initialize(url,handle=nil,type=nil,watchers=[]) + def initialize(url,handle=nil,type=nil,watchers=[], xml=nil) @url = url if handle @handle = handle @@ -48,9 +39,22 @@ class ::RssBlob end @type = type @watchers=[] + @refresh_rate = nil + @xml = xml + @title = nil + @items = nil + @mutex = Mutex.new sanitize_watchers(watchers) end + def dup + self.class.new(@url, + @handle, + @type ? @type.dup : nil, + @watchers.dup, + @xml ? @xml.dup : nil) + end + # Downcase all watchers, possibly turning them into Strings if they weren't def sanitize_watchers(list=@watchers) ls = list.dup @@ -81,7 +85,7 @@ class ::RssBlob end def to_a - [@handle,@url,@type,@watchers] + [@handle,@url,@type,@refresh_rate,@watchers] end def to_s(watchers=false) @@ -107,12 +111,8 @@ class RSSFeedsPlugin < Plugin :default => 300, :validate => Proc.new{|v| v > 30}, :desc => "How many seconds to sleep before checking RSS feeds again") - @@watchThreads = Hash.new - @@mutex = Mutex.new - def initialize super - kill_threads if @registry.has_key?(:feeds) @feeds = @registry[:feeds] @feeds.keys.grep(/[A-Z]/) { |k| @@ -120,11 +120,14 @@ class RSSFeedsPlugin < Plugin @feeds.delete(k) } @feeds.each { |k, f| + f.mutex = Mutex.new unless f.mutex f.sanitize_watchers + parseRss(f) if f.xml } else @feeds = Hash.new end + @watch = Hash.new rewatch_rss end @@ -137,21 +140,34 @@ class RSSFeedsPlugin < Plugin end def cleanup - kill_threads + stop_watches end def save - @registry[:feeds] = @feeds + unparsed = Hash.new() + @feeds.each { |k, f| + f.mutex.synchronize do + unparsed[k] = f.dup + end + } + @registry[:feeds] = unparsed end - def kill_threads - @@mutex.synchronize { - # Abort all running threads. - @@watchThreads.each { |url, thread| - debug "Killing thread for #{url}" - thread.kill - } - @@watchThreads = Hash.new + def stop_watch(handle) + if @watch.has_key?(handle) + begin + debug "Stopping watch #{handle}" + @bot.timer.remove(@watch[handle]) + @watch.delete(handle) + rescue => e + report_problem("Failed to stop watch for #{handle}", e, nil) + end + end + end + + def stop_watches + @watch.each_key { |k| + stop_watch(k) } end @@ -162,9 +178,13 @@ class RSSFeedsPlugin < Plugin when "list" "rss list [#{Bold}handle#{Bold}] : list all rss feeds (matching #{Bold}handle#{Bold})" when "watched" - "rss watched [#{Bold}handle#{Bold}] : list all watched rss feeds (matching #{Bold}handle#{Bold})" + "rss watched [#{Bold}handle#{Bold}] [in #{Bold}chan#{Bold}]: list all watched rss feeds (matching #{Bold}handle#{Bold}) (in channel #{Bold}chan#{Bold})" + when "who", "watches", "who watches" + "rss who watches [#{Bold}handle#{Bold}]]: list all watchers for rss feeds (matching #{Bold}handle#{Bold})" when "add" "rss add #{Bold}handle#{Bold} #{Bold}url#{Bold} [#{Bold}type#{Bold}] : add a new rss called #{Bold}handle#{Bold} from url #{Bold}url#{Bold} (of type #{Bold}type#{Bold})" + when "change" + "rss change #{Bold}what#{Bold} of #{Bold}handle#{Bold} to #{Bold}new#{Bold} : change the #{Underline}handle#{Underline}, #{Underline}url#{Underline}, #{Underline}type#{Underline} or #{Underline}refresh#{Underline} rate of rss called #{Bold}handle#{Bold} to value #{Bold}new#{Bold}" when /^(del(ete)?|rm)$/ "rss del(ete)|rm #{Bold}handle#{Bold} : delete rss feed #{Bold}handle#{Bold}" when "replace" @@ -172,13 +192,13 @@ class RSSFeedsPlugin < Plugin when "forcereplace" "rss forcereplace #{Bold}handle#{Bold} #{Bold}url#{Bold} [#{Bold}type#{Bold}] : replace the url of rss called #{Bold}handle#{Bold} with #{Bold}url#{Bold} (of type #{Bold}type#{Bold})" when "watch" - "rss watch #{Bold}handle#{Bold} [#{Bold}url#{Bold} [#{Bold}type#{Bold}]] : watch rss #{Bold}handle#{Bold} for changes; when the other parameters are present, it will be created if it doesn't exist yet" + "rss watch #{Bold}handle#{Bold} [#{Bold}url#{Bold} [#{Bold}type#{Bold}]] [in #{Bold}chan#{Bold}]: watch rss #{Bold}handle#{Bold} for changes (in channel #{Bold}chan#{Bold}); when the other parameters are present, the feed will be created if it doesn't exist yet" when /(un|rm)watch/ - "rss unwatch|rmwatch #{Bold}handle#{Bold} : stop watching rss #{Bold}handle#{Bold} for changes" + "rss unwatch|rmwatch #{Bold}handle#{Bold} [in #{Bold}chan#{Bold}]: stop watching rss #{Bold}handle#{Bold} (in channel #{Bold}chan#{Bold}) for changes" when "rewatch" "rss rewatch : restart threads that watch for changes in watched rss" else - "manage RSS feeds: rss show|list|watched|add|del(ete)|rm|(force)replace|watch|unwatch|rmwatch|rewatch" + "manage RSS feeds: rss show|list|watched|add|change|del(ete)|rm|(force)replace|watch|unwatch|rmwatch|rewatch" end end @@ -216,10 +236,17 @@ class RSSFeedsPlugin < Plugin m.reply "lemme fetch it..." title = items = nil - @@mutex.synchronize { - title, items = fetchRss(feed, m) - } - return unless items + fetched = fetchRss(feed, m) + return unless fetched or feed.xml + if not fetched and feed.items + m.reply "using old data" + else + parsed = parseRss(feed, m) + m.reply "using old data" unless parsed + end + return unless feed.items + title = feed.title + items = feed.items # We sort the feeds in freshness order (newer ones first) items = freshness_sort(items) @@ -248,13 +275,12 @@ class RSSFeedsPlugin < Plugin def list_rss(m, params) wanted = params[:handle] reply = String.new - @@mutex.synchronize { - @feeds.each { |handle, feed| - next if wanted and !handle.match(/#{wanted}/i) - reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" - (reply << " (watched)") if feed.watched_by?(m.replyto) - reply << "\n" - } + @feeds.each { |handle, feed| + next if wanted and !handle.match(/#{wanted}/i) + reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" + (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate + (reply << " (watched)") if feed.watched_by?(m.replyto) + reply << "\n" } if reply.empty? reply = "no feeds found" @@ -265,13 +291,31 @@ class RSSFeedsPlugin < Plugin def watched_rss(m, params) wanted = params[:handle] + chan = params[:chan] || m.replyto reply = String.new - @@mutex.synchronize { - watchlist.each { |handle, feed| - next if wanted and !handle.match(/#{wanted}/i) - next unless feed.watched_by?(m.replyto) - reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})\n" - } + watchlist.each { |handle, feed| + next if wanted and !handle.match(/#{wanted}/i) + next unless feed.watched_by?(chan) + reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" + (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate + reply << "\n" + } + if reply.empty? + reply = "no watched feeds" + reply << " matching #{wanted}" if wanted + end + m.reply reply + end + + def who_watches(m, params) + wanted = params[:handle] + reply = String.new + watchlist.each { |handle, feed| + next if wanted and !handle.match(/#{wanted}/i) + reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" + (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate + reply << ": watched by #{feed.watchers.join(', ')}" + reply << "\n" } if reply.empty? reply = "no watched feeds" @@ -296,9 +340,7 @@ class RSSFeedsPlugin < Plugin m.reply "You must specify both a handle and an url to add an RSS feed" return end - @@mutex.synchronize { - @feeds[handle.downcase] = RssBlob.new(url,handle,type) - } + @feeds[handle.downcase] = RssBlob.new(url,handle,type) reply = "Added RSS #{url} named #{handle}" if type reply << " (format: #{type})" @@ -307,15 +349,59 @@ class RSSFeedsPlugin < Plugin return handle end + def change_rss(m, params) + handle = params[:handle].downcase + feed = @feeds.fetch(handle, nil) + unless feed + m.reply "No such feed with handle #{handle}" + return + end + case params[:what].intern + when :handle + new = params[:new].downcase + if @feeds.key?(new) and @feeds[new] + m.reply "There already is a feed with handle #{new}" + return + else + feed.mutex.synchronize do + @feeds[new] = feed + @feeds.delete(handle) + feed.handle = new + end + handle = new + end + when :url + new = params[:new] + feed.mutex.synchronize do + feed.url = new + end + when :format, :type + new = params[:new] + new = nil if new == 'default' + feed.mutex.synchronize do + feed.type = new + end + when :refresh + new = params[:new].to_i + new = nil if new == 0 + feed.mutex.synchronize do + feed.refresh_rate = new + end + else + m.reply "Don't know how to change #{params[:what]} for feeds" + return + end + m.reply "Feed changed:" + list_rss(m, {:handle => handle}) + end + def del_rss(m, params, pass=false) feed = unwatch_rss(m, params, true) if feed.watched? m.reply "someone else is watching #{feed.handle}, I won't remove it from my list" return end - @@mutex.synchronize { - @feeds.delete(feed.handle.downcase) - } + @feeds.delete(feed.handle.downcase) m.okay unless pass return end @@ -338,24 +424,20 @@ class RSSFeedsPlugin < Plugin def watch_rss(m, params) handle = params[:handle] + chan = params[:chan] || m.replyto url = params[:url] type = params[:type] if url add_rss(m, params) end - feed = nil - @@mutex.synchronize { - feed = @feeds.fetch(handle.downcase, nil) - } + feed = @feeds.fetch(handle.downcase, nil) if feed - @@mutex.synchronize { - if feed.add_watch(m.replyto) - watchRss(feed, m) - m.okay - else - m.reply "Already watching #{feed.handle}" - end - } + if feed.add_watch(chan) + watchRss(feed, m) + m.okay + else + m.reply "Already watching #{feed.handle} in #{chan}" + end else m.reply "Couldn't watch feed #{handle} (no such feed found)" end @@ -363,30 +445,25 @@ class RSSFeedsPlugin < Plugin def unwatch_rss(m, params, pass=false) handle = params[:handle].downcase + chan = params[:chan] || m.replyto unless @feeds.has_key?(handle) m.reply("dunno that feed") return end feed = @feeds[handle] - if feed.rm_watch(m.replyto) - m.reply "#{m.replyto} has been removed from the watchlist for #{feed.handle}" + if feed.rm_watch(chan) + m.reply "#{chan} has been removed from the watchlist for #{feed.handle}" else - m.reply("#{m.replyto} wasn't watching #{feed.handle}") unless pass + m.reply("#{chan} wasn't watching #{feed.handle}") unless pass end if !feed.watched? - @@mutex.synchronize { - if @@watchThreads[handle].kind_of? Thread - @@watchThreads[handle].kill - debug "rmwatch: Killed thread for #{handle}" - @@watchThreads.delete(handle) - end - } + stop_watch(handle) end return feed end def rewatch_rss(m=nil, params=nil) - kill_threads + stop_watches # Read watches from list. watchlist.each{ |handle, feed| @@ -397,61 +474,69 @@ class RSSFeedsPlugin < Plugin private def watchRss(feed, m=nil) - if @@watchThreads.has_key?(feed.handle) + if @watch.has_key?(feed.handle) report_problem("watcher thread for #{feed.handle} is already running", nil, m) return end - @@watchThreads[feed.handle] = Thread.new do + status = Hash.new + status[:failures] = 0 + @watch[feed.handle] = @bot.timer.add(0, status) { debug "watcher for #{feed} started" - oldItems = [] - firstRun = true - failures = 0 - loop do - begin - debug "fetching #{feed}" - title = newItems = nil - @@mutex.synchronize { - title, newItems = fetchRss(feed) - } - unless newItems - debug "no items in feed #{feed}" - failures +=1 + failures = status[:failures] + begin + debug "fetching #{feed}" + oldxml = feed.xml ? feed.xml.dup : nil + unless fetchRss(feed) + failures += 1 + else + if oldxml and oldxml == feed.xml + debug "xml for #{feed} didn't change" + failures -= 1 if failures > 0 else - debug "Checking if new items are available for #{feed}" - if firstRun - debug "First run, we'll see next time" - firstRun = false + if not feed.items + debug "no previous items in feed #{feed}" + parseRss(feed) + failures -= 1 if failures > 0 else - otxt = oldItems.map { |item| item.to_s } - dispItems = newItems.reject { |item| - otxt.include?(item.to_s) - } - if dispItems.length > 0 - debug "Found #{dispItems.length} new items in #{feed}" - # When displaying watched feeds, publish them from older to newer - dispItems.reverse.each { |item| - @@mutex.synchronize { + otxt = feed.items.map { |item| item.to_s } + unless parseRss(feed) + debug "no items in feed #{feed}" + failures += 1 + else + debug "Checking if new items are available for #{feed}" + failures -= 1 if failures > 0 + dispItems = feed.items.reject { |item| + otxt.include?(item.to_s) + } + if dispItems.length > 0 + debug "Found #{dispItems.length} new items in #{feed}" + # When displaying watched feeds, publish them from older to newer + dispItems.reverse.each { |item| printFormattedRss(feed, item) } - } - else - debug "No new items found in #{feed}" + else + debug "No new items found in #{feed}" + end end end - oldItems = newItems.dup end - rescue Exception => e - error "Error watching #{feed}: #{e.inspect}" - debug e.backtrace.join("\n") - failures += 1 end + rescue Exception => e + error "Error watching #{feed}: #{e.inspect}" + debug e.backtrace.join("\n") + failures += 1 + end + + status[:failures] = failures - seconds = @bot.config['rss.thread_sleep'] * (failures + 1) + feed.mutex.synchronize do + seconds = (feed.refresh_rate || @bot.config['rss.thread_sleep']) * (failures + 1) seconds += seconds * (rand(100)-50)/100 debug "watcher for #{feed} going to sleep #{seconds} seconds.." - sleep seconds + @bot.timer.reschedule(@watch[feed.handle], seconds) end - end + } + debug "watcher for #{feed} added" end def printFormattedRss(feed, item, opts=nil) @@ -481,91 +566,106 @@ class RSSFeedsPlugin < Plugin end end title = "#{Bold}#{item.title.chomp.riphtml}#{Bold}" if item.title - desc = item.description.gsub(/\s+/,' ').strip.riphtml.shorten(@bot.config['rss.text_max']) if item.description + desc = item.description.gsub(/\s+/,' ').strip.riphtml if item.description link = item.link.chomp if item.link - places.each { |loc| - case feed.type - when 'blog' - @bot.say loc, "#{handle}#{date}#{item.category.content} blogged at #{link}" - @bot.say loc, "#{handle}#{title} - #{desc}" - when 'forum' - @bot.say loc, "#{handle}#{date}#{title}#{' @ ' if item.title && item.link}#{link}" - when 'wiki' - @bot.say loc, "#{handle}#{date}#{item.title} has been edited by #{item.dc_creator}. #{desc} #{link}" - when 'gmame' - @bot.say loc, "#{handle}#{date}Message #{title} sent by #{item.dc_creator}. #{desc}" - when 'trac' - @bot.say loc, "#{handle}#{date}#{title} @ #{link}" - unless item.title =~ /^Changeset \[(\d+)\]/ - @bot.say loc, "#{handle}#{date}#{desc}" - end - else - @bot.say loc, "#{handle}#{date}#{title}#{' @ ' if item.title && item.link}#{link}" + line1 = nil + line2 = nil + case feed.type + when 'blog' + line1 = "#{handle}#{date}#{item.category.content} blogged at #{link}" + line2 = "#{handle}#{title} - #{desc}" + when 'forum' + line1 = "#{handle}#{date}#{title}#{' @ ' if item.title && item.link}#{link}" + when 'wiki' + line1 = "#{handle}#{date}#{title}#{' @ ' if item.title && item.link}#{link} has been edited by #{item.dc_creator}. #{desc}" + when 'gmane' + line1 = "#{handle}#{date}Message #{title} sent by #{item.dc_creator}. #{desc}" + when 'trac' + line1 = "#{handle}#{date}#{title} @ #{link}" + unless item.title =~ /^Changeset \[(\d+)\]/ + line2 = "#{handle}#{date}#{desc}" end + else + line1 = "#{handle}#{date}#{title}#{' @ ' if item.title && item.link}#{link}" + end + places.each { |loc| + @bot.say loc, line1, :overlong => :truncate + next unless line2 + @bot.say loc, line2, :overlong => :truncate } end def fetchRss(feed, m=nil) begin # Use 60 sec timeout, cause the default is too low - # Do not use get_cached for RSS until we have proper cache handling - # xml = @bot.httputil.get_cached(feed.url,60,60) xml = @bot.httputil.get_cached(feed.url, 60, 60) rescue URI::InvalidURIError, URI::BadURIError => e report_problem("invalid rss feed #{feed.url}", e, m) - return + return nil rescue => e report_problem("error getting #{feed.url}", e, m) - return + return nil end debug "fetched #{feed}" unless xml report_problem("reading feed #{feed} failed", nil, m) - return + return nil + end + feed.mutex.synchronize do + feed.xml = xml end + return true + end - begin - ## do validate parse - rss = RSS::Parser.parse(xml) - debug "parsed #{feed}" - rescue RSS::InvalidRSSError - ## do non validate parse for invalid RSS 1.0 + def parseRss(feed, m=nil) + return nil unless feed.xml + feed.mutex.synchronize do + xml = feed.xml begin - rss = RSS::Parser.parse(xml, false) + ## do validate parse + rss = RSS::Parser.parse(xml) + debug "parsed #{feed}" + rescue RSS::InvalidRSSError + ## do non validate parse for invalid RSS 1.0 + begin + rss = RSS::Parser.parse(xml, false) + rescue RSS::Error => e + report_problem("parsing rss stream failed, whoops =(", e, m) + return nil + end rescue RSS::Error => e - report_problem("parsing rss stream failed, whoops =(", e, m) - return - end - rescue RSS::Error => e - report_problem("parsing rss stream failed, oioi", e, m) - return - rescue => e - report_problem("processing error occured, sorry =(", e, m) - return - end - items = [] - if rss.nil? - report_problem("#{feed} does not include RSS 1.0 or 0.9x/2.0", nil, m) - else - begin - rss.output_encoding = 'UTF-8' - rescue RSS::UnknownConvertMethod => e - report_problem("bah! something went wrong =(", e, m) - return + report_problem("parsing rss stream failed, oioi", e, m) + return nil + rescue => e + report_problem("processing error occured, sorry =(", e, m) + return nil end - rss.channel.title ||= "Unknown" - title = rss.channel.title - rss.items.each do |item| - item.title ||= "Unknown" - items << item + items = [] + if rss.nil? + report_problem("#{feed} does not include RSS 1.0 or 0.9x/2.0", nil, m) + else + begin + rss.output_encoding = 'UTF-8' + rescue RSS::UnknownConvertMethod => e + report_problem("bah! something went wrong =(", e, m) + return nil + end + rss.channel.title ||= "Unknown" + title = rss.channel.title + rss.items.each do |item| + item.title ||= "Unknown" + items << item + end end - end - if items.empty? - report_problem("no items found in the feed, maybe try weed?", e, m) - return + if items.empty? + report_problem("no items found in the feed, maybe try weed?", e, m) + return nil + end + feed.title = title + feed.items = items + return true end - return [title, items] end end @@ -577,13 +677,22 @@ plugin.map 'rss show :handle :limit', :defaults => {:limit => 5} plugin.map 'rss list :handle', :action => 'list_rss', - :defaults => {:handle => nil} -plugin.map 'rss watched :handle', + :defaults => {:handle => nil} +plugin.map 'rss watched :handle [in :chan]', :action => 'watched_rss', - :defaults => {:handle => nil} + :defaults => {:handle => nil} +plugin.map 'rss who watches :handle', + :action => 'who_watches', + :defaults => {:handle => nil} plugin.map 'rss add :handle :url :type', :action => 'add_rss', :defaults => {:type => nil} +plugin.map 'rss change :what of :handle to :new', + :action => 'change_rss', + :requirements => { :what => /handle|url|format|type|refresh/ } +plugin.map 'rss change :what for :handle to :new', + :action => 'change_rss', + :requirements => { :what => /handle|url|format|type|refesh/ } plugin.map 'rss del :handle', :action => 'del_rss' plugin.map 'rss delete :handle', @@ -596,12 +705,12 @@ plugin.map 'rss replace :handle :url :type', plugin.map 'rss forcereplace :handle :url :type', :action => 'forcereplace_rss', :defaults => {:type => nil} -plugin.map 'rss watch :handle :url :type', +plugin.map 'rss watch :handle :url :type [in :chan]', :action => 'watch_rss', :defaults => {:url => nil, :type => nil} -plugin.map 'rss unwatch :handle', +plugin.map 'rss unwatch :handle [in :chan]', :action => 'unwatch_rss' -plugin.map 'rss rmwatch :handle', +plugin.map 'rss rmwatch :handle [in :chan]', :action => 'unwatch_rss' plugin.map 'rss rewatch', :action => 'rewatch_rss'