X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=data%2Frbot%2Fplugins%2Frss.rb;h=905daad81ffcfa49c36fc88ea484441c8a5bdbda;hb=ad4317c01095675546e5d907a917168d82a0580e;hp=7d9266044f54c4084722f38177205ae1471f1a01;hpb=c36e4355b60dfc02aaf73bb1b7e7a9c82bd59cb7;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git diff --git a/data/rbot/plugins/rss.rb b/data/rbot/plugins/rss.rb index 7d926604..905daad8 100644 --- a/data/rbot/plugins/rss.rb +++ b/data/rbot/plugins/rss.rb @@ -1,49 +1,160 @@ #-- vim:sw=2:et #++ # -# RSS feed plugin for RubyBot -# (c) 2004 Stanislav Karchebny -# (c) 2005 Ian Monroe -# (c) 2005 Mark Kretschmann -# (c) 2006 Giuseppe Bilotta +# :title: RSS feed plugin for rbot # -# Licensed under MIT License. +# Author:: Stanislav Karchebny +# Author:: Ian Monroe +# Author:: Mark Kretschmann +# Author:: Giuseppe Bilotta +# +# Copyright:: (C) 2004 Stanislav Karchebny +# Copyright:: (C) 2005 Ian Monroe, Mark Kretschmann +# Copyright:: (C) 2006-2007 Giuseppe Bilotta +# +# License:: MIT license + +require 'rss' + +module ::RSS + + # Make an 'unique' ID for a given item, based on appropriate bot options + # Currently only suppored is bot.config['rss.show_updated']: when true, the + # description is included in the uid hashing, otherwise it's not + # + def RSS.item_uid_for_bot(item, opts={}) + options = { :show_updated => true}.merge(opts) + desc = options[:show_updated] ? item.description : nil + [item.title, item.link, desc].hash + end + + # Add support for Slashdot namespace in RDF. The code is just an adaptation + # of the DublinCore code. + unless defined?(SLASH_PREFIX) + SLASH_PREFIX = 'slash' + SLASH_URI = "http://purl.org/rss/1.0/modules/slash/" + + RDF.install_ns(SLASH_PREFIX, SLASH_URI) + + module BaseSlashModel + def append_features(klass) + super + + return if klass.instance_of?(Module) + SlashModel::ELEMENT_NAME_INFOS.each do |name, plural_name| + plural = plural_name || "#{name}s" + full_name = "#{SLASH_PREFIX}_#{name}" + full_plural_name = "#{SLASH_PREFIX}_#{plural}" + klass_name = "Slash#{Utils.to_class_name(name)}" + klass.install_must_call_validator(SLASH_PREFIX, SLASH_URI) + klass.install_have_children_element(name, SLASH_URI, "*", + full_name, full_plural_name) + klass.module_eval(<<-EOC, *get_file_and_line_from_caller(0)) + remove_method :#{full_name} + remove_method :#{full_name}= + remove_method :set_#{full_name} + + def #{full_name} + @#{full_name}.first and @#{full_name}.first.value + end + + def #{full_name}=(new_value) + @#{full_name}[0] = Utils.new_with_value_if_need(#{klass_name}, new_value) + end + alias set_#{full_name} #{full_name}= + EOC + end + end + end + + module SlashModel + extend BaseModel + extend BaseSlashModel + + TEXT_ELEMENTS = { + "department" => nil, + "section" => nil, + "comments" => nil, + "hit_parade" => nil + } + + ELEMENT_NAME_INFOS = SlashModel::TEXT_ELEMENTS.to_a + + ELEMENTS = TEXT_ELEMENTS.keys + + ELEMENTS.each do |name, plural_name| + module_eval(<<-EOC, *get_file_and_line_from_caller(0)) + class Slash#{Utils.to_class_name(name)} < Element + include RSS10 + + content_setup + + class << self + def required_prefix + SLASH_PREFIX + end + + def required_uri + SLASH_URI + end + end + + @tag_name = #{name.dump} -require 'rss/parser' -require 'rss/1.0' -require 'rss/2.0' -require 'rss/dublincore' -# begin -# require 'rss/dublincore/2.0' -# rescue -# warning "Unable to load RSS libraries, RSS plugin functionality crippled" -# end + alias_method(:value, :content) + alias_method(:value=, :content=) -class ::String - def shorten(limit) - if self.length > limit - self+". " =~ /^(.{#{limit}}[^.!;?]*[.!;?])/mi - return $1 + def initialize(*args) + if Utils.element_initialize_arguments?(args) + super + else + super() + self.content = args[0] + end + end + + def full_name + tag_name_with_prefix(SLASH_PREFIX) + end + + def maker_target(target) + target.new_#{name} + end + + def setup_maker_attributes(#{name}) + #{name}.content = content + end + end + EOC + end end - self - end - def riphtml - self.gsub(/<[^>]+>/, '').gsub(/&/,'&').gsub(/"/,'"').gsub(/</,'<').gsub(/>/,'>').gsub(/&ellip;/,'...').gsub(/'/, "'").gsub("\n",'') - end + class RDF + class Item; include SlashModel; end + end + + SlashModel::ELEMENTS.each do |name| + class_name = Utils.to_class_name(name) + BaseListener.install_class_name(SLASH_URI, name, "Slash#{class_name}") + end - def mysqlize - self.gsub(/'/, "''") + SlashModel::ELEMENTS.collect! {|name| "#{SLASH_PREFIX}_#{name}"} end end + class ::RssBlob - attr :url - attr :handle - attr :type + attr_accessor :url + attr_accessor :handle + attr_accessor :type attr :watchers + attr_accessor :refresh_rate + attr_accessor :xml + attr_accessor :title + attr_accessor :items + attr_accessor :mutex - def initialize(url,handle=nil,type=nil,watchers=[]) + def initialize(url,handle=nil,type=nil,watchers=[], xml=nil) @url = url if handle @handle = handle @@ -51,7 +162,32 @@ class ::RssBlob @handle = url end @type = type - @watchers = watchers + @watchers=[] + @refresh_rate = nil + @xml = xml + @title = nil + @items = nil + @mutex = Mutex.new + sanitize_watchers(watchers) + end + + def dup + @mutex.synchronize do + self.class.new(@url, + @handle, + @type ? @type.dup : nil, + @watchers.dup, + @xml ? @xml.dup : nil) + end + end + + # Downcase all watchers, possibly turning them into Strings if they weren't + def sanitize_watchers(list=@watchers) + ls = list.dup + @watchers.clear + ls.each { |w| + add_watch(w) + } end def watched? @@ -59,32 +195,27 @@ class ::RssBlob end def watched_by?(who) - # We need to check bot 'who' itself and the String form, because rss - # watches added before the new Irc framework represented watchers as - # Strings whereas they are now Channels. - # - @watchers.include?(who) || @watchers.include?(who.to_s) + @watchers.include?(who.downcase) end def add_watch(who) if watched_by?(who) return nil end - # TODO FIXME? should we just store watchers as Strings instead? - # This should then be @watchers << who.downcase - @watchers << who + @mutex.synchronize do + @watchers << who.downcase + end return who end def rm_watch(who) - # See comment to watched_by? - # - @watchers.delete(who) - @watchers.delete(who.to_s) + @mutex.synchronize do + @watchers.delete(who.downcase) + end end def to_a - [@handle,@url,@type,@watchers] + [@handle,@url,@type,@refresh_rate,@watchers] end def to_s(watchers=false) @@ -99,28 +230,82 @@ end class RSSFeedsPlugin < Plugin BotConfig.register BotConfigIntegerValue.new('rss.head_max', - :default => 30, :validate => Proc.new{|v| v > 0 && v < 200}, + :default => 100, :validate => Proc.new{|v| v > 0 && v < 200}, :desc => "How many characters to use of a RSS item header") BotConfig.register BotConfigIntegerValue.new('rss.text_max', - :default => 90, :validate => Proc.new{|v| v > 0 && v < 400}, + :default => 200, :validate => Proc.new{|v| v > 0 && v < 400}, :desc => "How many characters to use of a RSS item text") BotConfig.register BotConfigIntegerValue.new('rss.thread_sleep', :default => 300, :validate => Proc.new{|v| v > 30}, :desc => "How many seconds to sleep before checking RSS feeds again") - @@watchThreads = Hash.new - @@mutex = Mutex.new + BotConfig.register BotConfigBooleanValue.new('rss.show_updated', + :default => true, + :desc => "Whether feed items for which the description was changed should be shown as new") + + # We used to save the Mutex with the RssBlob, which was idiotic. And + # since Mutexes dumped in one version might not be resotrable in another, + # we need a few tricks to be able to restore data from other versions of Ruby + # + # When migrating 1.8.6 => 1.8.5, all we need to do is define an empty + # #marshal_load() method for Mutex. For 1.8.5 => 1.8.6 we need something + # dirtier, as seen later on in the initialization code. + unless Mutex.new.respond_to?(:marshal_load) + class ::Mutex + def marshal_load(str) + return + end + end + end + + attr_reader :feeds def initialize super - kill_threads if @registry.has_key?(:feeds) + # When migrating from Ruby 1.8.5 to 1.8.6, dumped Mutexes may render the + # data unrestorable. If this happens, we patch the data, thus allowing + # the restore to work. + # + # This is actually pretty safe for a number of reasons: + # * the code is only called if standard marshalling fails + # * the string we look for is quite unlikely to appear randomly + # * if the string appears somewhere and the patched string isn't recoverable + # either, we'll get another (unrecoverable) error, which makes the rss + # plugin unsable, just like it was if no recovery was attempted + # * if the string appears somewhere and the patched string is recoverable, + # we may get a b0rked feed, which is eventually overwritten by a clean + # one, so the worst thing that can happen is that a feed update spams + # the watchers once + @registry.recovery = Proc.new { |val| + patched = val.sub(":\v@mutexo:\nMutex", ":\v@mutexo:\vObject") + ret = Marshal.restore(patched) + ret.each_value { |blob| + blob.mutex = nil + blob + } + } + @feeds = @registry[:feeds] + raise unless @feeds + + @registry.recovery = nil + + @feeds.keys.grep(/[A-Z]/) { |k| + @feeds[k.downcase] = @feeds[k] + @feeds.delete(k) + } + @feeds.each { |k, f| + f.mutex = Mutex.new unless f.mutex + f.sanitize_watchers + parseRss(f) if f.xml + } else @feeds = Hash.new end + @watch = Hash.new rewatch_rss end @@ -133,21 +318,35 @@ class RSSFeedsPlugin < Plugin end def cleanup - kill_threads + stop_watches + super end def save - @registry[:feeds] = @feeds + unparsed = Hash.new() + @feeds.each { |k, f| + unparsed[k] = f.dup + # we don't want to save the mutex + unparsed[k].mutex = nil + } + @registry[:feeds] = unparsed end - def kill_threads - @@mutex.synchronize { - # Abort all running threads. - @@watchThreads.each { |url, thread| - debug "Killing thread for #{url}" - thread.kill - } - @@watchThreads = Hash.new + def stop_watch(handle) + if @watch.has_key?(handle) + begin + debug "Stopping watch #{handle}" + @bot.timer.remove(@watch[handle]) + @watch.delete(handle) + rescue Exception => e + report_problem("Failed to stop watch for #{handle}", e, nil) + end + end + end + + def stop_watches + @watch.each_key { |k| + stop_watch(k) } end @@ -158,9 +357,13 @@ class RSSFeedsPlugin < Plugin when "list" "rss list [#{Bold}handle#{Bold}] : list all rss feeds (matching #{Bold}handle#{Bold})" when "watched" - "rss watched [#{Bold}handle#{Bold}] : list all watched rss feeds (matching #{Bold}handle#{Bold})" + "rss watched [#{Bold}handle#{Bold}] [in #{Bold}chan#{Bold}]: list all watched rss feeds (matching #{Bold}handle#{Bold}) (in channel #{Bold}chan#{Bold})" + when "who", "watches", "who watches" + "rss who watches [#{Bold}handle#{Bold}]]: list all watchers for rss feeds (matching #{Bold}handle#{Bold})" when "add" "rss add #{Bold}handle#{Bold} #{Bold}url#{Bold} [#{Bold}type#{Bold}] : add a new rss called #{Bold}handle#{Bold} from url #{Bold}url#{Bold} (of type #{Bold}type#{Bold})" + when "change" + "rss change #{Bold}what#{Bold} of #{Bold}handle#{Bold} to #{Bold}new#{Bold} : change the #{Underline}handle#{Underline}, #{Underline}url#{Underline}, #{Underline}type#{Underline} or #{Underline}refresh#{Underline} rate of rss called #{Bold}handle#{Bold} to value #{Bold}new#{Bold}" when /^(del(ete)?|rm)$/ "rss del(ete)|rm #{Bold}handle#{Bold} : delete rss feed #{Bold}handle#{Bold}" when "replace" @@ -168,13 +371,13 @@ class RSSFeedsPlugin < Plugin when "forcereplace" "rss forcereplace #{Bold}handle#{Bold} #{Bold}url#{Bold} [#{Bold}type#{Bold}] : replace the url of rss called #{Bold}handle#{Bold} with #{Bold}url#{Bold} (of type #{Bold}type#{Bold})" when "watch" - "rss watch #{Bold}handle#{Bold} [#{Bold}url#{Bold} [#{Bold}type#{Bold}]] : watch rss #{Bold}handle#{Bold} for changes; when the other parameters are present, it will be created if it doesn't exist yet" + "rss watch #{Bold}handle#{Bold} [#{Bold}url#{Bold} [#{Bold}type#{Bold}]] [in #{Bold}chan#{Bold}]: watch rss #{Bold}handle#{Bold} for changes (in channel #{Bold}chan#{Bold}); when the other parameters are present, the feed will be created if it doesn't exist yet" when /(un|rm)watch/ - "rss unwatch|rmwatch #{Bold}handle#{Bold} : stop watching rss #{Bold}handle#{Bold} for changes" + "rss unwatch|rmwatch #{Bold}handle#{Bold} [in #{Bold}chan#{Bold}]: stop watching rss #{Bold}handle#{Bold} (in channel #{Bold}chan#{Bold}) for changes" when "rewatch" "rss rewatch : restart threads that watch for changes in watched rss" else - "manage RSS feeds: rss show|list|watched|add|del(ete)|rm|(force)replace|watch|unwatch|rmwatch|rewatch" + "manage RSS feeds: rss show|list|watched|add|change|del(ete)|rm|(force)replace|watch|unwatch|rmwatch|rewatch" end end @@ -200,11 +403,11 @@ class RSSFeedsPlugin < Plugin rev = lims[1].to_i > lims[2].to_i else ll = 0 - ul = [[lims[1].to_i-1, 1].max, 14].min + ul = [[lims[1].to_i-1, 0].max, 14].min rev = false end - feed = @feeds.fetch(handle, nil) + feed = @feeds.fetch(handle.downcase, nil) unless feed m.reply "I don't know any feeds named #{handle}" return @@ -212,10 +415,35 @@ class RSSFeedsPlugin < Plugin m.reply "lemme fetch it..." title = items = nil - @@mutex.synchronize { - title, items = fetchRss(feed, m) - } - return unless items + we_were_watching = false + + if @watch.key?(feed.handle) + # If a feed is being watched, we run the watcher thread + # so that all watchers can be informed of changes to + # the feed. Before we do that, though, we remove the + # show requester from the watchlist, if present, lest + # he gets the update twice. + if feed.watched_by?(m.replyto) + we_were_watching = true + feed.rm_watch(m.replyto) + end + @bot.timer.reschedule(@watch[feed.handle], 0) + if we_were_watching + feed.add_watch(m.replyto) + end + else + fetched = fetchRss(feed, m, false) + end + return unless fetched or feed.xml + if not fetched and feed.items + m.reply "using old data" + else + parsed = parseRss(feed, m) + m.reply "using old data" unless parsed + end + return unless feed.items + title = feed.title + items = feed.items # We sort the feeds in freshness order (newer ones first) items = freshness_sort(items) @@ -229,45 +457,62 @@ class RSSFeedsPlugin < Plugin end def itemDate(item,ex=nil) - return item.pubDate if item.respond_to?(:pubDate) - return item.date if item.respond_to?(:date) + return item.pubDate if item.respond_to?(:pubDate) and item.pubDate + return item.date if item.respond_to?(:date) and item.date return ex end def freshness_sort(items) notime = Time.at(0) items.sort { |a, b| - itemDate(b, notime) || 0 <=> itemDate(a, notime) || 0 + itemDate(b, notime) <=> itemDate(a, notime) } end def list_rss(m, params) wanted = params[:handle] reply = String.new - @@mutex.synchronize { - @feeds.each { |handle, feed| - next if wanted and !handle.match(wanted) - reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" - (reply << " (watched)") if feed.watched_by?(m.replyto) - reply << "\n" - } + @feeds.each { |handle, feed| + next if wanted and !handle.match(/#{wanted}/i) + reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" + (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate + (reply << " (watched)") if feed.watched_by?(m.replyto) + reply << "\n" } if reply.empty? reply = "no feeds found" reply << " matching #{wanted}" if wanted end - m.reply reply + m.reply reply, :max_lines => reply.length end def watched_rss(m, params) wanted = params[:handle] + chan = params[:chan] || m.replyto reply = String.new - @@mutex.synchronize { - watchlist.each { |handle, feed| - next if wanted and !handle.match(wanted) - next unless feed.watched_by?(m.replyto) - reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})\n" - } + watchlist.each { |handle, feed| + next if wanted and !handle.match(/#{wanted}/i) + next unless feed.watched_by?(chan) + reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" + (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate + reply << "\n" + } + if reply.empty? + reply = "no watched feeds" + reply << " matching #{wanted}" if wanted + end + m.reply reply + end + + def who_watches(m, params) + wanted = params[:handle] + reply = String.new + watchlist.each { |handle, feed| + next if wanted and !handle.match(/#{wanted}/i) + reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})" + (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate + reply << ": watched by #{feed.watchers.join(', ')}" + reply << "\n" } if reply.empty? reply = "no watched feeds" @@ -284,17 +529,15 @@ class RSSFeedsPlugin < Plugin return end type = params[:type] - if @feeds.fetch(handle, nil) && !force - m.reply "There is already a feed named #{handle} (URL: #{@feeds[handle].url})" + if @feeds.fetch(handle.downcase, nil) && !force + m.reply "There is already a feed named #{handle} (URL: #{@feeds[handle.downcase].url})" return end unless url m.reply "You must specify both a handle and an url to add an RSS feed" return end - @@mutex.synchronize { - @feeds[handle] = RssBlob.new(url,handle,type) - } + @feeds[handle.downcase] = RssBlob.new(url,handle,type) reply = "Added RSS #{url} named #{handle}" if type reply << " (format: #{type})" @@ -303,25 +546,70 @@ class RSSFeedsPlugin < Plugin return handle end + def change_rss(m, params) + handle = params[:handle].downcase + feed = @feeds.fetch(handle, nil) + unless feed + m.reply "No such feed with handle #{handle}" + return + end + case params[:what].intern + when :handle + new = params[:new].downcase + if @feeds.key?(new) and @feeds[new] + m.reply "There already is a feed with handle #{new}" + return + else + feed.mutex.synchronize do + @feeds[new] = feed + @feeds.delete(handle) + feed.handle = new + end + handle = new + end + when :url + new = params[:new] + feed.mutex.synchronize do + feed.url = new + end + when :format, :type + new = params[:new] + new = nil if new == 'default' + feed.mutex.synchronize do + feed.type = new + end + when :refresh + new = params[:new].to_i + new = nil if new == 0 + feed.mutex.synchronize do + feed.refresh_rate = new + end + else + m.reply "Don't know how to change #{params[:what]} for feeds" + return + end + m.reply "Feed changed:" + list_rss(m, {:handle => handle}) + end + def del_rss(m, params, pass=false) feed = unwatch_rss(m, params, true) + return unless feed if feed.watched? m.reply "someone else is watching #{feed.handle}, I won't remove it from my list" return end - @@mutex.synchronize { - @feeds.delete(feed.handle) - } + @feeds.delete(feed.handle.downcase) m.okay unless pass return end def replace_rss(m, params) handle = params[:handle] - if @feeds.key?(handle) + if @feeds.key?(handle.downcase) del_rss(m, {:handle => handle}, true) end - if @feeds.key?(handle) + if @feeds.key?(handle.downcase) m.reply "can't replace #{feed.handle}" else add_rss(m, params, true) @@ -334,55 +622,46 @@ class RSSFeedsPlugin < Plugin def watch_rss(m, params) handle = params[:handle] + chan = params[:chan] || m.replyto url = params[:url] type = params[:type] if url add_rss(m, params) end - feed = nil - @@mutex.synchronize { - feed = @feeds.fetch(handle, nil) - } + feed = @feeds.fetch(handle.downcase, nil) if feed - @@mutex.synchronize { - if feed.add_watch(m.replyto) - watchRss(feed, m) - m.okay - else - m.reply "Already watching #{feed.handle}" - end - } + if feed.add_watch(chan) + watchRss(feed, m) + m.okay + else + m.reply "Already watching #{feed.handle} in #{chan}" + end else m.reply "Couldn't watch feed #{handle} (no such feed found)" end end def unwatch_rss(m, params, pass=false) - handle = params[:handle] + handle = params[:handle].downcase + chan = params[:chan] || m.replyto unless @feeds.has_key?(handle) m.reply("dunno that feed") return end feed = @feeds[handle] - if feed.rm_watch(m.replyto) - m.reply "#{m.replyto} has been removed from the watchlist for #{feed.handle}" + if feed.rm_watch(chan) + m.reply "#{chan} has been removed from the watchlist for #{feed.handle}" else - m.reply("#{m.replyto} wasn't watching #{feed.handle}") unless pass + m.reply("#{chan} wasn't watching #{feed.handle}") unless pass end if !feed.watched? - @@mutex.synchronize { - if @@watchThreads[handle].kind_of? Thread - @@watchThreads[handle].kill - debug "rmwatch: Killed thread for #{handle}" - @@watchThreads.delete(handle) - end - } + stop_watch(handle) end return feed end def rewatch_rss(m=nil, params=nil) - kill_threads + stop_watches # Read watches from list. watchlist.each{ |handle, feed| @@ -393,61 +672,103 @@ class RSSFeedsPlugin < Plugin private def watchRss(feed, m=nil) - if @@watchThreads.has_key?(feed.handle) + if @watch.has_key?(feed.handle) report_problem("watcher thread for #{feed.handle} is already running", nil, m) return end - @@watchThreads[feed.handle] = Thread.new do + status = Hash.new + status[:failures] = 0 + status[:first_run] = true + @watch[feed.handle] = @bot.timer.add(0, status) { debug "watcher for #{feed} started" - oldItems = [] - firstRun = true - failures = 0 - loop do - begin - debug "fetching #{feed}" - title = newItems = nil - @@mutex.synchronize { - title, newItems = fetchRss(feed) - } - unless newItems - debug "no items in feed #{feed}" - failures +=1 + failures = status[:failures] + first_run = status.delete(:first_run) + begin + debug "fetching #{feed}" + oldxml = feed.xml ? feed.xml.dup : nil + unless fetchRss(feed) + failures += 1 + else + if first_run + debug "first run for #{feed}, getting items" + parseRss(feed) + elsif oldxml and oldxml == feed.xml + debug "xml for #{feed} didn't change" + failures -= 1 if failures > 0 else - debug "Checking if new items are available for #{feed}" - if firstRun - debug "First run, we'll see next time" - firstRun = false + if not feed.items + debug "no previous items in feed #{feed}" + parseRss(feed) + failures -= 1 if failures > 0 else - otxt = oldItems.map { |item| item.to_s } - dispItems = newItems.reject { |item| - otxt.include?(item.to_s) + # This one is used for debugging + otxt = [] + + # These are used for checking new items vs old ones + uid_opts = { :show_updated => @bot.config['rss.show_updated'] } + oids = Set.new feed.items.map { |item| + uid = RSS.item_uid_for_bot(item, uid_opts) + otxt << item.to_s + debug [uid, item].inspect + debug [uid, otxt.last].inspect + uid } - if dispItems.length > 0 - debug "Found #{dispItems.length} new items in #{feed}" - # When displaying watched feeds, publish them from older to newer - dispItems.reverse.each { |item| - @@mutex.synchronize { + + unless parseRss(feed) + debug "no items in feed #{feed}" + failures += 1 + else + debug "Checking if new items are available for #{feed}" + failures -= 1 if failures > 0 + # debug "Old:" + # debug oldxml + # debug "New:" + # debug feed.xml + + dispItems = feed.items.reject { |item| + uid = RSS.item_uid_for_bot(item, uid_opts) + txt = item.to_s + if oids.include?(uid) + debug "rejecting old #{uid} #{item.inspect}" + debug [uid, txt].inspect + true + else + debug "accepting new #{uid} #{item.inspect}" + debug [uid, txt].inspect + warning "same text! #{txt}" if otxt.include?(txt) + false + end + } + + if dispItems.length > 0 + debug "Found #{dispItems.length} new items in #{feed}" + # When displaying watched feeds, publish them from older to newer + dispItems.reverse.each { |item| printFormattedRss(feed, item) } - } - else - debug "No new items found in #{feed}" + else + debug "No new items found in #{feed}" + end end end - oldItems = newItems.dup end - rescue Exception => e - error "Error watching #{feed}: #{e.inspect}" - debug e.backtrace.join("\n") - failures += 1 end + rescue Exception => e + error "Error watching #{feed}: #{e.inspect}" + debug e.backtrace.join("\n") + failures += 1 + end + + status[:failures] = failures - seconds = @bot.config['rss.thread_sleep'] * (failures + 1) + feed.mutex.synchronize do + seconds = (feed.refresh_rate || @bot.config['rss.thread_sleep']) * (failures + 1) seconds += seconds * (rand(100)-50)/100 debug "watcher for #{feed} going to sleep #{seconds} seconds.." - sleep seconds + @bot.timer.reschedule(@watch[feed.handle], seconds) end - end + } + debug "watcher for #{feed} added" end def printFormattedRss(feed, item, opts=nil) @@ -460,13 +781,13 @@ class RSSFeedsPlugin < Plugin if opts.key?(:date) && opts[:date] if item.respond_to?(:pubDate) if item.pubDate.class <= Time - date = item.pubDate.strftime("%Y/%m/%d %H.%M.%S") + date = item.pubDate.strftime("%Y/%m/%d %H:%M") else date = item.pubDate.to_s end elsif item.respond_to?(:date) if item.date.class <= Time - date = item.date.strftime("%Y/%m/%d %H.%M.%S") + date = item.date.strftime("%Y/%m/%d %H:%M") else date = item.date.to_s end @@ -476,92 +797,135 @@ class RSSFeedsPlugin < Plugin date += " :: " end end - title = "#{Bold}#{item.title.chomp.riphtml}#{Bold}" if item.title - desc = item.description.gsub(/\s+/,' ').strip.riphtml.shorten(@bot.config['rss.text_max']) if item.description + + title = "#{Bold}#{item.title.ircify_html :limit => @bot.config['rss.head_max']}#{Bold}" if item.title + + desc = item.description.ircify_html(:limit => @bot.config['rss.text_max'], :a_href => :link_out) if item.description + link = item.link.chomp if item.link - places.each { |loc| - case feed.type - when 'blog' - @bot.say loc, "#{handle}#{date}#{item.category.content} blogged at #{link}" - @bot.say loc, "#{handle}#{title} - #{desc}" - when 'forum' - @bot.say loc, "#{handle}#{date}#{title}#{' @ ' if item.title && item.link}#{link}" - when 'wiki' - @bot.say loc, "#{handle}#{date}#{item.title} has been edited by #{item.dc_creator}. #{desc} #{link}" - when 'gmame' - @bot.say loc, "#{handle}#{date}Message #{title} sent by #{item.dc_creator}. #{desc}" - when 'trac' - @bot.say loc, "#{handle}#{date}#{title} @ #{link}" - unless item.title =~ /^Changeset \[(\d+)\]/ - @bot.say loc, "#{handle}#{date}#{desc}" - end - else - @bot.say loc, "#{handle}#{date}#{title}#{' @ ' if item.title && item.link}#{link}" + + debug item.inspect + category = item.dc_subject rescue item.category.content rescue nil + category = nil if category and category.empty? + author = item.dc_creator rescue item.author rescue nil + author = nil if author and author.empty? + + line1 = nil + line2 = nil + + at = ((item.title && item.link) ? ' @ ' : '') + case feed.type + when 'blog' + author << " " if author + abt = category ? "about #{category} " : "" + line1 = "#{handle}#{date}#{author}blogged #{abt}at #{link}" + line2 = "#{handle}#{title} - #{desc}" + when 'forum' + line1 = "#{handle}#{date}#{title}#{at}#{link}" + when 'wiki' + line1 = "#{handle}#{date}#{title}#{at}#{link} has been edited by #{author}. #{desc}" + when 'gmane' + line1 = "#{handle}#{date}Message #{title} sent by #{author}. #{desc}" + when 'trac' + line1 = "#{handle}#{date}#{title} @ #{link}" + unless item.title =~ /^Changeset \[(\d+)\]/ + line2 = "#{handle}#{date}#{desc}" end + when '/.' + dept = "(from the #{item.slash_department} dept) " rescue nil + sec = " in section #{item.slash_section}" rescue nil + + line1 = "#{handle}#{date}#{dept}#{title}#{at}#{link} (posted by #{author}#{sec})" + else + line1 = "#{handle}#{date}#{title}#{at}#{link}" + end + places.each { |loc| + @bot.say loc, line1, :overlong => :truncate + next unless line2 + @bot.say loc, line2, :overlong => :truncate } end - def fetchRss(feed, m=nil) + def fetchRss(feed, m=nil, cache=true) begin # Use 60 sec timeout, cause the default is too low - # Do not use get_cached for RSS until we have proper cache handling - # xml = @bot.httputil.get_cached(feed.url,60,60) - xml = @bot.httputil.get(feed.url,60,60) + xml = @bot.httputil.get(feed.url, + :read_timeout => 60, + :open_timeout => 60, + :cache => cache) rescue URI::InvalidURIError, URI::BadURIError => e report_problem("invalid rss feed #{feed.url}", e, m) - return + return nil rescue => e report_problem("error getting #{feed.url}", e, m) - return + return nil end debug "fetched #{feed}" unless xml report_problem("reading feed #{feed} failed", nil, m) - return + return nil + end + # Ok, 0.9 feeds are not supported, maybe because + # Netscape happily removed the DTD. So what we do is just to + # reassign the 0.9 RDFs to 1.0, and hope it goes right. + xml.gsub!("xmlns=\"http://my.netscape.com/rdf/simple/0.9/\"", + "xmlns=\"http://purl.org/rss/1.0/\"") + feed.mutex.synchronize do + feed.xml = xml end + return true + end - begin - ## do validate parse - rss = RSS::Parser.parse(xml) - debug "parsed #{feed}" - rescue RSS::InvalidRSSError - ## do non validate parse for invalid RSS 1.0 + def parseRss(feed, m=nil) + return nil unless feed.xml + feed.mutex.synchronize do + xml = feed.xml begin - rss = RSS::Parser.parse(xml, false) + ## do validate parse + rss = RSS::Parser.parse(xml) + debug "parsed and validated #{feed}" + rescue RSS::InvalidRSSError + ## do non validate parse for invalid RSS 1.0 + begin + rss = RSS::Parser.parse(xml, false) + debug "parsed but not validated #{feed}" + rescue RSS::Error => e + report_problem("parsing rss stream failed, whoops =(", e, m) + return nil + end rescue RSS::Error => e - report_problem("parsing rss stream failed, whoops =(", e, m) - return - end - rescue RSS::Error => e - report_problem("parsing rss stream failed, oioi", e, m) - return - rescue => e - report_problem("processing error occured, sorry =(", e, m) - return - end - items = [] - if rss.nil? - report_problem("#{feed} does not include RSS 1.0 or 0.9x/2.0", nil, m) - else - begin - rss.output_encoding = 'UTF-8' - rescue RSS::UnknownConvertMethod => e - report_problem("bah! something went wrong =(", e, m) - return + report_problem("parsing rss stream failed, oioi", e, m) + return nil + rescue => e + report_problem("processing error occured, sorry =(", e, m) + return nil end - rss.channel.title ||= "Unknown" - title = rss.channel.title - rss.items.each do |item| - item.title ||= "Unknown" - items << item + items = [] + if rss.nil? + report_problem("#{feed} does not include RSS 1.0 or 0.9x/2.0", nil, m) + else + begin + rss.output_encoding = 'UTF-8' + rescue RSS::UnknownConvertMethod => e + report_problem("bah! something went wrong =(", e, m) + return nil + end + rss.channel.title ||= "Unknown" + title = rss.channel.title + rss.items.each do |item| + item.title ||= "Unknown" + items << item + end end - end - if items.empty? - report_problem("no items found in the feed, maybe try weed?", e, m) - return + if items.empty? + report_problem("no items found in the feed, maybe try weed?", e, m) + return nil + end + feed.title = title + feed.items = items + return true end - return [title, items] end end @@ -573,13 +937,22 @@ plugin.map 'rss show :handle :limit', :defaults => {:limit => 5} plugin.map 'rss list :handle', :action => 'list_rss', - :defaults => {:handle => nil} -plugin.map 'rss watched :handle', + :defaults => {:handle => nil} +plugin.map 'rss watched :handle [in :chan]', :action => 'watched_rss', - :defaults => {:handle => nil} + :defaults => {:handle => nil} +plugin.map 'rss who watches :handle', + :action => 'who_watches', + :defaults => {:handle => nil} plugin.map 'rss add :handle :url :type', :action => 'add_rss', :defaults => {:type => nil} +plugin.map 'rss change :what of :handle to :new', + :action => 'change_rss', + :requirements => { :what => /handle|url|format|type|refresh/ } +plugin.map 'rss change :what for :handle to :new', + :action => 'change_rss', + :requirements => { :what => /handle|url|format|type|refesh/ } plugin.map 'rss del :handle', :action => 'del_rss' plugin.map 'rss delete :handle', @@ -592,12 +965,15 @@ plugin.map 'rss replace :handle :url :type', plugin.map 'rss forcereplace :handle :url :type', :action => 'forcereplace_rss', :defaults => {:type => nil} -plugin.map 'rss watch :handle :url :type', +plugin.map 'rss watch :handle [in :chan]', + :action => 'watch_rss', + :defaults => {:url => nil, :type => nil} +plugin.map 'rss watch :handle :url :type [in :chan]', :action => 'watch_rss', :defaults => {:url => nil, :type => nil} -plugin.map 'rss unwatch :handle', +plugin.map 'rss unwatch :handle [in :chan]', :action => 'unwatch_rss' -plugin.map 'rss rmwatch :handle', +plugin.map 'rss rmwatch :handle [in :chan]', :action => 'unwatch_rss' plugin.map 'rss rewatch', :action => 'rewatch_rss'