require 'rss'
-# Try to load rss/content/2.0 so we can access the data in <content:encoded>
+# Try to load rss/content/2.0 so we can access the data in <content:encoded>
# tags.
begin
require 'rss/content/2.0'
class ::RssBlob
attr_accessor :url, :handle, :type, :refresh_rate, :xml, :title, :items,
- :mutex, :watchers, :last_fetched
+ :mutex, :watchers, :last_fetched, :http_cache, :last_success
def initialize(url,handle=nil,type=nil,watchers=[], xml=nil, lf = nil)
@url = url
@type = type
@watchers=[]
@refresh_rate = nil
+ @http_cache = false
@xml = xml
@title = nil
@items = nil
@mutex = Mutex.new
@last_fetched = lf
+ @last_success = nil
sanitize_watchers(watchers)
end
:default => 300, :validate => Proc.new{|v| v > 30},
:desc => "How many seconds to sleep before checking RSS feeds again")
+ Config.register Config::IntegerValue.new('rss.announce_timeout',
+ :default => 0,
+ :desc => "Don't announce watched feed if these many seconds elapsed since the last successful update")
+
Config.register Config::BooleanValue.new('rss.show_updated',
:default => true,
:desc => "Whether feed items for which the description was changed should be shown as new")
# Make an 'unique' ID for a given item, based on appropriate bot options
# Currently only suppored is bot.config['rss.show_updated']: when false,
# only the guid/link is accounted for.
-
+
def make_uid(item)
uid = [item.guid! || item.link!]
if @bot.config['rss.show_updated']
def list_rss(m, params)
wanted = params[:handle]
- reply = String.new
- @feeds.each { |handle, feed|
- next if wanted and !handle.match(/#{wanted}/i)
- reply << "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})"
- (reply << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate
- (reply << " (watched)") if feed.watched_by?(m.replyto)
- reply << "\n"
- }
- if reply.empty?
+ listed = @feeds.keys
+ if wanted
+ wanted_rx = Regexp.new(wanted, true)
+ listed.reject! { |handle| !handle.match(wanted_rx) }
+ end
+ listed.sort!
+ debug listed
+ if @bot.config['send.max_lines'] > 0 and listed.size > @bot.config['send.max_lines']
+ reply = listed.inject([]) do |ar, handle|
+ feed = @feeds[handle]
+ string = handle.dup
+ (string << " (#{feed.type})") if feed.type
+ (string << " (watched)") if feed.watched_by?(m.replyto)
+ ar << string
+ end.join(', ')
+ elsif listed.size > 0
+ reply = listed.inject([]) do |ar, handle|
+ feed = @feeds[handle]
+ string = "#{feed.handle}: #{feed.url} (in format: #{feed.type ? feed.type : 'default'})"
+ (string << " refreshing every #{Utils.secs_to_string(feed.refresh_rate)}") if feed.refresh_rate
+ (string << " (watched)") if feed.watched_by?(m.replyto)
+ ar << string
+ end.join("\n")
+ else
reply = "no feeds found"
reply << " matching #{wanted}" if wanted
end
- m.reply reply, :max_lines => reply.length
+ m.reply reply, :max_lines => 0
end
def watched_rss(m, params)
if params and handle = params[:handle]
feed = @feeds.fetch(handle.downcase, nil)
if feed
+ feed.http_cache = false
@bot.timer.reschedule(@watch[feed.handle], (params[:delay] || 0).to_f)
m.okay if m
else
private
def watchRss(feed, m=nil)
if @watch.has_key?(feed.handle)
- report_problem("watcher thread for #{feed.handle} is already running", nil, m)
+ # report_problem("watcher thread for #{feed.handle} is already running", nil, m)
return
end
status = Hash.new
failures = status[:failures]
begin
debug "fetching #{feed}"
- first_run = !feed.last_fetched
+
+ first_run = !feed.last_success
+ if (@bot.config['rss.announce_timeout'] > 0 &&
+ (Time.now - feed.last_success > @bot.config['rss.announce_timeout']))
+ debug "#{feed} wasn't polled for too long, supressing output"
+ first_run = true
+ end
oldxml = feed.xml ? feed.xml.dup : nil
- unless fetchRss(feed)
+ unless fetchRss(feed, nil, feed.http_cache)
failures += 1
else
+ feed.http_cache = true
if first_run
debug "first run for #{feed}, getting items"
parseRss(feed)
debug "xml for #{feed} didn't change"
failures -= 1 if failures > 0
else
- if not feed.items
- debug "no previous items in feed #{feed}"
- parseRss(feed)
- failures -= 1 if failures > 0
- else
- # This one is used for debugging
- otxt = []
+ # This one is used for debugging
+ otxt = []
+ if feed.items.nil?
+ oids = []
+ else
# These are used for checking new items vs old ones
oids = Set.new feed.items.map { |item|
uid = make_uid item
debug [uid, otxt.last].inspect
uid
}
+ end
nitems = parseRss(feed)
if nitems.nil?
debug "No new items found in #{feed}"
end
end
- end
end
end
rescue Exception => e
else
date = item.source.updated.content.to_s
end
- elsif item.respond_to?(:pubDate)
+ elsif item.respond_to?(:pubDate)
if item.pubDate.class <= Time
date = item.pubDate.strftime("%Y/%m/%d %H:%M")
else
desc_opt[:limit] = @bot.config['rss.text_max']
desc_opt[:a_href] = :link_out if @bot.config['rss.show_links']
- # We prefer content_encoded here as it tends to provide more html formatting
+ # We prefer content_encoded here as it tends to provide more html formatting
# for use with ircify_html.
if item.respond_to?(:content_encoded) && item.content_encoded
desc = item.content_encoded.ircify_html(desc_opt)
end
feed.mutex.synchronize do
feed.xml = xml
+ feed.last_success = Time.now
end
return true
end
return nil unless feed.xml
feed.mutex.synchronize do
xml = feed.xml
- begin
- ## do validate parse
- rss = RSS::Parser.parse(xml)
- debug "parsed and validated #{feed}"
- rescue RSS::InvalidRSSError
- ## do non validate parse for invalid RSS 1.0
+ rss = nil
+ errors = []
+ RSS::AVAILABLE_PARSERS.each do |parser|
begin
- rss = RSS::Parser.parse(xml, false)
- debug "parsed but not validated #{feed}"
+ ## do validate parse
+ rss = RSS::Parser.parse(xml, true, true, parser)
+ debug "parsed and validated #{feed} with #{parser}"
+ break
+ rescue RSS::InvalidRSSError
+ begin
+ ## do non validate parse for invalid RSS 1.0
+ rss = RSS::Parser.parse(xml, false, true, parser)
+ debug "parsed but not validated #{feed} with #{parser}"
+ break
+ rescue RSS::Error => e
+ errors << [parser, e, "parsing rss stream failed, whoops =("]
+ end
rescue RSS::Error => e
- report_problem("parsing rss stream failed, whoops =(", e, m)
- return nil
+ errors << [parser, e, "parsing rss stream failed, oioi"]
+ rescue => e
+ errors << [parser, e, "processing error occured, sorry =("]
end
- rescue RSS::Error => e
- report_problem("parsing rss stream failed, oioi", e, m)
- return nil
- rescue => e
- report_problem("processing error occured, sorry =(", e, m)
- return nil
+ end
+ unless errors.empty?
+ debug errors
+ self.send(:report_problem, errors.last[2], errors.last[1], m)
+ return nil unless rss
end
items = []
if rss.nil?
projects / user / henk / code / ruby / rbot.git / blobdiff