-Url = Struct.new("Url", :channel, :nick, :time, :url)
+#-- vim:sw=2:et
+#++
+#
+# :title: Url plugin
+
+define_structure :Url, :channel, :nick, :time, :url, :info
class UrlPlugin < Plugin
+ LINK_INFO = "[Link Info]"
+ OUR_UNSAFE = Regexp.new("[^#{URI::PATTERN::UNRESERVED}#{URI::PATTERN::RESERVED}%# ]", false, 'N')
+
+ Config.register Config::IntegerValue.new('url.max_urls',
+ :default => 100, :validate => Proc.new{|v| v > 0},
+ :desc => "Maximum number of urls to store. New urls replace oldest ones.")
+ Config.register Config::IntegerValue.new('url.display_link_info',
+ :default => 0,
+ :desc => "Get the title of links pasted to the channel and display it (also tells if the link is broken or the site is down). Do it for at most this many links per line (set to 0 to disable)")
+ Config.register Config::BooleanValue.new('url.titles_only',
+ :default => false,
+ :desc => "Only show info for links that have <title> tags (in other words, don't display info for jpegs, mpegs, etc.)")
+ Config.register Config::BooleanValue.new('url.first_par',
+ :default => false,
+ :desc => "Also try to get the first paragraph of a web page")
+ Config.register Config::BooleanValue.new('url.info_on_list',
+ :default => false,
+ :desc => "Show link info when listing/searching for urls")
+ Config.register Config::ArrayValue.new('url.no_info_hosts',
+ :default => ['localhost', '^192\.168\.', '^10\.', '^127\.', '^172\.(1[6-9]|2\d|31)\.'],
+ :on_change => Proc.new { |bot, v| bot.plugins['url'].reset_no_info_hosts },
+ :desc => "A list of regular expressions matching hosts for which no info should be provided")
+ Config.register Config::ArrayValue.new('url.only_on_channels',
+ :desc => "Show link info only on these channels",
+ :default => [])
+ Config.register Config::ArrayValue.new('url.ignore',
+ :desc => "Don't show link info for urls from users represented as hostmasks on this list. Useful for ignoring other bots, for example.",
+ :default => [])
+
def initialize
super
@registry.set_default(Array.new)
+ unless @bot.config['url.display_link_info'].kind_of?(Integer)
+ @bot.config.items[:'url.display_link_info'].set_string(@bot.config['url.display_link_info'].to_s)
+ end
+ reset_no_info_hosts
+ self.filter_group = :htmlinfo
+ load_filters
end
+
+ def reset_no_info_hosts
+ @no_info_hosts = Regexp.new(@bot.config['url.no_info_hosts'].join('|'), true)
+ debug "no info hosts regexp set to #{@no_info_hosts}"
+ end
+
def help(plugin, topic="")
- "urls [<max>=4] => list <max> last urls mentioned in current channel, urls <channel> [<max>=4] => list <max> last urls mentioned in <channel>, urls search <regexp> => search for matching urls, urls search <channel> <regexp>, search for matching urls in channel <channel>"
+ "url info <url> => display link info for <url> (set url.display_link_info > 0 if you want the bot to do it automatically when someone writes an url), urls [<max>=4] => list <max> last urls mentioned in current channel, urls search [<max>=4] <regexp> => search for matching urls. In a private message, you must specify the channel to query, eg. urls <channel> [max], urls search <channel> [max] <regexp>"
end
- def listen(m)
- return unless m.kind_of?(PrivMessage)
- return if m.address?
- # TODO support multiple urls in one line
- if m.message =~ /(f|ht)tps?:\/\//
- if m.message =~ /((f|ht)tps?:\/\/.*?)(?:\s+|$)/
- url = Url.new(m.target, m.sourcenick, Time.new, $1)
- list = @registry[m.target]
- debug "#{list.length} urls so far"
- if list.length > 50
- list.pop
+
+ def get_title_from_html(pagedata)
+ return pagedata.ircify_html_title
+ end
+
+ def get_title_for_url(uri_str, opts = {})
+
+ url = uri_str.kind_of?(URI) ? uri_str : URI.parse(uri_str)
+ return if url.scheme !~ /https?/
+
+ # also check the ip, the canonical name and the aliases
+ begin
+ checks = TCPSocket.gethostbyname(url.host)
+ checks.delete_at(-2)
+ rescue => e
+ return "Unable to retrieve info for #{url.host}: #{e.message}"
+ end
+
+ checks << url.host
+ checks.flatten!
+
+ unless checks.grep(@no_info_hosts).empty?
+ return ( opts[:always_reply] ? "Sorry, info retrieval for #{url.host} (#{checks.first}) is disabled" : false )
+ end
+
+ logopts = opts.dup
+
+ title = nil
+ extra = []
+
+ begin
+ debug "+ getting info for #{url.request_uri}"
+ info = @bot.filter(:htmlinfo, url)
+ debug info
+ resp = info[:headers]
+
+ logopts[:title] = title = info[:title]
+
+ if info[:content]
+ logopts[:extra] = info[:content]
+ extra << "#{Bold}text#{Bold}: #{info[:content]}" if @bot.config['url.first_par']
+ else
+ logopts[:extra] = String.new
+ logopts[:extra] << "Content Type: #{resp['content-type']}"
+ extra << "#{Bold}type#{Bold}: #{resp['content-type']}" unless title
+ if enc = resp['content-encoding']
+ logopts[:extra] << ", encoding: #{enc}"
+ extra << "#{Bold}encoding#{Bold}: #{enc}" if @bot.config['url.first_par'] or not title
+ end
+
+ size = resp['content-length'].first.gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') rescue nil
+ if size
+ logopts[:extra] << ", size: #{size} bytes"
+ extra << "#{Bold}size#{Bold}: #{size} bytes" if @bot.config['url.first_par'] or not title
end
- debug "storing url #{url.url}"
- list.unshift url
- debug "#{list.length} urls now"
- @registry[m.target] = list
end
+ rescue Exception => e
+ case e
+ when UrlLinkError
+ raise e
+ else
+ error e
+ raise "connecting to site/processing information (#{e.message})"
+ end
+ end
+
+ call_event(:url_added, url.to_s, logopts)
+ if title
+ extra.unshift("#{Bold}title#{Bold}: #{title}")
end
+ return extra.join(", ") if title or not @bot.config['url.titles_only']
end
- def privmsg(m)
- case m.params
- when nil
- if m.public?
- urls m, m.target
- else
- m.reply "in a private message, you need to specify a channel name for urls"
+
+ def handle_urls(m, params={})
+ opts = {
+ :display_info => @bot.config['url.display_link_info'],
+ :channels => @bot.config['url.only_on_channels'],
+ :ignore => @bot.config['url.ignore']
+ }.merge params
+ urls = opts[:urls]
+ display_info= opts[:display_info]
+ channels = opts[:channels]
+ ignore = opts[:ignore]
+
+ unless channels.empty?
+ return unless channels.map { |c| c.downcase }.include?(m.channel.downcase)
+ end
+
+ ignore.each { |u| return if m.source.matches?(u) }
+
+ return if urls.empty?
+ debug "found urls #{urls.inspect}"
+ list = m.public? ? @registry[m.target] : nil
+ debug "display link info: #{display_info}"
+ urls_displayed = 0
+ urls.each do |urlstr|
+ debug "working on #{urlstr}"
+ next unless urlstr =~ /^https?:\/\/./
+ title = nil
+ debug "Getting title for #{urlstr}..."
+ reply = nil
+ begin
+ title = get_title_for_url(urlstr,
+ :always_reply => m.address?,
+ :nick => m.source.nick,
+ :channel => m.channel,
+ :ircline => m.message)
+ debug "Title #{title ? '' : 'not '} found"
+ reply = "#{LINK_INFO} #{title}" if title
+ rescue => e
+ debug e
+ # we might get a 404 because of trailing punctuation, so we try again
+ # with the last character stripped. this might generate invalid URIs
+ # (e.g. because "some.url" gets chopped to some.url%2, so catch that too
+ if e.message =~ /\(404 - Not Found\)/i or e.kind_of?(URI::InvalidURIError)
+ # chop off last character, and retry if we still have enough string to
+ # look like a minimal URL
+ retry if urlstr.chop! and urlstr =~ /^https?:\/\/./
+ end
+ reply = "Error #{e.message}"
end
- when (/^(\d+)$/)
- max = $1.to_i
- if m.public?
- urls m, m.target, max
- else
- m.reply "in a private message, you need to specify a channel name for urls"
+
+ if display_info > urls_displayed
+ if reply
+ m.reply reply, :overlong => :truncate, :to => :public,
+ :nick => (m.address? ? :auto : false)
+ urls_displayed += 1
+ end
end
- when (/^(#.*?)\s+(\d+)$/)
- channel = $1
- max = $2.to_i
- urls m, channel, max
- when (/^(#.*?)$/)
- channel = $1
- urls m, channel
- when (/^search\s+(#.*?)\s+(.*)$/)
- channel = $1
- string = $2
- search m, channel, string
- when (/^search\s+(.*)$/)
- string = $1
- if m.public?
- search m, m.target, string
- else
- m.reply "in a private message, you need to specify a channel name for urls"
+
+ next unless list
+
+ # check to see if this url is already listed
+ next if list.find {|u| u.url == urlstr }
+
+ url = Url.new(m.target, m.sourcenick, Time.new, urlstr, title)
+ debug "#{list.length} urls so far"
+ list.pop if list.length > @bot.config['url.max_urls']
+ debug "storing url #{url.url}"
+ list.unshift url
+ debug "#{list.length} urls now"
+ end
+ @registry[m.target] = list
+ end
+
+ def info(m, params)
+ escaped = URI.escape(params[:urls].to_s, OUR_UNSAFE)
+ urls = URI.extract(escaped)
+ Thread.new do
+ handle_urls(m,
+ :urls => urls,
+ :display_info => params[:urls].length,
+ :channels => [])
+ end
+ end
+
+ def message(m)
+ return if m.address?
+
+ escaped = URI.escape(m.message, OUR_UNSAFE)
+ urls = URI.extract(escaped, ['http', 'https'])
+ return if urls.empty?
+ Thread.new { handle_urls(m, :urls => urls) }
+ end
+
+ def reply_urls(opts={})
+ list = opts[:list]
+ max = opts[:max]
+ channel = opts[:channel]
+ m = opts[:msg]
+ return unless list and max and m
+ list[0..(max-1)].each do |url|
+ disp = "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}"
+ if @bot.config['url.info_on_list']
+ title = url.info ||
+ get_title_for_url(url.url,
+ :nick => url.nick, :channel => channel) rescue nil
+ # If the url info was missing and we now have some, try to upgrade it
+ if channel and title and not url.info
+ ll = @registry[channel]
+ debug ll
+ if el = ll.find { |u| u.url == url.url }
+ el.info = title
+ @registry[channel] = ll
+ end
+ end
+ disp << " --> #{title}" if title
end
- else
- m.reply "incorrect usage: " + help(m.plugin)
+ m.reply disp, :overlong => :truncate
end
end
- def urls(m, channel, max=4)
+ def urls(m, params)
+ channel = params[:channel] ? params[:channel] : m.target
+ max = params[:limit].to_i
max = 10 if max > 10
max = 1 if max < 1
list = @registry[channel]
if list.empty?
m.reply "no urls seen yet for channel #{channel}"
else
- list[0..(max-1)].each do |url|
- m.reply "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}"
- end
+ reply_urls :msg => m, :channel => channel, :list => list, :max => max
end
end
- def search(m, channel, string, max=4)
+ def search(m, params)
+ channel = params[:channel] ? params[:channel] : m.target
+ max = params[:limit].to_i
+ string = params[:string]
max = 10 if max > 10
max = 1 if max < 1
- regex = Regexp.new(string)
+ regex = Regexp.new(string, Regexp::IGNORECASE)
list = @registry[channel].find_all {|url|
- regex.match(url.url) || regex.match(url.nick)
+ regex.match(url.url) || regex.match(url.nick) ||
+ (@bot.config['url.info_on_list'] && regex.match(url.info))
}
if list.empty?
m.reply "no matches for channel #{channel}"
else
- list[0..(max-1)].each do |url|
- m.reply "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}"
- end
+ reply_urls :msg => m, :channel => channel, :list => list, :max => max
end
end
end
+
plugin = UrlPlugin.new
-plugin.register("urls")
+plugin.map 'urls info *urls', :action => 'info'
+plugin.map 'url info *urls', :action => 'info'
+plugin.map 'urls search :channel :limit :string', :action => 'search',
+ :defaults => {:limit => 4},
+ :requirements => {:limit => /^\d+$/},
+ :public => false
+plugin.map 'urls search :limit :string', :action => 'search',
+ :defaults => {:limit => 4},
+ :requirements => {:limit => /^\d+$/},
+ :private => false
+plugin.map 'urls :channel :limit', :defaults => {:limit => 4},
+ :requirements => {:limit => /^\d+$/},
+ :public => false
+plugin.map 'urls :limit', :defaults => {:limit => 4},
+ :requirements => {:limit => /^\d+$/},
+ :private => false
projects / user / henk / code / ruby / rbot.git / blobdiff