1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
|
#-- vim:sw=2:et
#++
#
# :title: Url plugin
define_structure :Url, :channel, :nick, :time, :url, :info
class UrlPlugin < Plugin
LINK_INFO = "[Link Info]"
OUR_UNSAFE = Regexp.new("[^#{URI::PATTERN::UNRESERVED}#{URI::PATTERN::RESERVED}%# ]", false, 'N')
Config.register Config::IntegerValue.new('url.max_urls',
:default => 100, :validate => Proc.new{|v| v > 0},
:desc => "Maximum number of urls to store. New urls replace oldest ones.")
Config.register Config::IntegerValue.new('url.display_link_info',
:default => 0,
:desc => "Get the title of links pasted to the channel and display it (also tells if the link is broken or the site is down). Do it for at most this many links per line (set to 0 to disable)")
Config.register Config::BooleanValue.new('url.titles_only',
:default => false,
:desc => "Only show info for links that have <title> tags (in other words, don't display info for jpegs, mpegs, etc.)")
Config.register Config::BooleanValue.new('url.first_par',
:default => false,
:desc => "Also try to get the first paragraph of a web page")
Config.register Config::BooleanValue.new('url.info_on_list',
:default => false,
:desc => "Show link info when listing/searching for urls")
Config.register Config::ArrayValue.new('url.no_info_hosts',
:default => ['localhost', '^192\.168\.', '^10\.', '^127\.', '^172\.(1[6-9]|2\d|31)\.'],
:on_change => Proc.new { |bot, v| bot.plugins['url'].reset_no_info_hosts },
:desc => "A list of regular expressions matching hosts for which no info should be provided")
def initialize
super
@registry.set_default(Array.new)
unless @bot.config['url.display_link_info'].kind_of?(Integer)
@bot.config.items[:'url.display_link_info'].set_string(@bot.config['url.display_link_info'].to_s)
end
reset_no_info_hosts
end
def reset_no_info_hosts
@no_info_hosts = Regexp.new(@bot.config['url.no_info_hosts'].join('|'), true)
debug "no info hosts regexp set to #{@no_info_hosts}"
end
def help(plugin, topic="")
"url info <url> => display link info for <url> (set url.display_link_info > 0 if you want the bot to do it automatically when someone writes an url), urls [<max>=4] => list <max> last urls mentioned in current channel, urls search [<max>=4] <regexp> => search for matching urls. In a private message, you must specify the channel to query, eg. urls <channel> [max], urls search <channel> [max] <regexp>"
end
def get_title_from_html(pagedata)
return pagedata.ircify_html_title
end
def get_title_for_url(uri_str, opts = {})
url = uri_str.kind_of?(URI) ? uri_str : URI.parse(uri_str)
return if url.scheme !~ /https?/
if url.host =~ @no_info_hosts
return "Sorry, info retrieval for #{url.host} is disabled"
end
logopts = opts.dup
title = nil
extra = []
begin
debug "+ getting info for #{url.request_uri}"
info = Utils.get_html_info(url)
debug info
resp = info[:headers]
logopts[:title] = title = info[:title]
if info[:content]
logopts[:extra] = info[:content]
extra << "#{Bold}text#{Bold}: #{info[:content]}" if @bot.config['url.first_par']
else
logopts[:extra] = String.new
logopts[:extra] << "Content Type: #{resp['content-type']}"
extra << "#{Bold}type#{Bold}: #{resp['content-type']}" unless title
if enc = resp['content-encoding']
logopts[:extra] << ", encoding: #{enc}"
extra << "#{Bold}encoding#{Bold}: #{enc}" if @bot.config['url.first_par'] or not title
end
size = resp['content-length'].first.gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') rescue nil
if size
logopts[:extra] << ", size: #{size} bytes"
extra << "#{Bold}size#{Bold}: #{size} bytes" if @bot.config['url.first_par'] or not title
end
end
rescue Exception => e
case e
when UrlLinkError
raise e
else
error e
raise "connecting to site/processing information (#{e.message})"
end
end
call_event(:url_added, url.to_s, logopts)
if title
extra.unshift("#{Bold}title#{Bold}: #{title}")
end
return extra.join(", ") if title or not @bot.config['url.titles_only']
end
def handle_urls(m, urls, display_info=@bot.config['url.display_link_info'])
return if urls.empty?
debug "found urls #{urls.inspect}"
list = m.public? ? @registry[m.target] : nil
debug "display link info: #{display_info}"
urls_displayed = 0
urls.each do |urlstr|
debug "working on #{urlstr}"
next unless urlstr =~ /^https?:/
title = nil
debug "Getting title for #{urlstr}..."
begin
title = get_title_for_url(urlstr,
:nick => m.source.nick,
:channel => m.channel,
:ircline => m.message)
debug "Title #{title ? '' : 'not '} found"
rescue => e
m.reply "Error #{e.message}"
end
if display_info > urls_displayed
if title
m.reply("#{LINK_INFO} #{title}", :overlong => :truncate)
urls_displayed += 1
end
end
next unless list
# check to see if this url is already listed
next if list.find {|u| u.url == urlstr }
url = Url.new(m.target, m.sourcenick, Time.new, urlstr, title)
debug "#{list.length} urls so far"
list.pop if list.length > @bot.config['url.max_urls']
debug "storing url #{url.url}"
list.unshift url
debug "#{list.length} urls now"
end
@registry[m.target] = list
end
def info(m, params)
escaped = URI.escape(params[:urls].to_s, OUR_UNSAFE)
urls = URI.extract(escaped)
Thread.new { handle_urls(m, urls, params[:urls].length) }
end
def listen(m)
return unless m.kind_of?(PrivMessage)
return if m.address?
escaped = URI.escape(m.message, OUR_UNSAFE)
urls = URI.extract(escaped)
Thread.new { handle_urls(m, urls) }
end
def reply_urls(opts={})
list = opts[:list]
max = opts[:max]
channel = opts[:channel]
m = opts[:msg]
return unless list and max and m
list[0..(max-1)].each do |url|
disp = "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}"
if @bot.config['url.info_on_list']
title = url.info ||
get_title_for_url(url.url,
:nick => url.nick, :channel => channel) rescue nil
# If the url info was missing and we now have some, try to upgrade it
if channel and title and not url.info
ll = @registry[channel]
debug ll
if el = ll.find { |u| u.url == url.url }
el.info = title
@registry[channel] = ll
end
end
disp << " --> #{title}" if title
end
m.reply disp, :overlong => :truncate
end
end
def urls(m, params)
channel = params[:channel] ? params[:channel] : m.target
max = params[:limit].to_i
max = 10 if max > 10
max = 1 if max < 1
list = @registry[channel]
if list.empty?
m.reply "no urls seen yet for channel #{channel}"
else
reply_urls :msg => m, :channel => channel, :list => list, :max => max
end
end
def search(m, params)
channel = params[:channel] ? params[:channel] : m.target
max = params[:limit].to_i
string = params[:string]
max = 10 if max > 10
max = 1 if max < 1
regex = Regexp.new(string, Regexp::IGNORECASE)
list = @registry[channel].find_all {|url|
regex.match(url.url) || regex.match(url.nick) ||
(@bot.config['url.info_on_list'] && regex.match(url.info))
}
if list.empty?
m.reply "no matches for channel #{channel}"
else
reply_urls :msg => m, :channel => channel, :list => list, :max => max
end
end
end
plugin = UrlPlugin.new
plugin.map 'urls info *urls', :action => 'info'
plugin.map 'url info *urls', :action => 'info'
plugin.map 'urls search :channel :limit :string', :action => 'search',
:defaults => {:limit => 4},
:requirements => {:limit => /^\d+$/},
:public => false
plugin.map 'urls search :limit :string', :action => 'search',
:defaults => {:limit => 4},
:requirements => {:limit => /^\d+$/},
:private => false
plugin.map 'urls :channel :limit', :defaults => {:limit => 4},
:requirements => {:limit => /^\d+$/},
:public => false
plugin.map 'urls :limit', :defaults => {:limit => 4},
:requirements => {:limit => /^\d+$/},
:private => false
|
