4 # :title: rbot utilities provider
6 # Author:: Tom Gilbert <tom@linuxbrit.co.uk>
7 # Author:: Giuseppe "Oblomov" Bilotta <giuseppe.bilotta@gmail.com>
9 # Copyright:: (C) 2002-2006 Tom Gilbert
10 # Copyright:: (C) 2007 Giuseppe Bilotta
12 # TODO some of these Utils should be rewritten as extensions to the approriate
13 # standard Ruby classes and accordingly be moved to extends.rb
18 # Try to load htmlentities, fall back to an HTML escape table.
20 require 'htmlentities'
24 gems = require 'rubygems'
121 AFTER_PAR_PATH = /^(?:div|span)$/
122 AFTER_PAR_EX = /^(?:td|tr|tbody|table)$/
123 AFTER_PAR_CLASS = /body|message|text/i
129 gems = require 'rubygems'
138 # Some regular expressions to manage HTML data
141 TITLE_REGEX = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im
144 HX_REGEX = /<h(\d)(?:\s+[^>]*)?>(.*?)<\/h\1>/im
146 PAR_REGEX = /<p(?:\s+[^>]*)?>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
148 # Some blogging and forum platforms use spans or divs with a 'body' or 'message' or 'text' in their class
149 # to mark actual text
150 AFTER_PAR1_REGEX = /<\w+\s+[^>]*(?:body|message|text)[^>]*>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
152 # At worst, we can try stuff which is comprised between two <br>
153 AFTER_PAR2_REGEX = /<br(?:\s+[^>]*)?\/?>.*?<\/?(?:br|p|div|html|body|table|td|tr)(?:\s+[^>]*)?\/?>/im
161 # Miscellaneous useful functions
163 @@bot = nil unless defined? @@bot
164 @@safe_save_dir = nil unless defined?(@@safe_save_dir)
171 # Set up some Utils routines which depend on the associated bot.
173 debug "initializing utils"
175 @@safe_save_dir = "#{@@bot.botclass}/safe_save"
182 SEC_PER_HR = SEC_PER_MIN * 60
184 SEC_PER_DAY = SEC_PER_HR * 24
185 # Seconds per (30-day) month
186 SEC_PER_MNTH = SEC_PER_DAY * 30
187 # Second per (30*12 = 360 day) year
188 SEC_PER_YR = SEC_PER_MNTH * 12
190 # Auxiliary method needed by Utils.secs_to_string
191 def Utils.secs_to_string_case(array, var, string, plural)
194 array << "1 #{string}"
196 array << "#{var} #{plural}"
200 # Turn a number of seconds into a human readable string, e.g
201 # 2 days, 3 hours, 18 minutes and 10 seconds
202 def Utils.secs_to_string(secs)
204 years, secs = secs.divmod SEC_PER_YR
205 secs_to_string_case(ret, years, _("year"), _("years")) if years > 0
206 months, secs = secs.divmod SEC_PER_MNTH
207 secs_to_string_case(ret, months, _("month"), _("months")) if months > 0
208 days, secs = secs.divmod SEC_PER_DAY
209 secs_to_string_case(ret, days, _("day"), _("days")) if days > 0
210 hours, secs = secs.divmod SEC_PER_HR
211 secs_to_string_case(ret, hours, _("hour"), _("hours")) if hours > 0
212 mins, secs = secs.divmod SEC_PER_MIN
213 secs_to_string_case(ret, mins, _("minute"), _("minutes")) if mins > 0
215 secs_to_string_case(ret, secs, _("second"), _("seconds")) if secs > 0 or ret.empty?
218 raise "Empty ret array!"
222 return [ret[0, ret.length-1].join(", ") , ret[-1]].join(_(" and "))
227 # Execute an external program, returning a String obtained by redirecting
228 # the program's standards errors and output
230 def Utils.safe_exec(command, *args)
233 return p.readlines.join("\n")
236 $stderr.reopen($stdout)
238 rescue Exception => e
239 puts "exec of #{command} led to exception: #{e.pretty_inspect}"
242 puts "exec of #{command} failed"
249 # Safely (atomically) save to _file_, by passing a tempfile to the block
250 # and then moving the tempfile to its final location when done.
252 # call-seq: Utils.safe_save(file, &block)
254 def Utils.safe_save(file)
255 raise 'No safe save directory defined!' if @@safe_save_dir.nil?
256 basename = File.basename(file)
257 temp = Tempfile.new(basename,@@safe_save_dir)
259 yield temp if block_given?
261 File.rename(temp.path, file)
265 # Decode HTML entities in the String _str_, using HTMLEntities if the
266 # package was found, or UNESCAPE_TABLE otherwise.
268 def Utils.decode_html_entities(str)
269 if defined? ::HTMLEntities
270 return HTMLEntities.decode_entities(str)
272 str.gsub(/(&(.+?);)/) {
274 # remove the 0-paddng from unicode integers
276 symbol = "##{$1.to_i.to_s}"
279 # output the symbol's irc-translated character, or a * if it's unknown
280 UNESCAPE_TABLE[symbol] || (symbol.match(/^\d+$/) ? [$0.to_i].pack("U") : '*')
285 # Try to grab and IRCify the first HTML par (<p> tag) in the given string.
286 # If possible, grab the one after the first heading
288 # It is possible to pass some options to determine how the stripping
289 # occurs. Currently supported options are
290 # strip:: Regex or String to strip at the beginning of the obtained
292 # min_spaces:: minimum number of spaces a paragraph should have
294 def Utils.ircify_first_html_par(xml_org, opts={})
295 if defined? ::Hpricot
296 Utils.ircify_first_html_par_wh(xml_org, opts)
298 Utils.ircify_first_html_par_woh(xml_org, opts)
302 # HTML first par grabber using hpricot
303 def Utils.ircify_first_html_par_wh(xml_org, opts={})
304 doc = Hpricot(xml_org)
306 # Strip styles and scripts
307 (doc/"style|script").remove
312 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
314 min_spaces = opts[:min_spaces] || 8
315 min_spaces = 0 if min_spaces < 0
319 pre_h = pars = by_span = nil
322 debug "Minimum number of spaces: #{min_spaces}"
324 # Initial attempt: <p> that follows <h\d>
326 pre_h = Hpricot::Elements[]
328 doc.search("*") { |e|
334 pre_h << e if found_h
337 debug "Hx: found: #{pre_h.pretty_inspect}"
342 txt = p.to_html.ircify_html
343 txt.sub!(strip, '') if strip
344 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
345 break unless txt.empty? or txt.count(" ") < min_spaces
348 return txt unless txt.empty? or txt.count(" ") < min_spaces
350 # Second natural attempt: just get any <p>
351 pars = doc/"p" if pars.nil?
352 debug "par: found: #{pars.pretty_inspect}"
355 txt = p.to_html.ircify_html
356 txt.sub!(strip, '') if strip
357 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
358 break unless txt.empty? or txt.count(" ") < min_spaces
361 return txt unless txt.empty? or txt.count(" ") < min_spaces
363 # Nothing yet ... let's get drastic: we look for non-par elements too,
364 # but only for those that match something that we know is likely to
367 # Some blogging and forum platforms use spans or divs with a 'body' or
368 # 'message' or 'text' in their class to mark actual text. Since we want
369 # the class match to be partial and case insensitive, we collect
370 # the common elements that may have this class and then filter out those
371 # we don't need. If no divs or spans are found, we'll accept additional
372 # elements too (td, tr, tbody, table).
374 by_span = Hpricot::Elements[]
375 extra = Hpricot::Elements[]
376 doc.search("*") { |el|
377 next if el.bogusetag?
380 by_span.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
382 extra.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
385 if by_span.empty? and not extra.empty?
388 debug "other \#1: found: #{by_span.pretty_inspect}"
393 txt = p.to_html.ircify_html
394 txt.sub!(strip, '') if strip
395 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
396 break unless txt.empty? or txt.count(" ") < min_spaces
399 return txt unless txt.empty? or txt.count(" ") < min_spaces
401 # At worst, we can try stuff which is comprised between two <br>
404 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
405 return txt unless txt.count(" ") < min_spaces
406 break if min_spaces == 0
411 # HTML first par grabber without hpricot
412 def Utils.ircify_first_html_par_woh(xml_org, opts={})
413 xml = xml_org.gsub(/<!--.*?-->/m, '').gsub(/<script(?:\s+[^>]*)?>.*?<\/script>/im, "").gsub(/<style(?:\s+[^>]*)?>.*?<\/style>/im, "")
416 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
418 min_spaces = opts[:min_spaces] || 8
419 min_spaces = 0 if min_spaces < 0
424 debug "Minimum number of spaces: #{min_spaces}"
425 header_found = xml.match(HX_REGEX)
428 while txt.empty? or txt.count(" ") < min_spaces
429 candidate = header_found[PAR_REGEX]
430 break unless candidate
431 txt = candidate.ircify_html
433 txt.sub!(strip, '') if strip
434 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
438 return txt unless txt.empty? or txt.count(" ") < min_spaces
440 # If we haven't found a first par yet, try to get it from the whole
443 while txt.empty? or txt.count(" ") < min_spaces
444 candidate = header_found[PAR_REGEX]
445 break unless candidate
446 txt = candidate.ircify_html
448 txt.sub!(strip, '') if strip
449 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
452 return txt unless txt.empty? or txt.count(" ") < min_spaces
454 # Nothing yet ... let's get drastic: we look for non-par elements too,
455 # but only for those that match something that we know is likely to
460 while txt.empty? or txt.count(" ") < min_spaces
461 candidate = header_found[AFTER_PAR1_REGEX]
462 break unless candidate
463 txt = candidate.ircify_html
465 txt.sub!(strip, '') if strip
466 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
469 return txt unless txt.empty? or txt.count(" ") < min_spaces
473 while txt.empty? or txt.count(" ") < min_spaces
474 candidate = header_found[AFTER_PAR2_REGEX]
475 break unless candidate
476 txt = candidate.ircify_html
478 txt.sub!(strip, '') if strip
479 debug "(other attempt \#2) #{txt.inspect} has #{txt.count(" ")} spaces"
482 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
483 return txt unless txt.count(" ") < min_spaces
484 break if min_spaces == 0
489 # This method extracts title, content (first par) and extra
490 # information from the given document _doc_.
492 # _doc_ can be an URI, a Net::HTTPResponse or a String.
494 # If _doc_ is a String, only title and content information
495 # are retrieved (if possible), using standard methods.
497 # If _doc_ is an URI or a Net::HTTPResponse, additional
498 # information is retrieved, and special title/summary
499 # extraction routines are used if possible.
501 def Utils.get_html_info(doc, opts={})
504 Utils.get_string_html_info(doc, opts)
505 when Net::HTTPResponse
506 Utils.get_resp_html_info(doc, opts)
509 @@bot.httputil.get_response(doc) { |resp|
510 ret = Utils.get_resp_html_info(resp, opts)
518 class ::UrlLinkError < RuntimeError
521 # This method extracts title, content (first par) and extra
522 # information from the given Net::HTTPResponse _resp_.
524 # Currently, the only accepted option (in _opts_) is
525 # uri_fragment:: the URI fragment of the original request
527 # Returns a Hash with the following keys:
528 # title:: the title of the document (if any)
529 # content:: the first paragraph of the document (if any)
531 # the headers of the Net::HTTPResponse. The value is
532 # a Hash whose keys are lowercase forms of the HTTP
533 # header fields, and whose values are Arrays.
535 def Utils.get_resp_html_info(resp, opts={})
538 when Net::HTTPSuccess
539 ret[:headers] = resp.to_hash
541 partial = resp.partial_body(@@bot.config['http.info_bytes'])
542 if resp['content-type'] =~ /^text\/|(?:x|ht)ml/
543 loc = URI.parse(resp['x-rbot-location'] || resp['location']) rescue nil
544 if loc and loc.fragment and not loc.fragment.empty?
545 opts[:uri_fragment] ||= loc.fragment
547 ret.merge!(Utils.get_string_html_info(partial, opts))
551 raise UrlLinkError, "getting link (#{resp.code} - #{resp.message})"
555 # This method extracts title and content (first par)
556 # from the given HTML or XML document _text_, using
557 # standard methods (String#ircify_html_title,
558 # Utils.ircify_first_html_par)
560 # Currently, the only accepted option (in _opts_) is
561 # uri_fragment:: the URI fragment of the original request
563 def Utils.get_string_html_info(text, opts={})
565 title = txt.ircify_html_title
566 if frag = opts[:uri_fragment] and not frag.empty?
567 fragreg = /.*?<a\s+[^>]*name=["']?#{frag}["']?.*?>/im
571 c_opts[:strip] ||= title
572 content = Utils.ircify_first_html_par(txt, c_opts)
573 content = nil if content.empty?
574 return {:title => title, :content => content}
577 # Get the first pars of the first _count_ _urls_.
578 # The pages are downloaded using the bot httputil service.
579 # Returns an array of the first paragraphs fetched.
580 # If (optional) _opts_ :message is specified, those paragraphs are
581 # echoed as replies to the IRC message passed as _opts_ :message
583 def Utils.get_first_pars(urls, count, opts={})
587 while count > 0 and urls.length > 0
592 info = Utils.get_html_info(URI.parse(url), opts)
598 msg.reply "[#{idx}] #{par}", :overlong => :truncate if msg
602 debug "Unable to retrieve #{url}: #{$!}"
612 Irc::Utils.bot = Irc::Bot::Plugins.manager.bot