4 # :title: rbot utilities provider
6 # Author:: Tom Gilbert <tom@linuxbrit.co.uk>
7 # Author:: Giuseppe "Oblomov" Bilotta <giuseppe.bilotta@gmail.com>
9 # Copyright:: (C) 2002-2006 Tom Gilbert
10 # Copyright:: (C) 2007 Giuseppe Bilotta
12 # TODO some of these Utils should be rewritten as extensions to the approriate
13 # standard Ruby classes and accordingly be moved to extends.rb
18 # Try to load htmlentities, fall back to an HTML escape table.
20 require 'htmlentities'
24 gems = require 'rubygems'
53 # extras codes, for future use...
67 'otimes' => '⊗',
76 'Epsilon' => 'Ε',
80 'Upsilon' => 'Υ',
82 'there4' => '∴',
87 'rsaquo' => '›',
101 'rdquo' => '”',
109 'lfloor' => '⌊',
116 'clubs' => '♣',
117 'diams' => '♦',
124 'Scaron' => 'Š',
130 'sbquo' => '‚',
143 'infin' => '∞',
148 'thinsp' => ' ',
150 'bdquo' => '„',
157 'mdash' => '—',
159 'permil' => '‰',
164 'forall' => '∀',
166 'rceil' => '⌉',
169 'lambda' => 'λ',
173 'dagger' => '†',
176 'image' => 'ℑ',
177 'alefsym' => 'ℵ',
183 'frasl' => '⁄',
185 'lowast' => '∗',
196 'oline' => '‾',
203 'empty' => '∅',
210 'weierp' => '℘',
215 'omicron' => 'ο',
216 'upsilon' => 'υ',
218 'Lambda' => 'Λ',
225 'scaron' => 'š',
226 'lsquo' => '‘',
234 'hellip' => '…',
238 'rfloor' => '⌋',
240 'crarr' => '↵',
242 'notin' => '∉',
243 'exist' => '∃',
246 'Dagger' => '‡',
247 'oplus' => '⊕',
253 'lsaquo' => '‹',
255 'Omicron' => 'Ο',
270 'sigmaf' => 'ς',
272 'minus' => '−',
275 'epsilon' => 'ε',
286 'spades' => '♠',
287 'rsquo' => '’',
291 'thetasym' => 'ϑ',
295 'ldquo' => '“',
296 'hearts' => '♥',
310 AFTER_PAR_PATH = /^(?:div|span|td|tr|tbody|table)$/
311 AFTER_PAR_CLASS = /body|message|text/i
317 gems = require 'rubygems'
326 # Some regular expressions to manage HTML data
329 TITLE_REGEX = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im
332 HX_REGEX = /<h(\d)(?:\s+[^>]*)?>(.*?)<\/h\1>/im
334 PAR_REGEX = /<p(?:\s+[^>]*)?>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
336 # Some blogging and forum platforms use spans or divs with a 'body' or 'message' or 'text' in their class
337 # to mark actual text
338 AFTER_PAR1_REGEX = /<\w+\s+[^>]*(?:body|message|text)[^>]*>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
340 # At worst, we can try stuff which is comprised between two <br>
341 AFTER_PAR2_REGEX = /<br(?:\s+[^>]*)?\/?>.*?<\/?(?:br|p|div|html|body|table|td|tr)(?:\s+[^>]*)?\/?>/im
349 # Miscellaneous useful functions
351 @@bot = nil unless defined? @@bot
352 @@safe_save_dir = nil unless defined?(@@safe_save_dir)
359 # Set up some Utils routines which depend on the associated bot.
361 debug "initializing utils"
363 @@safe_save_dir = "#{@@bot.botclass}/safe_save"
370 SEC_PER_HR = SEC_PER_MIN * 60
372 SEC_PER_DAY = SEC_PER_HR * 24
373 # Seconds per (30-day) month
374 SEC_PER_MNTH = SEC_PER_DAY * 30
375 # Second per (30*12 = 360 day) year
376 SEC_PER_YR = SEC_PER_MNTH * 12
378 # Auxiliary method needed by Utils.secs_to_string
379 def Utils.secs_to_string_case(array, var, string, plural)
382 array << "1 #{string}"
384 array << "#{var} #{plural}"
388 # Turn a number of seconds into a human readable string, e.g
389 # 2 days, 3 hours, 18 minutes and 10 seconds
390 def Utils.secs_to_string(secs)
392 years, secs = secs.divmod SEC_PER_YR
393 secs_to_string_case(ret, years, _("year"), _("years")) if years > 0
394 months, secs = secs.divmod SEC_PER_MNTH
395 secs_to_string_case(ret, months, _("month"), _("months")) if months > 0
396 days, secs = secs.divmod SEC_PER_DAY
397 secs_to_string_case(ret, days, _("day"), _("days")) if days > 0
398 hours, secs = secs.divmod SEC_PER_HR
399 secs_to_string_case(ret, hours, _("hour"), _("hours")) if hours > 0
400 mins, secs = secs.divmod SEC_PER_MIN
401 secs_to_string_case(ret, mins, _("minute"), _("minutes")) if mins > 0
403 secs_to_string_case(ret, secs, _("second"), _("seconds")) if secs > 0 or ret.empty?
406 raise "Empty ret array!"
410 return [ret[0, ret.length-1].join(", ") , ret[-1]].join(_(" and "))
415 # Execute an external program, returning a String obtained by redirecting
416 # the program's standards errors and output
418 def Utils.safe_exec(command, *args)
421 return p.readlines.join("\n")
424 $stderr.reopen($stdout)
426 rescue Exception => e
427 puts "exec of #{command} led to exception: #{e.pretty_inspect}"
430 puts "exec of #{command} failed"
437 # Safely (atomically) save to _file_, by passing a tempfile to the block
438 # and then moving the tempfile to its final location when done.
440 # call-seq: Utils.safe_save(file, &block)
442 def Utils.safe_save(file)
443 raise 'No safe save directory defined!' if @@safe_save_dir.nil?
444 basename = File.basename(file)
445 temp = Tempfile.new(basename,@@safe_save_dir)
447 yield temp if block_given?
449 File.rename(temp.path, file)
453 # Decode HTML entities in the String _str_, using HTMLEntities if the
454 # package was found, or UNESCAPE_TABLE otherwise.
456 def Utils.decode_html_entities(str)
457 if defined? ::HTMLEntities
458 return HTMLEntities.decode_entities(str)
460 str.gsub(/(&(.+?);)/) {
462 # remove the 0-paddng from unicode integers
464 symbol = "##{$1.to_i.to_s}"
467 # output the symbol's irc-translated character, or a * if it's unknown
468 UNESCAPE_TABLE[symbol] || [symbol[/\d+/].to_i].pack("U") rescue '*'
473 # Try to grab and IRCify the first HTML par (<p> tag) in the given string.
474 # If possible, grab the one after the first heading
476 # It is possible to pass some options to determine how the stripping
477 # occurs. Currently supported options are
478 # strip:: Regex or String to strip at the beginning of the obtained
480 # min_spaces:: minimum number of spaces a paragraph should have
482 def Utils.ircify_first_html_par(xml_org, opts={})
483 if defined? ::Hpricot
484 Utils.ircify_first_html_par_wh(xml_org, opts)
486 Utils.ircify_first_html_par_woh(xml_org, opts)
490 # HTML first par grabber using hpricot
491 def Utils.ircify_first_html_par_wh(xml_org, opts={})
492 doc = Hpricot(xml_org)
494 # Strip styles and scripts
495 (doc/"style|script").remove
500 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
502 min_spaces = opts[:min_spaces] || 8
503 min_spaces = 0 if min_spaces < 0
507 pre_h = pars = by_span = nil
510 debug "Minimum number of spaces: #{min_spaces}"
512 # Initial attempt: <p> that follows <h\d>
514 pre_h = Hpricot::Elements[]
516 doc.search("*") { |e|
522 pre_h << e if found_h
525 debug "Hx: found: #{pre_h.pretty_inspect}"
530 txt = p.to_html.ircify_html
531 txt.sub!(strip, '') if strip
532 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
533 break unless txt.empty? or txt.count(" ") < min_spaces
536 return txt unless txt.empty? or txt.count(" ") < min_spaces
538 # Second natural attempt: just get any <p>
539 pars = doc/"p" if pars.nil?
540 debug "par: found: #{pars.pretty_inspect}"
543 txt = p.to_html.ircify_html
544 txt.sub!(strip, '') if strip
545 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
546 break unless txt.empty? or txt.count(" ") < min_spaces
549 return txt unless txt.empty? or txt.count(" ") < min_spaces
551 # Nothing yet ... let's get drastic: we look for non-par elements too,
552 # but only for those that match something that we know is likely to
555 # Some blogging and forum platforms use spans or divs with a 'body' or
556 # 'message' or 'text' in their class to mark actual text. Since we want
557 # the class match to be partial and case insensitive, we collect
558 # the common elements that may have this class and then filter out those
561 by_span = Hpricot::Elements[]
562 doc.search("*") { |el|
563 next if el.bogusetag?
564 by_span.push el if el.pathname =~ AFTER_PAR_PATH and (el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS)
566 debug "other \#1: found: #{by_span.pretty_inspect}"
571 txt = p.to_html.ircify_html
572 txt.sub!(strip, '') if strip
573 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
574 break unless txt.empty? or txt.count(" ") < min_spaces
577 return txt unless txt.empty? or txt.count(" ") < min_spaces
579 # At worst, we can try stuff which is comprised between two <br>
582 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
583 return txt unless txt.count(" ") < min_spaces
584 break if min_spaces == 0
589 # HTML first par grabber without hpricot
590 def Utils.ircify_first_html_par_woh(xml_org, opts={})
591 xml = xml_org.gsub(/<!--.*?-->/m, '').gsub(/<script(?:\s+[^>]*)?>.*?<\/script>/im, "").gsub(/<style(?:\s+[^>]*)?>.*?<\/style>/im, "")
594 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
596 min_spaces = opts[:min_spaces] || 8
597 min_spaces = 0 if min_spaces < 0
602 debug "Minimum number of spaces: #{min_spaces}"
603 header_found = xml.match(HX_REGEX)
606 while txt.empty? or txt.count(" ") < min_spaces
607 candidate = header_found[PAR_REGEX]
608 break unless candidate
609 txt = candidate.ircify_html
611 txt.sub!(strip, '') if strip
612 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
616 return txt unless txt.empty? or txt.count(" ") < min_spaces
618 # If we haven't found a first par yet, try to get it from the whole
621 while txt.empty? or txt.count(" ") < min_spaces
622 candidate = header_found[PAR_REGEX]
623 break unless candidate
624 txt = candidate.ircify_html
626 txt.sub!(strip, '') if strip
627 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
630 return txt unless txt.empty? or txt.count(" ") < min_spaces
632 # Nothing yet ... let's get drastic: we look for non-par elements too,
633 # but only for those that match something that we know is likely to
638 while txt.empty? or txt.count(" ") < min_spaces
639 candidate = header_found[AFTER_PAR1_REGEX]
640 break unless candidate
641 txt = candidate.ircify_html
643 txt.sub!(strip, '') if strip
644 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
647 return txt unless txt.empty? or txt.count(" ") < min_spaces
651 while txt.empty? or txt.count(" ") < min_spaces
652 candidate = header_found[AFTER_PAR2_REGEX]
653 break unless candidate
654 txt = candidate.ircify_html
656 txt.sub!(strip, '') if strip
657 debug "(other attempt \#2) #{txt.inspect} has #{txt.count(" ")} spaces"
660 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
661 return txt unless txt.count(" ") < min_spaces
662 break if min_spaces == 0
667 # This method extracts title, content (first par) and extra
668 # information from the given document _doc_.
670 # _doc_ can be an URI, a Net::HTTPResponse or a String.
672 # If _doc_ is a String, only title and content information
673 # are retrieved (if possible), using standard methods.
675 # If _doc_ is an URI or a Net::HTTPResponse, additional
676 # information is retrieved, and special title/summary
677 # extraction routines are used if possible.
679 def Utils.get_html_info(doc, opts={})
682 Utils.get_string_html_info(doc, opts)
683 when Net::HTTPResponse
684 Utils.get_resp_html_info(doc, opts)
686 if doc.fragment and not doc.fragment.empty?
687 opts[:uri_fragment] ||= doc.fragment
690 @@bot.httputil.get_response(doc) { |resp|
691 ret = Utils.get_resp_html_info(resp, opts)
699 class ::UrlLinkError < RuntimeError
702 # This method extracts title, content (first par) and extra
703 # information from the given Net::HTTPResponse _resp_.
705 # Currently, the only accepted option (in _opts_) is
706 # uri_fragment:: the URI fragment of the original request
708 # Returns a Hash with the following keys:
709 # title:: the title of the document (if any)
710 # content:: the first paragraph of the document (if any)
712 # the headers of the Net::HTTPResponse. The value is
713 # a Hash whose keys are lowercase forms of the HTTP
714 # header fields, and whose values are Arrays.
716 def Utils.get_resp_html_info(resp, opts={})
719 when Net::HTTPSuccess
720 ret[:headers] = resp.to_hash
722 if resp['content-type'] =~ /^text\/|(?:x|ht)ml/
723 partial = resp.partial_body(@@bot.config['http.info_bytes'])
724 ret.merge!(Utils.get_string_html_info(partial, opts))
728 raise UrlLinkError, "getting link (#{resp.code} - #{resp.message})"
732 # This method extracts title and content (first par)
733 # from the given HTML or XML document _text_, using
734 # standard methods (String#ircify_html_title,
735 # Utils.ircify_first_html_par)
737 # Currently, the only accepted option (in _opts_) is
738 # uri_fragment:: the URI fragment of the original request
740 def Utils.get_string_html_info(text, opts={})
742 title = txt.ircify_html_title
743 if frag = opts[:uri_fragment] and not frag.empty?
744 fragreg = /.*?<a\s+[^>]*name=["']?#{frag}["']?.*?>/im
748 c_opts[:strip] ||= title
749 content = Utils.ircify_first_html_par(txt, c_opts)
750 content = nil if content.empty?
751 return {:title => title, :content => content}
754 # Get the first pars of the first _count_ _urls_.
755 # The pages are downloaded using the bot httputil service.
756 # Returns an array of the first paragraphs fetched.
757 # If (optional) _opts_ :message is specified, those paragraphs are
758 # echoed as replies to the IRC message passed as _opts_ :message
760 def Utils.get_first_pars(urls, count, opts={})
764 while count > 0 and urls.length > 0
769 info = Utils.get_html_info(URI.parse(url), opts)
775 msg.reply "[#{idx}] #{par}", :overlong => :truncate if msg
779 debug "Unable to retrieve #{url}: #{$!}"
789 Irc::Utils.bot = Irc::Bot::Plugins.manager.bot