4 # :title: rbot utilities provider
6 # Author:: Tom Gilbert <tom@linuxbrit.co.uk>
7 # Author:: Giuseppe "Oblomov" Bilotta <giuseppe.bilotta@gmail.com>
9 # Copyright:: (C) 2002-2006 Tom Gilbert
10 # Copyright:: (C) 2007 Giuseppe Bilotta
12 # TODO some of these Utils should be rewritten as extensions to the approriate
13 # standard Ruby classes and accordingly be moved to extends.rb
18 # Try to load htmlentities, fall back to an HTML escape table.
20 require 'htmlentities'
24 gems = require 'rubygems'
53 # extras codes, for future use...
67 'otimes' => '⊗',
76 'Epsilon' => 'Ε',
80 'Upsilon' => 'Υ',
82 'there4' => '∴',
87 'rsaquo' => '›',
101 'rdquo' => '”',
109 'lfloor' => '⌊',
116 'clubs' => '♣',
117 'diams' => '♦',
124 'Scaron' => 'Š',
130 'sbquo' => '‚',
143 'infin' => '∞',
148 'thinsp' => ' ',
150 'bdquo' => '„',
157 'mdash' => '—',
159 'permil' => '‰',
164 'forall' => '∀',
166 'rceil' => '⌉',
169 'lambda' => 'λ',
173 'dagger' => '†',
176 'image' => 'ℑ',
177 'alefsym' => 'ℵ',
183 'frasl' => '⁄',
185 'lowast' => '∗',
196 'oline' => '‾',
203 'empty' => '∅',
210 'weierp' => '℘',
215 'omicron' => 'ο',
216 'upsilon' => 'υ',
218 'Lambda' => 'Λ',
225 'scaron' => 'š',
226 'lsquo' => '‘',
234 'hellip' => '…',
238 'rfloor' => '⌋',
240 'crarr' => '↵',
242 'notin' => '∉',
243 'exist' => '∃',
246 'Dagger' => '‡',
247 'oplus' => '⊕',
253 'lsaquo' => '‹',
255 'Omicron' => 'Ο',
270 'sigmaf' => 'ς',
272 'minus' => '−',
275 'epsilon' => 'ε',
286 'spades' => '♠',
287 'rsquo' => '’',
291 'thetasym' => 'ϑ',
295 'ldquo' => '“',
296 'hearts' => '♥',
307 require 'htmlentities'
311 gems = require 'rubygems'
320 # define some regular expressions to be used for first_html_par
323 HX_REGEX = /<h(\d)(?:\s+[^>]*)?>(.*?)<\/h\1>/im
325 PAR_REGEX = /<p(?:\s+[^>]*)?>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
327 # Some blogging and forum platforms use spans or divs with a 'body' or 'message' or 'text' in their class
328 # to mark actual text
329 AFTER_PAR1_REGEX = /<\w+\s+[^>]*(?:body|message|text)[^>]*>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
331 # At worst, we can try stuff which is comprised between two <br>
332 AFTER_PAR2_REGEX = /<br(?:\s+[^>]*)?\/?>.*?<\/?(?:br|p|div|html|body|table|td|tr)(?:\s+[^>]*)?\/?>/im
340 # miscellaneous useful functions
342 @@bot = nil unless defined? @@bot
343 @@safe_save_dir = nil unless defined?(@@safe_save_dir)
350 debug "initializing utils"
352 @@safe_save_dir = "#{@@bot.botclass}/safe_save"
357 SEC_PER_HR = SEC_PER_MIN * 60
358 SEC_PER_DAY = SEC_PER_HR * 24
359 SEC_PER_MNTH = SEC_PER_DAY * 30
360 SEC_PER_YR = SEC_PER_MNTH * 12
362 def Utils.secs_to_string_case(array, var, string, plural)
365 array << "1 #{string}"
367 array << "#{var} #{plural}"
371 # turn a number of seconds into a human readable string, e.g
372 # 2 days, 3 hours, 18 minutes, 10 seconds
373 def Utils.secs_to_string(secs)
375 years, secs = secs.divmod SEC_PER_YR
376 secs_to_string_case(ret, years, _("year"), _("years")) if years > 0
377 months, secs = secs.divmod SEC_PER_MNTH
378 secs_to_string_case(ret, months, _("month"), _("months")) if months > 0
379 days, secs = secs.divmod SEC_PER_DAY
380 secs_to_string_case(ret, days, _("day"), _("days")) if days > 0
381 hours, secs = secs.divmod SEC_PER_HR
382 secs_to_string_case(ret, hours, _("hour"), _("hours")) if hours > 0
383 mins, secs = secs.divmod SEC_PER_MIN
384 secs_to_string_case(ret, mins, _("minute"), _("minutes")) if mins > 0
386 secs_to_string_case(ret, secs, _("second"), _("seconds")) if secs > 0 or ret.empty?
389 raise "Empty ret array!"
393 return [ret[0, ret.length-1].join(", ") , ret[-1]].join(_(" and "))
398 def Utils.safe_exec(command, *args)
401 return p.readlines.join("\n")
404 $stderr.reopen($stdout)
406 rescue Exception => e
407 puts "exec of #{command} led to exception: #{e.pretty_inspect}"
410 puts "exec of #{command} failed"
417 def Utils.safe_save(file)
418 raise 'No safe save directory defined!' if @@safe_save_dir.nil?
419 basename = File.basename(file)
420 temp = Tempfile.new(basename,@@safe_save_dir)
422 yield temp if block_given?
424 File.rename(temp.path, file)
428 def Utils.decode_html_entities(str)
429 if defined? ::HTMLEntities
430 return HTMLEntities.decode_entities(str)
432 str.gsub(/(&(.+?);)/) {
434 # remove the 0-paddng from unicode integers
436 symbol = "##{$1.to_i.to_s}"
439 # output the symbol's irc-translated character, or a * if it's unknown
440 UNESCAPE_TABLE[symbol] || [symbol[/\d+/].to_i].pack("U") rescue '*'
445 # Try to grab and IRCify the first HTML par (<p> tag) in the given string.
446 # If possible, grab the one after the first heading
448 # It is possible to pass some options to determine how the stripping
449 # occurs. Currently supported options are
450 # * :strip => Regex or String to strip at the beginning of the obtained
452 # * :min_spaces => Minimum number of spaces a paragraph should have
454 def Utils.ircify_first_html_par(xml_org, opts={})
455 if defined? ::Hpricot
456 Utils.ircify_first_html_par_wh(xml_org, opts)
458 Utils.ircify_first_html_par_woh(xml_org, opts)
463 def Utils.ircify_first_html_par_wh(xml_org, opts={})
464 doc = Hpricot(xml_org)
466 # Strip styles and scripts
467 (doc/"style|script").remove
472 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
474 min_spaces = opts[:min_spaces] || 8
475 min_spaces = 0 if min_spaces < 0
479 h = %w{h1 h2 h3 h4 h5 h6}
487 h_p_css = ar.join("|")
488 debug "css search: #{h_p_css}"
490 pre_h = pars = by_span = nil
493 debug "Minimum number of spaces: #{min_spaces}"
495 # Initial attempt: <p> that follows <h\d>
496 pre_h = doc/h_p_css if pre_h.nil?
497 debug "Hx: found: #{pre_h.pretty_inspect}"
500 txt = p.to_html.ircify_html
501 txt.sub!(strip, '') if strip
502 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
503 break unless txt.empty? or txt.count(" ") < min_spaces
506 return txt unless txt.empty? or txt.count(" ") < min_spaces
508 # Second natural attempt: just get any <p>
509 pars = doc/"p" if pars.nil?
510 debug "par: found: #{pars.pretty_inspect}"
513 txt = p.to_html.ircify_html
514 txt.sub!(strip, '') if strip
515 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
516 break unless txt.empty? or txt.count(" ") < min_spaces
519 return txt unless txt.empty? or txt.count(" ") < min_spaces
521 # Nothing yet ... let's get drastic: we look for non-par elements too,
522 # but only for those that match something that we know is likely to
525 # Some blogging and forum platforms use spans or divs with a 'body' or
526 # 'message' or 'text' in their class to mark actual text. Since we want
527 # the class match to be partial and case insensitive, we collect
528 # the common elements that may have this class and then filter out those
531 by_span = Hpricot::Elements[]
532 pre_pars = doc/"div|span|td|tr|tbody|table"
534 by_span.push el if el.class =~ /body|message|text/i
536 debug "other \#1: found: #{by_span.pretty_inspect}"
541 txt = p.to_html.ircify_html
542 txt.sub!(strip, '') if strip
543 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
544 break unless txt.empty? or txt.count(" ") < min_spaces
547 return txt unless txt.empty? or txt.count(" ") < min_spaces
549 # At worst, we can try stuff which is comprised between two <br>
552 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
553 return txt unless txt.count(" ") < min_spaces
554 break if min_spaces == 0
560 def Utils.ircify_first_html_par_woh(xml_org, opts={})
561 xml = xml_org.gsub(/<!--.*?-->/m, '').gsub(/<script(?:\s+[^>]*)?>.*?<\/script>/im, "").gsub(/<style(?:\s+[^>]*)?>.*?<\/style>/im, "")
564 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
566 min_spaces = opts[:min_spaces] || 8
567 min_spaces = 0 if min_spaces < 0
572 debug "Minimum number of spaces: #{min_spaces}"
573 header_found = xml.match(HX_REGEX)
576 while txt.empty? or txt.count(" ") < min_spaces
577 candidate = header_found[PAR_REGEX]
578 break unless candidate
579 txt = candidate.ircify_html
581 txt.sub!(strip, '') if strip
582 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
586 return txt unless txt.empty? or txt.count(" ") < min_spaces
588 # If we haven't found a first par yet, try to get it from the whole
591 while txt.empty? or txt.count(" ") < min_spaces
592 candidate = header_found[PAR_REGEX]
593 break unless candidate
594 txt = candidate.ircify_html
596 txt.sub!(strip, '') if strip
597 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
600 return txt unless txt.empty? or txt.count(" ") < min_spaces
602 # Nothing yet ... let's get drastic: we look for non-par elements too,
603 # but only for those that match something that we know is likely to
608 while txt.empty? or txt.count(" ") < min_spaces
609 candidate = header_found[AFTER_PAR1_REGEX]
610 break unless candidate
611 txt = candidate.ircify_html
613 txt.sub!(strip, '') if strip
614 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
617 return txt unless txt.empty? or txt.count(" ") < min_spaces
621 while txt.empty? or txt.count(" ") < min_spaces
622 candidate = header_found[AFTER_PAR2_REGEX]
623 break unless candidate
624 txt = candidate.ircify_html
626 txt.sub!(strip, '') if strip
627 debug "(other attempt \#2) #{txt.inspect} has #{txt.count(" ")} spaces"
630 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
631 return txt unless txt.count(" ") < min_spaces
632 break if min_spaces == 0
637 # Get the first pars of the first _count_ _urls_.
638 # The pages are downloaded using the bot httputil service.
639 # Returns an array of the first paragraphs fetched.
640 # If (optional) _opts_ :message is specified, those paragraphs are
641 # echoed as replies to the IRC message passed as _opts_ :message
643 def Utils.get_first_pars(urls, count, opts={})
647 while count > 0 and urls.length > 0
651 # FIXME what happens if some big file is returned? We should share
652 # code with the url plugin to only retrieve partial file content!
653 xml = self.bot.httputil.get(url)
655 debug "Unable to retrieve #{url}"
658 par = Utils.ircify_first_html_par(xml, opts)
660 debug "No first par found\n#{xml}"
661 # FIXME only do this if the 'url' plugin is loaded
662 # TODO even better, put the code here
663 # par = @bot.plugins['url'].get_title_from_html(xml)
669 msg.reply "[#{idx}] #{par}", :overlong => :truncate if msg
679 Irc::Utils.bot = Irc::Bot::Plugins.manager.bot