4 # :title: rbot utilities provider
6 # Author:: Tom Gilbert <tom@linuxbrit.co.uk>
7 # Author:: Giuseppe "Oblomov" Bilotta <giuseppe.bilotta@gmail.com>
9 # TODO some of these Utils should be rewritten as extensions to the approriate
10 # standard Ruby classes and accordingly be moved to extends.rb
15 # Try to load htmlentities, fall back to an HTML escape table.
17 require 'htmlentities'
21 gems = require 'rubygems'
118 AFTER_PAR_PATH = /^(?:div|span)$/
119 AFTER_PAR_EX = /^(?:td|tr|tbody|table)$/
120 AFTER_PAR_CLASS = /body|message|text/i
126 gems = require 'rubygems'
135 # Some regular expressions to manage HTML data
138 TITLE_REGEX = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im
141 HX_REGEX = /<h(\d)(?:\s+[^>]*)?>(.*?)<\/h\1>/im
143 PAR_REGEX = /<p(?:\s+[^>]*)?>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
145 # Some blogging and forum platforms use spans or divs with a 'body' or 'message' or 'text' in their class
146 # to mark actual text
147 AFTER_PAR1_REGEX = /<\w+\s+[^>]*(?:body|message|text)[^>]*>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
149 # At worst, we can try stuff which is comprised between two <br>
150 AFTER_PAR2_REGEX = /<br(?:\s+[^>]*)?\/?>.*?<\/?(?:br|p|div|html|body|table|td|tr)(?:\s+[^>]*)?\/?>/im
158 # Miscellaneous useful functions
160 @@bot = nil unless defined? @@bot
161 @@safe_save_dir = nil unless defined?(@@safe_save_dir)
168 # Set up some Utils routines which depend on the associated bot.
170 debug "initializing utils"
172 @@safe_save_dir = "#{@@bot.botclass}/safe_save"
179 SEC_PER_HR = SEC_PER_MIN * 60
181 SEC_PER_DAY = SEC_PER_HR * 24
182 # Seconds per (30-day) month
183 SEC_PER_MNTH = SEC_PER_DAY * 30
184 # Second per (30*12 = 360 day) year
185 SEC_PER_YR = SEC_PER_MNTH * 12
187 # Auxiliary method needed by Utils.secs_to_string
188 def Utils.secs_to_string_case(array, var, string, plural)
191 array << "1 #{string}"
193 array << "#{var} #{plural}"
197 # Turn a number of seconds into a human readable string, e.g
198 # 2 days, 3 hours, 18 minutes and 10 seconds
199 def Utils.secs_to_string(secs)
201 years, secs = secs.divmod SEC_PER_YR
202 secs_to_string_case(ret, years, _("year"), _("years")) if years > 0
203 months, secs = secs.divmod SEC_PER_MNTH
204 secs_to_string_case(ret, months, _("month"), _("months")) if months > 0
205 days, secs = secs.divmod SEC_PER_DAY
206 secs_to_string_case(ret, days, _("day"), _("days")) if days > 0
207 hours, secs = secs.divmod SEC_PER_HR
208 secs_to_string_case(ret, hours, _("hour"), _("hours")) if hours > 0
209 mins, secs = secs.divmod SEC_PER_MIN
210 secs_to_string_case(ret, mins, _("minute"), _("minutes")) if mins > 0
212 secs_to_string_case(ret, secs, _("second"), _("seconds")) if secs > 0 or ret.empty?
215 raise "Empty ret array!"
219 return [ret[0, ret.length-1].join(", ") , ret[-1]].join(_(" and "))
223 # Turn a number of seconds into a hours:minutes:seconds e.g.
224 # 3:18:10 or 5'12" or 7s
226 def Utils.secs_to_short(seconds)
227 secs = seconds.to_i # make sure it's an integer
228 mins, secs = secs.divmod 60
229 hours, mins = mins.divmod 60
231 return ("%s:%s:%s" % [hours, mins, secs])
233 return ("%s'%s\"" % [mins, secs])
235 return ("%ss" % [secs])
239 # Returns human readable time.
243 # :start_date, sets the time to measure against, defaults to now
244 # :date_format, used with <tt>to_formatted_s<tt>, default to :default
245 def Utils.timeago(time, options = {})
246 start_date = options.delete(:start_date) || Time.new
247 date_format = options.delete(:date_format) || :default
248 delta_minutes = (start_date.to_i - time.to_i).floor / 60
249 if delta_minutes.abs <= (8724*60) # eight weeks? I'm lazy to count days for longer than that
250 distance = Utils.distance_of_time_in_words(delta_minutes);
252 _("%{d} from now") % {:d => distance}
254 _("%{d} ago") % {:d => distance}
257 return _("on %{date}") % {:date => system_date.to_formatted_s(date_format)}
260 # Translates a number of minutes into verbal distances.
261 # e.g. 0.5 => less than a minute
262 # 70 => about one hour
263 def Utils.distance_of_time_in_words(minutes)
266 _("less than a minute")
268 _("%{m} minutes") % {:m => minutes}
272 _("%{m} hours") % {:m => (minutes / 60).round}
278 _("%{m} days") % {:m => (minutes / 1440).round}
283 # Execute an external program, returning a String obtained by redirecting
284 # the program's standards errors and output
286 def Utils.safe_exec(command, *args)
289 return p.readlines.join("\n")
292 $stderr.reopen($stdout)
294 rescue Exception => e
295 puts "exec of #{command} led to exception: #{e.pretty_inspect}"
298 puts "exec of #{command} failed"
305 # Safely (atomically) save to _file_, by passing a tempfile to the block
306 # and then moving the tempfile to its final location when done.
308 # call-seq: Utils.safe_save(file, &block)
310 def Utils.safe_save(file)
311 raise 'No safe save directory defined!' if @@safe_save_dir.nil?
312 basename = File.basename(file)
313 temp = Tempfile.new(basename,@@safe_save_dir)
315 yield temp if block_given?
317 File.rename(temp.path, file)
321 # Decode HTML entities in the String _str_, using HTMLEntities if the
322 # package was found, or UNESCAPE_TABLE otherwise.
324 def Utils.decode_html_entities(str)
325 if defined? ::HTMLEntities
326 return HTMLEntities.decode_entities(str)
328 str.gsub(/(&(.+?);)/) {
330 # remove the 0-paddng from unicode integers
331 if symbol =~ /^#(\d+)$/
332 symbol = $1.to_i.to_s
335 # output the symbol's irc-translated character, or a * if it's unknown
336 UNESCAPE_TABLE[symbol] || (symbol.match(/^\d+$/) ? [symbol.to_i].pack("U") : '*')
341 # Try to grab and IRCify the first HTML par (<p> tag) in the given string.
342 # If possible, grab the one after the first heading
344 # It is possible to pass some options to determine how the stripping
345 # occurs. Currently supported options are
346 # strip:: Regex or String to strip at the beginning of the obtained
348 # min_spaces:: minimum number of spaces a paragraph should have
350 def Utils.ircify_first_html_par(xml_org, opts={})
351 if defined? ::Hpricot
352 Utils.ircify_first_html_par_wh(xml_org, opts)
354 Utils.ircify_first_html_par_woh(xml_org, opts)
358 # HTML first par grabber using hpricot
359 def Utils.ircify_first_html_par_wh(xml_org, opts={})
360 doc = Hpricot(xml_org)
362 # Strip styles and scripts
363 (doc/"style|script").remove
368 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
370 min_spaces = opts[:min_spaces] || 8
371 min_spaces = 0 if min_spaces < 0
375 pre_h = pars = by_span = nil
378 debug "Minimum number of spaces: #{min_spaces}"
380 # Initial attempt: <p> that follows <h\d>
382 pre_h = Hpricot::Elements[]
384 doc.search("*") { |e|
390 pre_h << e if found_h
393 debug "Hx: found: #{pre_h.pretty_inspect}"
398 txt = p.to_html.ircify_html
399 txt.sub!(strip, '') if strip
400 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
401 break unless txt.empty? or txt.count(" ") < min_spaces
404 return txt unless txt.empty? or txt.count(" ") < min_spaces
406 # Second natural attempt: just get any <p>
407 pars = doc/"p" if pars.nil?
408 debug "par: found: #{pars.pretty_inspect}"
411 txt = p.to_html.ircify_html
412 txt.sub!(strip, '') if strip
413 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
414 break unless txt.empty? or txt.count(" ") < min_spaces
417 return txt unless txt.empty? or txt.count(" ") < min_spaces
419 # Nothing yet ... let's get drastic: we look for non-par elements too,
420 # but only for those that match something that we know is likely to
423 # Some blogging and forum platforms use spans or divs with a 'body' or
424 # 'message' or 'text' in their class to mark actual text. Since we want
425 # the class match to be partial and case insensitive, we collect
426 # the common elements that may have this class and then filter out those
427 # we don't need. If no divs or spans are found, we'll accept additional
428 # elements too (td, tr, tbody, table).
430 by_span = Hpricot::Elements[]
431 extra = Hpricot::Elements[]
432 doc.search("*") { |el|
433 next if el.bogusetag?
436 by_span.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
438 extra.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
441 if by_span.empty? and not extra.empty?
444 debug "other \#1: found: #{by_span.pretty_inspect}"
449 txt = p.to_html.ircify_html
450 txt.sub!(strip, '') if strip
451 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
452 break unless txt.empty? or txt.count(" ") < min_spaces
455 return txt unless txt.empty? or txt.count(" ") < min_spaces
457 # At worst, we can try stuff which is comprised between two <br>
460 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
461 return txt unless txt.count(" ") < min_spaces
462 break if min_spaces == 0
467 # HTML first par grabber without hpricot
468 def Utils.ircify_first_html_par_woh(xml_org, opts={})
469 xml = xml_org.gsub(/<!--.*?-->/m, '').gsub(/<script(?:\s+[^>]*)?>.*?<\/script>/im, "").gsub(/<style(?:\s+[^>]*)?>.*?<\/style>/im, "")
472 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
474 min_spaces = opts[:min_spaces] || 8
475 min_spaces = 0 if min_spaces < 0
480 debug "Minimum number of spaces: #{min_spaces}"
481 header_found = xml.match(HX_REGEX)
484 while txt.empty? or txt.count(" ") < min_spaces
485 candidate = header_found[PAR_REGEX]
486 break unless candidate
487 txt = candidate.ircify_html
489 txt.sub!(strip, '') if strip
490 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
494 return txt unless txt.empty? or txt.count(" ") < min_spaces
496 # If we haven't found a first par yet, try to get it from the whole
499 while txt.empty? or txt.count(" ") < min_spaces
500 candidate = header_found[PAR_REGEX]
501 break unless candidate
502 txt = candidate.ircify_html
504 txt.sub!(strip, '') if strip
505 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
508 return txt unless txt.empty? or txt.count(" ") < min_spaces
510 # Nothing yet ... let's get drastic: we look for non-par elements too,
511 # but only for those that match something that we know is likely to
516 while txt.empty? or txt.count(" ") < min_spaces
517 candidate = header_found[AFTER_PAR1_REGEX]
518 break unless candidate
519 txt = candidate.ircify_html
521 txt.sub!(strip, '') if strip
522 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
525 return txt unless txt.empty? or txt.count(" ") < min_spaces
529 while txt.empty? or txt.count(" ") < min_spaces
530 candidate = header_found[AFTER_PAR2_REGEX]
531 break unless candidate
532 txt = candidate.ircify_html
534 txt.sub!(strip, '') if strip
535 debug "(other attempt \#2) #{txt.inspect} has #{txt.count(" ")} spaces"
538 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
539 return txt unless txt.count(" ") < min_spaces
540 break if min_spaces == 0
545 # This method extracts title, content (first par) and extra
546 # information from the given document _doc_.
548 # _doc_ can be an URI, a Net::HTTPResponse or a String.
550 # If _doc_ is a String, only title and content information
551 # are retrieved (if possible), using standard methods.
553 # If _doc_ is an URI or a Net::HTTPResponse, additional
554 # information is retrieved, and special title/summary
555 # extraction routines are used if possible.
557 def Utils.get_html_info(doc, opts={})
560 Utils.get_string_html_info(doc, opts)
561 when Net::HTTPResponse
562 Utils.get_resp_html_info(doc, opts)
565 @@bot.httputil.get_response(doc) { |resp|
566 ret.replace Utils.get_resp_html_info(resp, opts)
574 class ::UrlLinkError < RuntimeError
577 # This method extracts title, content (first par) and extra
578 # information from the given Net::HTTPResponse _resp_.
580 # Currently, the only accepted options (in _opts_) are
581 # uri_fragment:: the URI fragment of the original request
582 # full_body:: get the whole body instead of
583 # @@bot.config['http.info_bytes'] bytes only
585 # Returns a DataStream with the following keys:
586 # text:: the (partial) body
587 # title:: the title of the document (if any)
588 # content:: the first paragraph of the document (if any)
590 # the headers of the Net::HTTPResponse. The value is
591 # a Hash whose keys are lowercase forms of the HTTP
592 # header fields, and whose values are Arrays.
594 def Utils.get_resp_html_info(resp, opts={})
596 when Net::HTTPSuccess
597 loc = URI.parse(resp['x-rbot-location'] || resp['location']) rescue nil
598 if loc and loc.fragment and not loc.fragment.empty?
599 opts[:uri_fragment] ||= loc.fragment
601 ret = DataStream.new(opts.dup)
602 ret[:headers] = resp.to_hash
603 ret[:text] = partial = opts[:full_body] ? resp.body : resp.partial_body(@@bot.config['http.info_bytes'])
605 filtered = Utils.try_htmlinfo_filters(ret)
609 elsif resp['content-type'] =~ /^text\/|(?:x|ht)ml/
610 ret.merge!(Utils.get_string_html_info(partial, opts))
614 raise UrlLinkError, "getting link (#{resp.code} - #{resp.message})"
618 # This method runs an appropriately-crafted DataStream _ds_ through the
619 # filters in the :htmlinfo filter group, in order. If one of the filters
620 # returns non-nil, its results are merged in _ds_ and returned. Otherwise
623 # The input DataStream shuold have the downloaded HTML as primary key
624 # (:text) and possibly a :headers key holding the resonse headers.
626 def Utils.try_htmlinfo_filters(ds)
627 filters = @@bot.filter_names(:htmlinfo)
628 return nil if filters.empty?
630 # TODO filter priority
632 debug "testing filter #{n}"
633 cur = @@bot.filter(@@bot.global_filter_name(n, :htmlinfo), ds)
634 debug "returned #{cur.pretty_inspect}"
637 return ds.merge(cur) if cur
640 # HTML info filters often need to check if the webpage location
641 # of a passed DataStream _ds_ matches a given Regexp.
642 def Utils.check_location(ds, rx)
645 loc = [h['x-rbot-location'],h['location']].flatten.grep(rx)
649 return loc.empty? ? nil : loc
652 # This method extracts title and content (first par)
653 # from the given HTML or XML document _text_, using
654 # standard methods (String#ircify_html_title,
655 # Utils.ircify_first_html_par)
657 # Currently, the only accepted option (in _opts_) is
658 # uri_fragment:: the URI fragment of the original request
660 def Utils.get_string_html_info(text, opts={})
661 debug "getting string html info"
663 title = txt.ircify_html_title
665 if frag = opts[:uri_fragment] and not frag.empty?
666 fragreg = /<a\s+(?:[^>]+\s+)?(?:name|id)=["']?#{frag}["']?[^>]*>/im
669 if txt.match(fragreg)
670 # grab the post-match
676 c_opts[:strip] ||= title
677 content = Utils.ircify_first_html_par(txt, c_opts)
678 content = nil if content.empty?
679 return {:title => title, :content => content}
682 # Get the first pars of the first _count_ _urls_.
683 # The pages are downloaded using the bot httputil service.
684 # Returns an array of the first paragraphs fetched.
685 # If (optional) _opts_ :message is specified, those paragraphs are
686 # echoed as replies to the IRC message passed as _opts_ :message
688 def Utils.get_first_pars(urls, count, opts={})
692 while count > 0 and urls.length > 0
697 info = Utils.get_html_info(URI.parse(url), opts)
703 msg.reply "[#{idx}] #{par}", :overlong => :truncate if msg
707 debug "Unable to retrieve #{url}: #{$!}"
717 Irc::Utils.bot = Irc::Bot::Plugins.manager.bot