5 # :title: rbot utilities provider
7 # Author:: Tom Gilbert <tom@linuxbrit.co.uk>
8 # Author:: Giuseppe "Oblomov" Bilotta <giuseppe.bilotta@gmail.com>
10 # TODO some of these Utils should be rewritten as extensions to the approriate
11 # standard Ruby classes and accordingly be moved to extends.rb
16 # Try to load htmlentities, fall back to an HTML escape table.
18 require 'htmlentities'
111 AFTER_PAR_PATH = /^(?:div|span)$/
112 AFTER_PAR_EX = /^(?:td|tr|tbody|table)$/
113 AFTER_PAR_CLASS = /body|message|text/i
119 # Some regular expressions to manage HTML data
122 TITLE_REGEX = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im
125 HX_REGEX = /<h(\d)(?:\s+[^>]*)?>(.*?)<\/h\1>/im
127 PAR_REGEX = /<p(?:\s+[^>]*)?>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
129 # Some blogging and forum platforms use spans or divs with a 'body' or 'message' or 'text' in their class
130 # to mark actual text
131 AFTER_PAR1_REGEX = /<\w+\s+[^>]*(?:body|message|text|post)[^>]*>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
133 # At worst, we can try stuff which is comprised between two <br>
134 AFTER_PAR2_REGEX = /<br(?:\s+[^>]*)?\/?>.*?<\/?(?:br|p|div|html|body|table|td|tr)(?:\s+[^>]*)?\/?>/im
141 # Miscellaneous useful functions
143 @@bot = nil unless defined? @@bot
144 @@safe_save_dir = nil unless defined?(@@safe_save_dir)
151 # Set up some Utils routines which depend on the associated bot.
153 debug "initializing utils"
155 @@safe_save_dir = @@bot.path('safe_save')
162 SEC_PER_HR = SEC_PER_MIN * 60
164 SEC_PER_DAY = SEC_PER_HR * 24
166 SEC_PER_WK = SEC_PER_DAY * 7
167 # Seconds per (30-day) month
168 SEC_PER_MNTH = SEC_PER_DAY * 30
169 # Second per (non-leap) year
170 SEC_PER_YR = SEC_PER_DAY * 365
172 # Auxiliary method needed by Utils.secs_to_string
173 def Utils.secs_to_string_case(array, var, string, plural)
176 array << "1 #{string}"
178 array << "#{var} #{plural}"
182 # Turn a number of seconds into a human readable string, e.g
183 # 2 days, 3 hours, 18 minutes and 10 seconds
184 def Utils.secs_to_string(secs)
186 years, secs = secs.divmod SEC_PER_YR
187 secs_to_string_case(ret, years, _("year"), _("years")) if years > 0
188 months, secs = secs.divmod SEC_PER_MNTH
189 secs_to_string_case(ret, months, _("month"), _("months")) if months > 0
190 days, secs = secs.divmod SEC_PER_DAY
191 secs_to_string_case(ret, days, _("day"), _("days")) if days > 0
192 hours, secs = secs.divmod SEC_PER_HR
193 secs_to_string_case(ret, hours, _("hour"), _("hours")) if hours > 0
194 mins, secs = secs.divmod SEC_PER_MIN
195 secs_to_string_case(ret, mins, _("minute"), _("minutes")) if mins > 0
197 secs_to_string_case(ret, secs, _("second"), _("seconds")) if secs > 0 or ret.empty?
200 raise "Empty ret array!"
204 return [ret[0, ret.length-1].join(", ") , ret[-1]].join(_(" and "))
208 # Turn a number of seconds into a hours:minutes:seconds e.g.
209 # 3:18:10 or 5'12" or 7s
211 def Utils.secs_to_short(seconds)
212 secs = seconds.to_i # make sure it's an integer
213 mins, secs = secs.divmod 60
214 hours, mins = mins.divmod 60
216 return ("%s:%s:%s" % [hours, mins, secs])
218 return ("%s'%s\"" % [mins, secs])
220 return ("%ss" % [secs])
224 # Returns human readable time.
228 # :start_date, sets the time to measure against, defaults to now
229 # :date_format, used with <tt>to_formatted_s<tt>, default to :default
230 def Utils.timeago(time, options = {})
231 start_date = options.delete(:start_date) || Time.new
232 date_format = options.delete(:date_format) || "%x"
233 delta = (start_date - time).round
237 distance = Utils.age_string(delta)
239 _("%{d} from now") % {:d => distance}
241 _("%{d} ago") % {:d => distance}
246 # Converts age in seconds to "nn units". Inspired by previous attempts
247 # but also gitweb's age_string() sub
248 def Utils.age_string(secs)
251 Utils.age_string(-secs)
252 when secs > 2*SEC_PER_YR
253 _("%{m} years") % { :m => secs/SEC_PER_YR }
254 when secs > 2*SEC_PER_MNTH
255 _("%{m} months") % { :m => secs/SEC_PER_MNTH }
256 when secs > 2*SEC_PER_WK
257 _("%{m} weeks") % { :m => secs/SEC_PER_WK }
258 when secs > 2*SEC_PER_DAY
259 _("%{m} days") % { :m => secs/SEC_PER_DAY }
260 when secs > 2*SEC_PER_HR
261 _("%{m} hours") % { :m => secs/SEC_PER_HR }
262 when (20*SEC_PER_MIN..40*SEC_PER_MIN).include?(secs)
264 when (50*SEC_PER_MIN..70*SEC_PER_MIN).include?(secs)
265 # _("about one hour")
267 when (80*SEC_PER_MIN..100*SEC_PER_MIN).include?(secs)
268 _("an hour and a half")
269 when secs > 2*SEC_PER_MIN
270 _("%{m} minutes") % { :m => secs/SEC_PER_MIN }
272 _("%{m} seconds") % { :m => secs }
278 # Execute an external program, returning a String obtained by redirecting
279 # the program's standards errors and output
281 # TODO: find a way to expose some common errors (e.g. Errno::NOENT)
283 def Utils.safe_exec(command, *args)
284 output = IO.popen("-") { |p|
286 break p.readlines.join("\n")
289 $stderr.reopen($stdout)
291 rescue Exception => e
292 puts "exception #{e.pretty_inspect} trying to run #{command}"
295 puts "exec of #{command} failed"
299 raise "safe execution of #{command} returned #{$?}" unless $?.success?
303 # Try executing an external program, returning true if the run was successful
304 # and false otherwise
305 def Utils.try_exec(command, *args)
309 $stderr.reopen($stdout)
311 rescue Exception => e
323 # Safely (atomically) save to _file_, by passing a tempfile to the block
324 # and then moving the tempfile to its final location when done.
326 # call-seq: Utils.safe_save(file, &block)
328 def Utils.safe_save(file)
329 raise 'No safe save directory defined!' if @@safe_save_dir.nil?
330 basename = File.basename(file)
331 temp = Tempfile.new(basename,@@safe_save_dir)
333 yield temp if block_given?
335 File.rename(temp.path, file)
339 # Decode HTML entities in the String _str_, using HTMLEntities if the
340 # package was found, or UNESCAPE_TABLE otherwise.
343 if defined? ::HTMLEntities
344 if ::HTMLEntities.respond_to? :decode_entities
345 def Utils.decode_html_entities(str)
346 return HTMLEntities.decode_entities(str)
349 @@html_entities = HTMLEntities.new
350 def Utils.decode_html_entities(str)
351 return @@html_entities.decode str
355 def Utils.decode_html_entities(str)
356 return str.gsub(/(&(.+?);)/) {
358 # remove the 0-paddng from unicode integers
360 when /^#x([0-9a-fA-F]+)$/
361 symbol = $1.to_i(16).to_s
363 symbol = $1.to_i.to_s
366 # output the symbol's irc-translated character, or a * if it's unknown
367 UNESCAPE_TABLE[symbol] || (symbol.match(/^\d+$/) ? [symbol.to_i].pack("U") : '*')
372 # Try to grab and IRCify the first HTML par (<p> tag) in the given string.
373 # If possible, grab the one after the first heading
375 # It is possible to pass some options to determine how the stripping
376 # occurs. Currently supported options are
377 # strip:: Regex or String to strip at the beginning of the obtained
379 # min_spaces:: minimum number of spaces a paragraph should have
381 def Utils.ircify_first_html_par(xml_org, opts={})
382 if defined? ::Hpricot
383 Utils.ircify_first_html_par_wh(xml_org, opts)
385 Utils.ircify_first_html_par_woh(xml_org, opts)
389 # HTML first par grabber using hpricot
390 def Utils.ircify_first_html_par_wh(xml_org, opts={})
391 doc = Hpricot(xml_org)
393 # Strip styles and scripts
394 (doc/"style|script").remove
399 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
401 min_spaces = opts[:min_spaces] || 8
402 min_spaces = 0 if min_spaces < 0
406 pre_h = pars = by_span = nil
409 debug "Minimum number of spaces: #{min_spaces}"
411 # Initial attempt: <p> that follows <h\d>
413 pre_h = Hpricot::Elements[]
415 doc.search("*") { |e|
421 pre_h << e if found_h
424 debug "Hx: found: #{pre_h.pretty_inspect}"
429 txt = p.to_html.ircify_html
430 txt.sub!(strip, '') if strip
431 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
432 break unless txt.empty? or txt.count(" ") < min_spaces
435 return txt unless txt.empty? or txt.count(" ") < min_spaces
437 # Second natural attempt: just get any <p>
438 pars = doc/"p" if pars.nil?
439 debug "par: found: #{pars.pretty_inspect}"
442 txt = p.to_html.ircify_html
443 txt.sub!(strip, '') if strip
444 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
445 break unless txt.empty? or txt.count(" ") < min_spaces
448 return txt unless txt.empty? or txt.count(" ") < min_spaces
450 # Nothing yet ... let's get drastic: we look for non-par elements too,
451 # but only for those that match something that we know is likely to
454 # Some blogging and forum platforms use spans or divs with a 'body' or
455 # 'message' or 'text' in their class to mark actual text. Since we want
456 # the class match to be partial and case insensitive, we collect
457 # the common elements that may have this class and then filter out those
458 # we don't need. If no divs or spans are found, we'll accept additional
459 # elements too (td, tr, tbody, table).
461 by_span = Hpricot::Elements[]
462 extra = Hpricot::Elements[]
463 doc.search("*") { |el|
464 next if el.bogusetag?
467 by_span.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
469 extra.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
472 if by_span.empty? and not extra.empty?
475 debug "other \#1: found: #{by_span.pretty_inspect}"
480 txt = p.to_html.ircify_html
481 txt.sub!(strip, '') if strip
482 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
483 break unless txt.empty? or txt.count(" ") < min_spaces
486 return txt unless txt.empty? or txt.count(" ") < min_spaces
488 # At worst, we can try stuff which is comprised between two <br>
491 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
492 return txt unless txt.count(" ") < min_spaces
493 break if min_spaces == 0
498 # HTML first par grabber without hpricot
499 def Utils.ircify_first_html_par_woh(xml_org, opts={})
500 xml = xml_org.gsub(/<!--.*?-->/m,
501 "").gsub(/<script(?:\s+[^>]*)?>.*?<\/script>/im,
502 "").gsub(/<style(?:\s+[^>]*)?>.*?<\/style>/im,
503 "").gsub(/<select(?:\s+[^>]*)?>.*?<\/select>/im,
507 strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
509 min_spaces = opts[:min_spaces] || 8
510 min_spaces = 0 if min_spaces < 0
515 debug "Minimum number of spaces: #{min_spaces}"
516 header_found = xml.match(HX_REGEX)
519 while txt.empty? or txt.count(" ") < min_spaces
520 candidate = header_found[PAR_REGEX]
521 break unless candidate
522 txt = candidate.ircify_html
524 txt.sub!(strip, '') if strip
525 debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
529 return txt unless txt.empty? or txt.count(" ") < min_spaces
531 # If we haven't found a first par yet, try to get it from the whole
534 while txt.empty? or txt.count(" ") < min_spaces
535 candidate = header_found[PAR_REGEX]
536 break unless candidate
537 txt = candidate.ircify_html
539 txt.sub!(strip, '') if strip
540 debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
543 return txt unless txt.empty? or txt.count(" ") < min_spaces
545 # Nothing yet ... let's get drastic: we look for non-par elements too,
546 # but only for those that match something that we know is likely to
551 while txt.empty? or txt.count(" ") < min_spaces
552 candidate = header_found[AFTER_PAR1_REGEX]
553 break unless candidate
554 txt = candidate.ircify_html
556 txt.sub!(strip, '') if strip
557 debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
560 return txt unless txt.empty? or txt.count(" ") < min_spaces
564 while txt.empty? or txt.count(" ") < min_spaces
565 candidate = header_found[AFTER_PAR2_REGEX]
566 break unless candidate
567 txt = candidate.ircify_html
569 txt.sub!(strip, '') if strip
570 debug "(other attempt \#2) #{txt.inspect} has #{txt.count(" ")} spaces"
573 debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
574 return txt unless txt.count(" ") < min_spaces
575 break if min_spaces == 0
580 # This method extracts title, content (first par) and extra
581 # information from the given document _doc_.
583 # _doc_ can be an URI, a Net::HTTPResponse or a String.
585 # If _doc_ is a String, only title and content information
586 # are retrieved (if possible), using standard methods.
588 # If _doc_ is an URI or a Net::HTTPResponse, additional
589 # information is retrieved, and special title/summary
590 # extraction routines are used if possible.
592 def Utils.get_html_info(doc, opts={})
595 Utils.get_string_html_info(doc, opts)
596 when Net::HTTPResponse
597 Utils.get_resp_html_info(doc, opts)
600 @@bot.httputil.get_response(doc) { |resp|
601 ret.replace Utils.get_resp_html_info(resp, opts)
609 class ::UrlLinkError < RuntimeError
612 # This method extracts title, content (first par) and extra
613 # information from the given Net::HTTPResponse _resp_.
615 # Currently, the only accepted options (in _opts_) are
616 # uri_fragment:: the URI fragment of the original request
617 # full_body:: get the whole body instead of
618 # @@bot.config['http.info_bytes'] bytes only
620 # Returns a DataStream with the following keys:
621 # text:: the (partial) body
622 # title:: the title of the document (if any)
623 # content:: the first paragraph of the document (if any)
625 # the headers of the Net::HTTPResponse. The value is
626 # a Hash whose keys are lowercase forms of the HTTP
627 # header fields, and whose values are Arrays.
629 def Utils.get_resp_html_info(resp, opts={})
631 when Net::HTTPSuccess
632 loc = URI.parse(resp['x-rbot-location'] || resp['location']) rescue nil
633 if loc and loc.fragment and not loc.fragment.empty?
634 opts[:uri_fragment] ||= loc.fragment
636 ret = DataStream.new(opts.dup)
637 ret[:headers] = resp.to_hash
638 ret[:text] = partial = opts[:full_body] ? resp.body : resp.partial_body(@@bot.config['http.info_bytes'])
640 filtered = Utils.try_htmlinfo_filters(ret)
644 elsif resp['content-type'] =~ /^text\/|(?:x|ht)ml/
645 ret.merge!(Utils.get_string_html_info(partial, opts))
649 raise UrlLinkError, "getting link (#{resp.code} - #{resp.message})"
653 # This method runs an appropriately-crafted DataStream _ds_ through the
654 # filters in the :htmlinfo filter group, in order. If one of the filters
655 # returns non-nil, its results are merged in _ds_ and returned. Otherwise
658 # The input DataStream should have the downloaded HTML as primary key
659 # (:text) and possibly a :headers key holding the resonse headers.
661 def Utils.try_htmlinfo_filters(ds)
662 filters = @@bot.filter_names(:htmlinfo)
663 return nil if filters.empty?
665 # TODO filter priority
667 debug "testing htmlinfo filter #{n}"
668 cur = @@bot.filter(@@bot.global_filter_name(n, :htmlinfo), ds)
669 debug "returned #{cur.pretty_inspect}"
672 return ds.merge(cur) if cur
675 # HTML info filters often need to check if the webpage location
676 # of a passed DataStream _ds_ matches a given Regexp.
677 def Utils.check_location(ds, rx)
680 loc = [h['x-rbot-location'],h['location']].flatten.grep(rx)
684 return loc.empty? ? nil : loc
687 # This method extracts title and content (first par)
688 # from the given HTML or XML document _text_, using
689 # standard methods (String#ircify_html_title,
690 # Utils.ircify_first_html_par)
692 # Currently, the only accepted option (in _opts_) is
693 # uri_fragment:: the URI fragment of the original request
695 def Utils.get_string_html_info(text, opts={})
696 debug "getting string html info"
698 title = txt.ircify_html_title
700 if frag = opts[:uri_fragment] and not frag.empty?
701 fragreg = /<a\s+(?:[^>]+\s+)?(?:name|id)=["']?#{frag}["']?[^>]*>/im
704 if txt.match(fragreg)
705 # grab the post-match
711 c_opts[:strip] ||= title
712 content = Utils.ircify_first_html_par(txt, c_opts)
713 content = nil if content.empty?
714 return {:title => title, :content => content}
717 # Get the first pars of the first _count_ _urls_.
718 # The pages are downloaded using the bot httputil service.
719 # Returns an array of the first paragraphs fetched.
720 # If (optional) _opts_ :message is specified, those paragraphs are
721 # echoed as replies to the IRC message passed as _opts_ :message
723 def Utils.get_first_pars(urls, count, opts={})
727 while count > 0 and urls.length > 0
732 info = Utils.get_html_info(URI.parse(url), opts)
738 msg.reply "[#{idx}] #{par}", :overlong => :truncate if msg
742 debug "Unable to retrieve #{url}: #{$!}"
749 # Returns a comma separated list except for the last element
750 # which is joined in with specified conjunction
752 def Utils.comma_list(words, options={})
753 defaults = { :join_with => ", ", :join_last_with => _(" and ") }
754 opts = defaults.merge(options)
759 [words[0..-2].join(opts[:join_with]), words.last].join(opts[:join_last_with])
766 Irc::Utils.bot = Irc::Bot::Plugins.manager.bot