Stringlib/rbot/core/utils/extends.rbutfy_xml() method that tries to transcode a webpa...

diff --git a/lib/rbot/core/utils/extends.rb b/lib/rbot/core/utils/extends.rb
index 0ecf7aa2bde2a5d352aa340701b03ab30c7fac88..95569b71ff261b6e1bbfbb7e28273bab9eb8578c 100644 (file)
--- a/lib/rbot/core/utils/extends.rb
+++ b/lib/rbot/core/utils/extends.rb
@@ -27,6 +27,12 @@ class ::Array
   end
 end
 
+begin
+  require 'iconv'
+  $we_have_iconv = true
+rescue LoadError
+  $we_have_iconv = false
+end
 
 # Extensions to the String class
 #
@@ -35,6 +41,43 @@ end
 #
 class ::String
 
+  # This method will try to transcode a String supposed to hold an XML or HTML
+  # document from the original charset to UTF-8.
+  #
+  # To find the original encoding, it will first see if the String responds to
+  # #http_headers(), and if it does it will assume that the charset indicated
+  # there is the correct one. Otherwise, it will try to detect the charset from
+  # some typical XML and HTML headers
+  def utfy_xml
+    return self unless $we_have_iconv
+
+    charset = nil
+
+    if self.respond_to?(:http_headers) and headers = self.http_headers
+      if headers['content-type'].first.match(/charset="?(\S+?)"?\s*;?/i)
+        charset = $1
+      end
+    end
+
+    if not charset
+      case self
+      when /<\?xml.*encoding="(\S+)".*\?>/i
+        charset = $1
+      when /<meta\s+http-equiv\s*=\s*"Content-Type".*charset\s*=\s*"?(\S+?)"?\s*;?/i
+        charset = $1
+      end
+    end
+
+    if charset
+      debug "charset: #{charset}"
+      return Iconv.iconv('utf-8', charset, self).join rescue self
+    else
+      debug "Couldn't find charset for #{self.inspect}"
+      return self
+    end
+
+  end
+
   # This method will return a purified version of the receiver, with all HTML
   # stripped off and some of it converted to IRC formatting
   #

diff --git a/lib/rbot/core/utils/httputil.rb b/lib/rbot/core/utils/httputil.rb
index 78445abebcebdc39d99d5601612cc153a044ac1a..78ea9063e35ec6fbc22797ffea1ac5a3968bd607 100644 (file)
--- a/lib/rbot/core/utils/httputil.rb
+++ b/lib/rbot/core/utils/httputil.rb
@@ -301,6 +301,22 @@ class HttpUtil
       resp.body
     end
 
+    class << resp.body
+      def http_headers
+        if defined?(@http_headers)
+          @http_headers
+        else
+          nil
+        end
+      end
+
+      def http_headers=(rsp)
+        @http_headers=rsp
+      end
+    end
+
+    resp.body.http_headers = resp.to_hash
+
     return resp
   end
 

diff --git a/lib/rbot/core/utils/utils.rb b/lib/rbot/core/utils/utils.rb
index 717630e30ec7f0dc5d46af366520301833de27e3..63cd58da62cbc079c767378eeb8b486541e488c1 100644 (file)
--- a/lib/rbot/core/utils/utils.rb
+++ b/lib/rbot/core/utils/utils.rb
@@ -433,7 +433,7 @@ module ::Irc
     #   * :min_spaces => Minimum number of spaces a paragraph should have
     #
     def Utils.ircify_first_html_par(xml_org, opts={})
-      xml = xml_org.gsub(/<!--.*?-->/, '')
+      xml = xml_org.gsub(/<!--.*?-->/, '').utfy_xml
 
       strip = opts[:strip]
       strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)