X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=lib%2Frbot%2Fcore%2Futils%2Fhttputil.rb;h=fb275547169efefce78632b9eb62d3efc0daf39c;hb=391c2ce19c0954d3ea0f7b6cc17c001d26491ca6;hp=5c7db444583553993ade60afbf88d4db2be8ccc4;hpb=4875fd2668ba6393edeea2d85b14a7e2f3a18050;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git

diff --git a/lib/rbot/core/utils/httputil.rb b/lib/rbot/core/utils/httputil.rb
index 5c7db444..fb275547 100644
--- a/lib/rbot/core/utils/httputil.rb
+++ b/lib/rbot/core/utils/httputil.rb
@@ -1,3 +1,4 @@
+# encoding: UTF-8
 #-- vim:sw=2:et
 #++
 #
@@ -10,12 +11,11 @@
 require 'resolv'
 require 'net/http'
 require 'cgi'
-require 'iconv'
+
 begin
-  require 'net/https'
+  require 'nokogiri'
 rescue LoadError => e
-  error "Couldn't load 'net/https':  #{e.pretty_inspect}"
-  error "Secured HTTP connections will fail"
+  error "No nokogiri library found, some features might not be available!"
 end
 
 # To handle Gzipped pages
@@ -33,13 +33,15 @@ module ::Net
       ctype = self['content-type'] || 'text/html'
       return nil unless ctype =~ /^text/i || ctype =~ /x(ht)?ml/i
 
-      charsets = ['latin1'] # should be in config
+      charsets = ['ISO-8859-1'] # should be in config
 
       if ctype.match(/charset=["']?([^\s"']+)["']?/i)
         charsets << $1
         debug "charset #{charsets.last} added from header"
       end
 
+      # str might be invalid utf-8 that will crash on the pattern match:
+      str.encode!('UTF-8', 'UTF-8', :invalid => :replace)
       case str
       when /<\?xml\s[^>]*encoding=['"]([^\s"'>]+)["'][^>]*\?>/i
         charsets << $1
@@ -58,20 +60,20 @@ module ::Net
       charsets = self.body_charset(str) or return str
 
       charsets.reverse_each do |charset|
-        # XXX: this one is really ugly, but i don't know how to make it better
-        #  -jsn
-
-        0.upto(5) do |off|
-          begin
-            debug "trying #{charset} / offset #{off}"
-            return Iconv.iconv('utf-8//ignore',
-                               charset,
-                               str.slice(0 .. (-1 - off))).first
-          rescue
-            debug "conversion failed for #{charset} / offset #{off}"
+        begin
+          debug "try decoding using #{charset}"
+          str.force_encoding(charset)
+          tmp = str.encode('UTF-16le', :invalid => :replace, :replace => '').encode('UTF-8')
+          if tmp
+            str = tmp
+            break
           end
+        rescue
+          error 'failed to use encoding'
+          error $!
         end
       end
+
       return str
     end
 
@@ -88,7 +90,8 @@ module ::Net
           # If we can't unpack the whole stream (e.g. because we're doing a
           # partial read
           debug "full gunzipping failed (#{e}), trying to recover as much as possible"
-          ret = ""
+          ret = ''
+          ret.force_encoding(Encoding::ASCII_8BIT)
           begin
             Zlib::GzipReader.new(StringIO.new(str)).each_byte { |byte|
               ret << byte
@@ -112,7 +115,7 @@ module ::Net
       when /^(?:iso-8859-\d+|windows-\d+|utf-8|utf8)$/i
         # B0rked servers (Freshmeat being one of them) sometimes return the charset
         # in the content-encoding; in this case we assume that the document has
-        # a standarc content-encoding
+        # a standard content-encoding
         old_hsh = self.to_hash
         self['content-type']= self['content-type']+"; charset="+method.downcase
         warning "Charset vs content-encoding confusion, trying to recover: from\n#{old_hsh.pretty_inspect}to\n#{self.to_hash.pretty_inspect}"
@@ -149,11 +152,18 @@ module ::Net
 
       return self.body_to_utf(self.decompress_body(partial))
     end
+
+    def xpath(path)
+      document = Nokogiri::HTML.parse(self.body)
+      document.xpath(path)
+    end
+
+    def to_json
+      JSON::parse(self.body)
+    end
   end
 end
 
-Net::HTTP.version_1_2
-
 module ::Irc
 module Utils
 
@@ -438,9 +448,23 @@ class HttpUtil
           new_opts[:method] = :get
         end
         if resp['set-cookie']
-          debug "setting cookie #{resp['set-cookie']}"
-          new_opts[:headers] ||= Hash.new
-          new_opts[:headers]['Cookie'] = resp['set-cookie']
+          debug "set cookie request for #{resp['set-cookie']}"
+          cookie, cookie_flags = (resp['set-cookie']+'; ').split('; ', 2)
+          domain = uri.host
+          cookie_flags.scan(/(\S+)=(\S+);/) { |key, val|
+            if key.intern == :domain
+              domain = val
+              break
+            end
+          }
+          debug "cookie domain #{domain} / #{new_loc.host}"
+          if new_loc.host.rindex(domain) == new_loc.host.length - domain.length
+            debug "setting cookie"
+            new_opts[:headers] ||= Hash.new
+            new_opts[:headers]['Cookie'] = cookie
+          else
+            debug "cookie is for another domain, ignoring"
+          end
         end
         debug "following the redirect to #{new_loc}"
         return get_response(new_loc, new_opts, &block)
@@ -517,8 +541,6 @@ class HttpUtil
       :method => :GET
     }.merge(options)
 
-    resp = nil
-
     req_class = case opts[:method].to_s.downcase.intern
                 when :head, :"net::http::head"
                   opts[:max_redir] = -1
@@ -623,7 +645,11 @@ class HttpUtil
       resp = get_response(uri, options, &block)
       raise "http error: #{resp}" unless Net::HTTPOK === resp ||
         Net::HTTPPartialContent === resp
-      return resp.body
+      if options[:resp]
+        return resp
+      else
+        return resp.body
+      end
     rescue Exception => e
       error e
     end