HttpUtil: fix gunzipping with partial content; and debug response in url plugin earlier

diff --git a/data/rbot/plugins/url.rb b/data/rbot/plugins/url.rb
index 8a05cafa85126d4c5c4ad57c4692b9a4187f2524..10cc419a5a4de2ff73bfce3e0199bd74b7fa6b58 100644 (file)
--- a/data/rbot/plugins/url.rb
+++ b/data/rbot/plugins/url.rb
@@ -52,6 +52,8 @@ class UrlPlugin < Plugin
         case resp
         when Net::HTTPSuccess
 
+          debug resp.to_hash
+
           if resp['content-type'] =~ /^text\/|(?:x|ht)ml/
             # The page is text or HTML, so we can try finding a title and, if
             # requested, the first par.
@@ -76,8 +78,6 @@ class UrlPlugin < Plugin
           # if nothing was found, provide more basic info, as for non-html pages
           end
 
-          debug resp.to_hash.inspect
-
           enc = resp['content-encoding']
 
           extra << ", #{Bold}encoding#{Bold}: #{enc}" if enc

diff --git a/lib/rbot/core/utils/httputil.rb b/lib/rbot/core/utils/httputil.rb
index b4219f66f2b8b348b5a11116d2c827cc7b41c400..3c94968658f75d7b28910bb36487ecbf4c933a0c 100644 (file)
--- a/lib/rbot/core/utils/httputil.rb
+++ b/lib/rbot/core/utils/httputil.rb
@@ -78,7 +78,21 @@ module ::Net
         return str
       when 'gzip', 'x-gzip'
         debug "gunzipping body"
-        return Zlib::GzipReader.new(StringIO.new(str)).read
+        begin
+          return Zlib::GzipReader.new(StringIO.new(str)).read
+        rescue Zlib::Error => e
+          # If we can't unpack the whole stream (e.g. because we're doing a
+          # partial read
+          debug "full gunzipping failed (#{e}), trying to recover as much as possible"
+          ret = ""
+          begin
+            Zlib::GzipReader.new(StringIO.new(str)).each_byte { |byte|
+              ret << byte
+            }
+          rescue
+          end
+          return ret
+        end
       else
         raise "Unhandled content encoding #{method}"
       end