filters: DataStream can now be initialized with a String followed by a Hash

[user/henk/code/ruby/rbot.git] / lib / rbot / core / utils / httputil.rb

diff --git a/lib/rbot/core/utils/httputil.rb b/lib/rbot/core/utils/httputil.rb
index 3a267da121fc9328ef2898516252b6602ff142ce..3d4133e0a80a2b52ebd0878072272d9db94717e6 100644 (file)
--- a/lib/rbot/core/utils/httputil.rb
+++ b/lib/rbot/core/utils/httputil.rb
@@ -39,7 +39,7 @@ module ::Net
 
       charsets = ['latin1'] # should be in config
 
-      if self['content-type'].match(/charset=["']?([^\s"']+)["']?/i)
+      if ctype.match(/charset=["']?([^\s"']+)["']?/i)
         charsets << $1
         debug "charset #{charsets.last} added from header"
       end
@@ -101,6 +101,18 @@ module ::Net
           end
           return ret
         end
+      when 'deflate'
+        debug "inflating body"
+        # From http://www.koders.com/ruby/fid927B4382397E5115AC0ABE21181AB5C1CBDD5C17.aspx?s=thread: 
+        # -MAX_WBITS stops zlib from looking for a zlib header
+        inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+        begin
+          return inflater.inflate(str)
+        rescue Zlib::Error => e
+          raise e
+          # TODO
+          # debug "full inflation failed (#{e}), trying to recover as much as possible"
+        end
       else
         raise "Unhandled content encoding #{method}"
       end
@@ -137,6 +149,10 @@ module Utils
 # this class can check the bot proxy configuration to determine if a proxy
 # needs to be used, which includes support for per-url proxy configuration.
 class HttpUtil
+    Bot::Config.register Bot::Config::IntegerValue.new('http.read_timeout',
+      :default => 10, :desc => "Default read timeout for HTTP connections")
+    Bot::Config.register Bot::Config::IntegerValue.new('http.open_timeout',
+      :default => 20, :desc => "Default open timeout for HTTP connections")
     Bot::Config.register Bot::Config::BooleanValue.new('http.use_proxy',
       :default => false, :desc => "should a proxy be used for HTTP requests?")
     Bot::Config.register Bot::Config::StringValue.new('http.proxy_uri', :default => false,
@@ -265,9 +281,9 @@ class HttpUtil
     @cache = Hash.new
     @headers = {
       'Accept-Charset' => 'utf-8;q=1.0, *;q=0.8',
-      'Accept-Encoding' => 'gzip;q=1, identity;q=0.8, *;q=0.2',
+      'Accept-Encoding' => 'gzip;q=1, deflate;q=1, identity;q=0.8, *;q=0.2',
       'User-Agent' =>
-        "rbot http util #{$version} (http://linuxbrit.co.uk/rbot/)"
+        "rbot http util #{$version} (#{Irc::Bot::SOURCE_URL})"
     }
     debug "starting http cache cleanup timer"
     @timer = @bot.timer.add(300) {
@@ -333,8 +349,8 @@ class HttpUtil
   #
   def get_proxy(uri, options = {})
     opts = {
-      :read_timeout => 10,
-      :open_timeout => 20
+      :read_timeout => @bot.config["http.read_timeout"],
+      :open_timeout => @bot.config["http.open_timeout"]
     }.merge(options)
 
     proxy = nil
@@ -404,6 +420,22 @@ class HttpUtil
       undef_method :body
       alias :body :cooked_body
     end
+    unless resp['content-type']
+      debug "No content type, guessing"
+      resp['content-type'] =
+        case resp['x-rbot-location']
+        when /.html?$/i
+          'text/html'
+        when /.xml$/i
+          'application/xml'
+        when /.xhtml$/i
+          'application/xml+xhtml'
+        when /.(gif|png|jpe?g|jp2|tiff?)$/i
+          "image/#{$1.sub(/^jpg$/,'jpeg').sub(/^tif$/,'tiff')}"
+        else
+          'application/octetstream'
+        end
+    end
     if block_given?
       yield(resp)
     else
@@ -433,6 +465,8 @@ class HttpUtil
   # range::      make a ranged request (usually GET). accepts a string
   #              for HTTP/1.1 "Range:" header (i.e. "bytes=0-1000")
   # body::       request body (usually for POST requests)
+  # headers::    additional headers to be set for the request. Its value must
+  #              be a Hash in the form { 'Header' => 'value' }
   #
   def get_response(uri_or_s, options = {}, &block) # :yields: resp
     uri = uri_or_s.kind_of?(URI) ? uri_or_s : URI.parse(uri_or_s.to_s)
@@ -500,6 +534,7 @@ class HttpUtil
     req.body = opts[:body] if req_class == Net::HTTP::Post
     debug "prepared request: #{req.to_hash.inspect}"
 
+    begin
     get_proxy(uri, opts).start do |http|
       http.request(req) do |resp|
         resp['x-rbot-location'] = uri.to_s
@@ -529,6 +564,10 @@ class HttpUtil
         return handle_response(uri, resp, opts, &block)
       end
     end
+    rescue Exception => e
+      error e
+      raise e.message
+    end
   end
 
   # _uri_::     uri to query (URI object or String)