+ charsets.reverse_each do |charset|
+ # XXX: this one is really ugly, but i don't know how to make it better
+ # -jsn
+
+ 0.upto(5) do |off|
+ begin
+ debug "trying #{charset} / offset #{off}"
+ return Iconv.iconv('utf-8//ignore',
+ charset,
+ str.slice(0 .. (-1 - off))).first
+ rescue
+ debug "conversion failed for #{charset} / offset #{off}"
+ end
+ end
+ end
+ return str
+ end
+
+ def decompress_body(str)
+ method = self['content-encoding']
+ case method
+ when nil
+ return str
+ when /gzip/ # Matches gzip, x-gzip, and the non-rfc-compliant gzip;q=\d sent by some servers
+ debug "gunzipping body"
+ begin
+ return Zlib::GzipReader.new(StringIO.new(str)).read
+ rescue Zlib::Error => e
+ # If we can't unpack the whole stream (e.g. because we're doing a
+ # partial read
+ debug "full gunzipping failed (#{e}), trying to recover as much as possible"
+ ret = ""
+ begin
+ Zlib::GzipReader.new(StringIO.new(str)).each_byte { |byte|
+ ret << byte
+ }
+ rescue
+ end
+ return ret
+ end
+ when 'deflate'
+ debug "inflating body"
+ # From http://www.koders.com/ruby/fid927B4382397E5115AC0ABE21181AB5C1CBDD5C17.aspx?s=thread:
+ # -MAX_WBITS stops zlib from looking for a zlib header
+ inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+ begin
+ return inflater.inflate(str)
+ rescue Zlib::Error => e
+ raise e
+ # TODO
+ # debug "full inflation failed (#{e}), trying to recover as much as possible"
+ end
+ else
+ raise "Unhandled content encoding #{method}"
+ end
+ end
+
+ def cooked_body
+ return self.body_to_utf(self.decompress_body(self.raw_body))
+ end
+
+ # Read chunks from the body until we have at least _size_ bytes, yielding
+ # the partial text at each chunk. Return the partial body.
+ def partial_body(size=0, &block)
+
+ self.no_cache = true
+ partial = String.new
+
+ self.read_body { |chunk|
+ partial << chunk
+ yield self.body_to_utf(self.decompress_body(partial)) if block_given?
+ break if size and size > 0 and partial.length >= size
+ }
+
+ return self.body_to_utf(self.decompress_body(partial))
+ end
+ end