readme: added build status image

[user/henk/code/ruby/rbot.git] / lib / rbot / core / utils / httputil.rb

diff --git a/lib/rbot/core/utils/httputil.rb b/lib/rbot/core/utils/httputil.rb
index 39a4781662fe4d03512839413b32e04160875129..e629becb40b8e3e83e7e28bfc0c7aa3b600aa356 100644 (file)
--- a/lib/rbot/core/utils/httputil.rb
+++ b/lib/rbot/core/utils/httputil.rb
@@ -29,6 +29,12 @@ rescue LoadError => e
   EOC
 end
 
+begin
+  require 'nokogiri'
+rescue LoadError => e
+  error "No nokogiri library found, some features might not be available!"
+end
+
 # To handle Gzipped pages
 require 'stringio'
 require 'zlib'
@@ -51,6 +57,8 @@ module ::Net
         debug "charset #{charsets.last} added from header"
       end
 
+      # str might be invalid utf-8 that will crash on the pattern match:
+      str.encode!('UTF-8', 'UTF-8', :invalid => :replace)
       case str
       when /<\?xml\s[^>]*encoding=['"]([^\s"'>]+)["'][^>]*\?>/i
         charsets << $1
@@ -161,6 +169,15 @@ module ::Net
 
       return self.body_to_utf(self.decompress_body(partial))
     end
+
+    def xpath(path)
+      document = Nokogiri::HTML.parse(self.body)
+      document.xpath(path)
+    end
+
+    def to_json
+      JSON::parse(self.body)
+    end
   end
 end
 
@@ -647,7 +664,11 @@ class HttpUtil
       resp = get_response(uri, options, &block)
       raise "http error: #{resp}" unless Net::HTTPOK === resp ||
         Net::HTTPPartialContent === resp
-      return resp.body
+      if options[:resp]
+        return resp
+      else
+        return resp.body
+      end
     rescue Exception => e
       error e
     end