X-Git-Url: https://git.netwichtig.de/gitweb/?a=blobdiff_plain;f=lib%2Frbot%2Fcore%2Futils%2Fhttputil.rb;h=e629becb40b8e3e83e7e28bfc0c7aa3b600aa356;hb=acc946be5b8d9042fb2a9263ba11d5ded2a509bb;hp=39a4781662fe4d03512839413b32e04160875129;hpb=98d6450f694f87f0bacd94dd20eee6cec67f7aaa;p=user%2Fhenk%2Fcode%2Fruby%2Frbot.git

diff --git a/lib/rbot/core/utils/httputil.rb b/lib/rbot/core/utils/httputil.rb
index 39a47816..e629becb 100644
--- a/lib/rbot/core/utils/httputil.rb
+++ b/lib/rbot/core/utils/httputil.rb
@@ -29,6 +29,12 @@ rescue LoadError => e
   EOC
 end
 
+begin
+  require 'nokogiri'
+rescue LoadError => e
+  error "No nokogiri library found, some features might not be available!"
+end
+
 # To handle Gzipped pages
 require 'stringio'
 require 'zlib'
@@ -51,6 +57,8 @@ module ::Net
         debug "charset #{charsets.last} added from header"
       end
 
+      # str might be invalid utf-8 that will crash on the pattern match:
+      str.encode!('UTF-8', 'UTF-8', :invalid => :replace)
       case str
       when /<\?xml\s[^>]*encoding=['"]([^\s"'>]+)["'][^>]*\?>/i
         charsets << $1
@@ -161,6 +169,15 @@ module ::Net
 
       return self.body_to_utf(self.decompress_body(partial))
     end
+
+    def xpath(path)
+      document = Nokogiri::HTML.parse(self.body)
+      document.xpath(path)
+    end
+
+    def to_json
+      JSON::parse(self.body)
+    end
   end
 end
 
@@ -647,7 +664,11 @@ class HttpUtil
       resp = get_response(uri, options, &block)
       raise "http error: #{resp}" unless Net::HTTPOK === resp ||
         Net::HTTPPartialContent === resp
-      return resp.body
+      if options[:resp]
+        return resp
+      else
+        return resp.body
+      end
     rescue Exception => e
       error e
     end