@headers = {
'User-Agent' => "rbot http util #{$version} (http://linuxbrit.co.uk/rbot/)",
}
+ @last_response = nil
end
+ attr_reader :last_response
# if http_proxy_include or http_proxy_exclude are set, then examine the
# uri to see if this is a proxied uri
# simple get request, returns (if possible) response body following redirs
# and caching if requested
# if a block is given, it yields the urls it gets redirected to
- def get(uri, readtimeout=10, opentimeout=5, max_redir=@bot.config["http.max_redir"], cache=false)
+ # TODO we really need something to implement proper caching
+ def get(uri_or_str, readtimeout=10, opentimeout=5, max_redir=@bot.config["http.max_redir"], cache=false)
+ if uri_or_str.kind_of?(URI)
+ uri = uri_or_str
+ else
+ uri = URI.parse(uri_or_str.to_s)
+ end
+
proxy = get_proxy(uri)
proxy.open_timeout = opentimeout
proxy.read_timeout = readtimeout
resp = http.get(uri.request_uri(), @headers)
case resp
when Net::HTTPSuccess
- if cache
+ if cache && !(resp.key?('cache-control') && resp['cache-control']=='must-revalidate')
k = uri.to_s
@cache[k] = Hash.new
@cache[k][:body] = resp.body
else
debug "HttpUtil.get return code #{resp.code} #{resp.body}"
end
+ @last_response = resp
return nil
}
rescue StandardError, Timeout::Error => e
error "HttpUtil.get exception: #{e.inspect}, while trying to get #{uri}"
debug e.backtrace.join("\n")
end
+ @last_response = nil
return nil
end
# just like the above, but only gets the head
- def head(uri, readtimeout=10, opentimeout=5, max_redir=@bot.config["http.max_redir"])
+ def head(uri_or_str, readtimeout=10, opentimeout=5, max_redir=@bot.config["http.max_redir"])
+ if uri_or_str.kind_of?(URI)
+ uri = uri_or_str
+ else
+ uri = URI.parse(uri_or_str.to_s)
+ end
+
proxy = get_proxy(uri)
proxy.open_timeout = opentimeout
proxy.read_timeout = readtimeout
begin
proxy.start() {|http|
yield uri.request_uri() if block_given?
- resp = http.head(uri.request_uri(), @headers)
+ resp = http.request_head(uri.request_uri(), @headers)
case resp
when Net::HTTPSuccess
return resp
else
debug "HttpUtil.head return code #{resp.code}"
end
+ @last_response = resp
return nil
}
rescue StandardError, Timeout::Error => e
error "HttpUtil.head exception: #{e.inspect}, while trying to get #{uri}"
debug e.backtrace.join("\n")
end
+ @last_response = nil
return nil
end
# gets a page from the cache if it's still (assumed to be) valid
# TODO remove stale cached pages, except when called with noexpire=true
- def get_cached(uri, readtimeout=10, opentimeout=5,
+ def get_cached(uri_or_str, readtimeout=10, opentimeout=5,
max_redir=@bot.config['http.max_redir'],
noexpire=@bot.config['http.no_expire_cache'])
+ if uri_or_str.kind_of?(URI)
+ uri = uri_or_str
+ else
+ uri = URI.parse(uri_or_str.to_s)
+ end
+
k = uri.to_s
if !@cache.key?(k)
remove_stale_cache unless noexpire
h = head(uri, readtimeout, opentimeout, max_redir)
if h.key?('last-modified')
if Time.httpdate(h['last-modified']) == @cache[k][:last_mod]
- if resp.key?('date')
- @cache[k][:last_use] = Time.httpdate(resp['date'])
+ if h.key?('date')
+ @cache[k][:last_use] = Time.httpdate(h['date'])
else
@cache[k][:last_use] = now
end