9 # class for making http requests easier (mainly for plugins to use)
10 # this class can check the bot proxy configuration to determine if a proxy
11 # needs to be used, which includes support for per-url proxy configuration.
13 BotConfig.register BotConfigBooleanValue.new('http.use_proxy',
14 :default => false, :desc => "should a proxy be used for HTTP requests?")
15 BotConfig.register BotConfigStringValue.new('http.proxy_uri', :default => false,
16 :desc => "Proxy server to use for HTTP requests (URI, e.g http://proxy.host:port)")
17 BotConfig.register BotConfigStringValue.new('http.proxy_user',
19 :desc => "User for authenticating with the http proxy (if required)")
20 BotConfig.register BotConfigStringValue.new('http.proxy_pass',
22 :desc => "Password for authenticating with the http proxy (if required)")
23 BotConfig.register BotConfigArrayValue.new('http.proxy_include',
25 :desc => "List of regexps to check against a URI's hostname/ip to see if we should use the proxy to access this URI. All URIs are proxied by default if the proxy is set, so this is only required to re-include URIs that might have been excluded by the exclude list. e.g. exclude /.*\.foo\.com/, include bar\.foo\.com")
26 BotConfig.register BotConfigArrayValue.new('http.proxy_exclude',
28 :desc => "List of regexps to check against a URI's hostname/ip to see if we should use avoid the proxy to access this URI and access it directly")
29 BotConfig.register BotConfigIntegerValue.new('http.max_redir',
31 :desc => "Maximum number of redirections to be used when getting a document")
32 BotConfig.register BotConfigIntegerValue.new('http.expire_time',
34 :desc => "After how many minutes since last use a cached document is considered to be expired")
35 BotConfig.register BotConfigIntegerValue.new('http.max_cache_time',
37 :desc => "After how many minutes since first use a cached document is considered to be expired")
38 BotConfig.register BotConfigIntegerValue.new('http.no_expire_cache',
40 :desc => "Set this to true if you want the bot to never expire the cached pages")
46 'User-Agent' => "rbot http util #{$version} (http://linuxbrit.co.uk/rbot/)",
50 # if http_proxy_include or http_proxy_exclude are set, then examine the
51 # uri to see if this is a proxied uri
52 # the in/excludes are a list of regexps, and each regexp is checked against
53 # the server name, and its IP addresses
54 def proxy_required(uri)
56 if @bot.config["http.proxy_exclude"].empty? && @bot.config["http.proxy_include"].empty?
62 list.concat Resolv.getaddresses(uri.host)
63 rescue StandardError => err
64 warning "couldn't resolve host uri.host"
67 unless @bot.config["http.proxy_exclude"].empty?
68 re = @bot.config["http.proxy_exclude"].collect{|r| Regexp.new(r)}
78 unless @bot.config["http.proxy_include"].empty?
79 re = @bot.config["http.proxy_include"].collect{|r| Regexp.new(r)}
89 debug "using proxy for uri #{uri}?: #{use_proxy}"
93 # uri:: Uri to create a proxy for
95 # return a net/http Proxy object, which is configured correctly for
96 # proxying based on the bot's proxy configuration.
97 # This will include per-url proxy configuration based on the bot config
98 # +http_proxy_include/exclude+ options.
106 if @bot.config["http.use_proxy"]
107 if (ENV['http_proxy'])
108 proxy = URI.parse ENV['http_proxy'] rescue nil
110 if (@bot.config["http.proxy_uri"])
111 proxy = URI.parse @bot.config["http.proxy_uri"] rescue nil
114 debug "proxy is set to #{proxy.host} port #{proxy.port}"
115 if proxy_required(uri)
116 proxy_host = proxy.host
117 proxy_port = proxy.port
118 proxy_user = @bot.config["http.proxy_user"]
119 proxy_pass = @bot.config["http.proxy_pass"]
124 h = Net::HTTP.new(uri.host, uri.port, proxy_host, proxy_port, proxy_user, proxy_port)
125 h.use_ssl = true if uri.scheme == "https"
129 # uri:: uri to query (Uri object)
130 # readtimeout:: timeout for reading the response
131 # opentimeout:: timeout for opening the connection
133 # simple get request, returns (if possible) response body following redirs
134 # and caching if requested
135 # if a block is given, it yields the urls it gets redirected to
136 def get(uri_or_str, readtimeout=10, opentimeout=5, max_redir=@bot.config["http.max_redir"], cache=false)
137 if uri_or_str.class <= URI
140 uri = URI.parse(uri_or_str.to_s)
143 proxy = get_proxy(uri)
144 proxy.open_timeout = opentimeout
145 proxy.read_timeout = readtimeout
148 proxy.start() {|http|
149 yield uri.request_uri() if block_given?
150 resp = http.get(uri.request_uri(), @headers)
152 when Net::HTTPSuccess
156 @cache[k][:body] = resp.body
157 @cache[k][:last_mod] = Time.httpdate(resp['last-modified']) if resp.key?('last-modified')
159 @cache[k][:first_use] = Time.httpdate(resp['date'])
160 @cache[k][:last_use] = Time.httpdate(resp['date'])
163 @cache[k][:first_use] = now
164 @cache[k][:last_use] = now
166 @cache[k][:count] = 1
169 when Net::HTTPRedirection
170 debug "Redirecting #{uri} to #{resp['location']}"
171 yield resp['location'] if block_given?
173 return get( URI.parse(resp['location']), readtimeout, opentimeout, max_redir-1, cache)
175 warning "Max redirection reached, not going to #{resp['location']}"
178 debug "HttpUtil.get return code #{resp.code} #{resp.body}"
182 rescue StandardError, Timeout::Error => e
183 error "HttpUtil.get exception: #{e.inspect}, while trying to get #{uri}"
184 debug e.backtrace.join("\n")
189 # just like the above, but only gets the head
190 def head(uri_or_str, readtimeout=10, opentimeout=5, max_redir=@bot.config["http.max_redir"])
191 if uri_or_str.class <= URI
194 uri = URI.parse(uri_or_str.to_s)
197 proxy = get_proxy(uri)
198 proxy.open_timeout = opentimeout
199 proxy.read_timeout = readtimeout
202 proxy.start() {|http|
203 yield uri.request_uri() if block_given?
204 resp = http.head(uri.request_uri(), @headers)
206 when Net::HTTPSuccess
208 when Net::HTTPRedirection
209 debug "Redirecting #{uri} to #{resp['location']}"
210 yield resp['location'] if block_given?
212 return head( URI.parse(resp['location']), readtimeout, opentimeout, max_redir-1)
214 warning "Max redirection reached, not going to #{resp['location']}"
217 debug "HttpUtil.head return code #{resp.code}"
221 rescue StandardError, Timeout::Error => e
222 error "HttpUtil.head exception: #{e.inspect}, while trying to get #{uri}"
223 debug e.backtrace.join("\n")
228 # gets a page from the cache if it's still (assumed to be) valid
229 # TODO remove stale cached pages, except when called with noexpire=true
230 def get_cached(uri_or_str, readtimeout=10, opentimeout=5,
231 max_redir=@bot.config['http.max_redir'],
232 noexpire=@bot.config['http.no_expire_cache'])
233 if uri_or_str.class <= URI
236 uri = URI.parse(uri_or_str.to_s)
241 remove_stale_cache unless noexpire
242 return get(uri, readtimeout, opentimeout, max_redir, true)
246 # See if the last-modified header can be used
247 # Assumption: the page was not modified if both the header
248 # and the cached copy have the last-modified value, and it's the same time
249 # If only one of the cached copy and the header have the value, or if the
250 # value is different, we assume that the cached copyis invalid and therefore
252 # On our first try, we tested for last-modified in the webpage first,
253 # and then on the local cache. however, this is stupid (in general),
254 # so we only test for the remote page if the local copy had the header
255 # in the first place.
256 if @cache[k].key?(:last_mod)
257 h = head(uri, readtimeout, opentimeout, max_redir)
258 if h.key?('last-modified')
259 if Time.httpdate(h['last-modified']) == @cache[k][:last_mod]
261 @cache[k][:last_use] = Time.httpdate(h['date'])
263 @cache[k][:last_use] = now
265 @cache[k][:count] += 1
266 return @cache[k][:body]
268 remove_stale_cache unless noexpire
269 return get(uri, readtimeout, opentimeout, max_redir, true)
271 remove_stale_cache unless noexpire
272 return get(uri, readtimeout, opentimeout, max_redir, true)
275 warning "Error #{e.inspect} getting the page #{uri}, using cache"
276 debug e.backtrace.join("\n")
277 return @cache[k][:body]
279 # If we still haven't returned, we are dealing with a non-redirected document
280 # that doesn't have the last-modified attribute
281 debug "Could not use last-modified attribute for URL #{uri}, guessing cache validity"
282 if noexpire or !expired?(@cache[k], now)
283 @cache[k][:count] += 1
284 @cache[k][:last_use] = now
286 return @cache[k][:body]
288 debug "Cache expired, getting anew"
290 remove_stale_cache unless noexpire
291 return get(uri, readtimeout, opentimeout, max_redir, true)
294 def expired?(hash, time)
295 (time - hash[:last_use] > @bot.config['http.expire_time']*60) or
296 (time - hash[:first_use] > @bot.config['http.max_cache_time']*60)
299 def remove_stale_cache
301 @cache.reject! { |k, val|
302 !val.key?(:last_modified) && expired?(val, now)