def get(uri_or_str, readtimeout=10, opentimeout=5, max_redir=@bot.config["http.max_redir"], cache=false)
if uri_or_str.class <= URI
uri = uri_or_str
else
uri = URI.parse(uri_or_str.to_s)
end
proxy = get_proxy(uri)
proxy.open_timeout = opentimeout
proxy.read_timeout = readtimeout
begin
proxy.start() {|http|
yield uri.request_uri() if block_given?
resp = http.get(uri.request_uri(), @headers)
case resp
when Net::HTTPSuccess
if cache && !(resp.key?('cache-control') && resp['cache-control']=='must-revalidate')
k = uri.to_s
@cache[k] = Hash.new
@cache[k][:body] = resp.body
@cache[k][:last_mod] = Time.httpdate(resp['last-modified']) if resp.key?('last-modified')
if resp.key?('date')
@cache[k][:first_use] = Time.httpdate(resp['date'])
@cache[k][:last_use] = Time.httpdate(resp['date'])
else
now = Time.new
@cache[k][:first_use] = now
@cache[k][:last_use] = now
end
@cache[k][:count] = 1
end
return resp.body
when Net::HTTPRedirection
debug "Redirecting #{uri} to #{resp['location']}"
yield resp['location'] if block_given?
if max_redir > 0
return get( URI.parse(resp['location']), readtimeout, opentimeout, max_redir-1, cache)
else
warning "Max redirection reached, not going to #{resp['location']}"
end
else
debug "HttpUtil.get return code #{resp.code} #{resp.body}"
end
return nil
}
rescue StandardError, Timeout::Error => e
error "HttpUtil.get exception: #{e.inspect}, while trying to get #{uri}"
debug e.backtrace.join("\n")
end
return nil
end