1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
# encoding: UTF-8
require 'log4r'
require 'net/http'
require 'net/https'
require 'uri'
module Mauve
#
# This is a generic client that can talk HTTP to other apps to get data.
#
class GenericHttpApiClient
class << self
# @return [Log4r::Logger]
def logger
@logger ||= Log4r::Logger.new(self.to_s)
end
private
# Grab a URL from the wide web.
#
# @todo boot this in its own class since list of ips will need it too.
#
# @param [String] uri -- a URL
# @return [String or nil] -- the contents of the URI or nil if an error has been encountered.
#
def do_get (uri, limit = 11)
if 0 > limit
logger.warn("HTTP redirect too deep for #{uri}.")
return nil
end
begin
uri = URI.parse(uri) unless uri.is_a?(URI::HTTP)
raise ArgumentError, "#{uri_str.inspect} doesn't look like an HTTP uri" unless uri.is_a?(URI::HTTP)
http = Net::HTTP.new(uri.host, uri.port)
#
# Five second timeouts.
#
http.open_timeout = http.read_timeout = Configuration.current.remote_http_timeout || 5
if (uri.scheme == "https")
http.use_ssl = true
http.ca_path = "/etc/ssl/certs/" if File.directory?("/etc/ssl/certs")
http.verify_mode = Configuration.current.remote_https_verify_mode || OpenSSL::SSL::VERIFY_NONE
end
response = http.start { http.get(uri.request_uri()) }
if response.is_a?(Net::HTTPOK)
#
# Parse the string as YAML.
#
result = (response.body.is_a?(String) ? response.body : nil)
return result
elsif response.is_a?(Net::HTTPRedirection) and response.key?('Location')
location = response['Location']
#
# Bodge locations..
#
if location =~ /^\//
location = uri.class.build([uri.userinfo, uri.host, uri.port, nil, nil, nil]).to_s + location
end
return do_get(location, limit-1)
else
logger.warn("Request to #{uri.to_s} returned #{response.code} #{response.message}.")
return nil
end
rescue Timeout::Error => ex
logger.error("Timeout caught during fetch of #{uri.to_s}.")
rescue StandardError => ex
logger.error("#{ex.class} caught during fetch of #{uri.to_s}: #{ex.to_s}.")
logger.debug(ex.backtrace.join("\n"))
end
return nil
end
# This does HTTP fetches with a 5 minute cache
#
# @param [String] url
# @param [Time] cache_until
#
# @return [String or nil]
def do_get_with_cache(url, cache_until = Time.now + 5.minutes)
@cache ||= {}
if @cache.has_key?(url)
result, cached_until = @cache[url]
return result if cached_until > Time.now and not result.nil?
end
result = do_get(url)
@cache[url] = [result, cache_until] unless result.nil?
return result
end
#
# This should get called periodically.
#
def clean_cache
@cache.keys.select do |url|
result, cached_until = @cache[url]
@cache.delete(url) if !cached_until.is_a?(Time) or cached_until <= Time.now
end
@cache
end
end
end
end
|
