diff options
| -rw-r--r-- | lib/mauve/calendar_interface.rb | 14 | ||||
| -rw-r--r-- | lib/mauve/generic_http_api_client.rb | 170 | ||||
| -rw-r--r-- | lib/mauve/source_list.rb | 64 |
3 files changed, 131 insertions, 117 deletions
diff --git a/lib/mauve/calendar_interface.rb b/lib/mauve/calendar_interface.rb index 5282d79..f67c70b 100644 --- a/lib/mauve/calendar_interface.rb +++ b/lib/mauve/calendar_interface.rb @@ -5,10 +5,18 @@ module Mauve # Interface to the Bytemark calendar. # - class CalendarInterface < GenericHttpApiClient + class CalendarInterface class << self + include GenericHttpApiClient + + # return [Log4r::Logger] + def logger + @logger ||= Log4r::Logger.new(self.to_s) + end + + def get_attendees(klass, at=Time.now) # # Returns nil if no calendar_url has been set. @@ -63,11 +71,11 @@ module Mauve private - def get_yaml(url) + def do_get_yaml(url) resp = do_get(url) return (resp.is_a?(String) ? YAML.load(resp) : nil) - rescue StandardError => err + rescue StandardError => ex logger.error "Caught #{ex.class.to_s} (#{ex.to_s}) whilst querying #{url.to_s}." logger.debug err.backtrace.join("\n") nil diff --git a/lib/mauve/generic_http_api_client.rb b/lib/mauve/generic_http_api_client.rb index 31177e9..0883dce 100644 --- a/lib/mauve/generic_http_api_client.rb +++ b/lib/mauve/generic_http_api_client.rb @@ -9,122 +9,118 @@ module Mauve # # This is a generic client that can talk HTTP to other apps to get data. # - class GenericHttpApiClient - - class << self + module GenericHttpApiClient - # @return [Log4r::Logger] - def logger - @logger ||= Log4r::Logger.new(self.to_s) + # return [Log4r::Logger] + def logger + @logger ||= Log4r::Logger.new(self.to_s) + end + + private + + # Grab a URL from the wide web. + # + # @todo boot this in its own class since list of ips will need it too. + # + # @param [String] uri -- a URL + # @return [String or nil] -- the contents of the URI or nil if an error has been encountered. + # + def do_get (uri, limit = 11) + + if 0 > limit + logger.warn("HTTP redirect too deep for #{uri}.") + return nil end + + begin + uri = URI.parse(uri) unless uri.is_a?(URI::HTTP) - private + raise ArgumentError, "#{uri_str.inspect} doesn't look like an HTTP uri" unless uri.is_a?(URI::HTTP) - # Grab a URL from the wide web. - # - # @todo boot this in its own class since list of ips will need it too. - # - # @param [String] uri -- a URL - # @return [String or nil] -- the contents of the URI or nil if an error has been encountered. - # - def do_get (uri, limit = 11) + http = Net::HTTP.new(uri.host, uri.port) - if 0 > limit - logger.warn("HTTP redirect too deep for #{uri}.") - return nil + # + # Five second timeouts. + # + http.open_timeout = http.read_timeout = Configuration.current.remote_http_timeout || 5 + + if (uri.scheme == "https") + http.use_ssl = true + http.ca_path = "/etc/ssl/certs/" if File.directory?("/etc/ssl/certs") + http.verify_mode = Configuration.current.remote_https_verify_mode || OpenSSL::SSL::VERIFY_NONE end - - begin - uri = URI.parse(uri) unless uri.is_a?(URI::HTTP) - raise ArgumentError, "#{uri_str.inspect} doesn't look like an HTTP uri" unless uri.is_a?(URI::HTTP) + response = http.start { http.get(uri.request_uri()) } - http = Net::HTTP.new(uri.host, uri.port) + if response.is_a?(Net::HTTPOK) + # + # Parse the string as YAML. + # + result = (response.body.is_a?(String) ? response.body : nil) + return result + elsif response.is_a?(Net::HTTPRedirection) and response.key?('Location') + location = response['Location'] + # - # Five second timeouts. + # Bodge locations.. # - http.open_timeout = http.read_timeout = Configuration.current.remote_http_timeout || 5 - - if (uri.scheme == "https") - http.use_ssl = true - http.ca_path = "/etc/ssl/certs/" if File.directory?("/etc/ssl/certs") - http.verify_mode = Configuration.current.remote_https_verify_mode || OpenSSL::SSL::VERIFY_NONE + if location =~ /^\// + location = uri.class.build([uri.userinfo, uri.host, uri.port, nil, nil, nil]).to_s + location end - response = http.start { http.get(uri.request_uri()) } - - if response.is_a?(Net::HTTPOK) - # - # Parse the string as YAML. - # - result = (response.body.is_a?(String) ? response.body : nil) - - return result - elsif response.is_a?(Net::HTTPRedirection) and response.key?('Location') - location = response['Location'] - - # - # Bodge locations.. - # - if location =~ /^\// - location = uri.class.build([uri.userinfo, uri.host, uri.port, nil, nil, nil]).to_s + location - end - - return do_get(location, limit-1) - - else - logger.warn("Request to #{uri.to_s} returned #{response.code} #{response.message}.") - return nil + return do_get(location, limit-1) - end + else + logger.warn("Request to #{uri.to_s} returned #{response.code} #{response.message}.") + return nil - rescue Timeout::Error => ex - logger.error("Timeout caught during fetch of #{uri.to_s}.") + end - rescue StandardError => ex - logger.error("#{ex.class} caught during fetch of #{uri.to_s}: #{ex.to_s}.") - logger.debug(ex.backtrace.join("\n")) + rescue Timeout::Error => ex + logger.error("Timeout caught during fetch of #{uri.to_s}.") - end + rescue StandardError => ex + logger.error("#{ex.class} caught during fetch of #{uri.to_s}: #{ex.to_s}.") + logger.debug(ex.backtrace.join("\n")) - return nil end - # This does HTTP fetches with a 5 minute cache - # - # @param [String] url - # @param [Time] cache_until - # - # @return [String or nil] - def do_get_with_cache(url, cache_until = Time.now + 5.minutes) - @cache ||= {} - - if @cache.has_key?(url) - result, cached_until = @cache[url] + return nil + end - return result if cached_until > Time.now and not result.nil? - end + # This does HTTP fetches with a 5 minute cache + # + # @param [String] url + # @param [Time] cache_until + # + # @return [String or nil] + def do_get_with_cache(url, cache_until = Time.now + 5.minutes) + @cache ||= {} - result = do_get(url) - @cache[url] = [result, cache_until] unless result.nil? + if @cache.has_key?(url) + result, cached_until = @cache[url] - return result + return result if cached_until > Time.now and not result.nil? end - # - # This should get called periodically. - # - def clean_cache + result = do_get(url) + @cache[url] = [result, cache_until] unless result.nil? - @cache.keys.select do |url| - result, cached_until = @cache[url] - @cache.delete(url) if !cached_until.is_a?(Time) or cached_until <= Time.now - end + return result + end + + # + # This should get called periodically. + # + def clean_cache - @cache + @cache.keys.select do |url| + result, cached_until = @cache[url] + @cache.delete(url) if !cached_until.is_a?(Time) or cached_until <= Time.now end + @cache end end diff --git a/lib/mauve/source_list.rb b/lib/mauve/source_list.rb index 8248758..0596177 100644 --- a/lib/mauve/source_list.rb +++ b/lib/mauve/source_list.rb @@ -18,7 +18,9 @@ module Mauve # addresses and only one of tbhose is included in the list, a match # will occur. # - class SourceList + class SourceList + + include GenericHttpApiClient attr_reader :label, :last_resolved_at @@ -55,28 +57,8 @@ module Mauve # def +(l) arr = [l].flatten.collect do |h| - # "*" means [^\.]+ - # "(\d+)\.\.(\d+)" is expanded to every integer between $1 and $2 - # joined by a pipe, e.g. 1..5 means 1|2|3|4|5 - # "." is literal, not a single-character match - if h.is_a?(String) and (h =~ /[\[\]\*]/ or h =~ /(\d+)\.\.(\d+)/) - Regexp.new( - h. - gsub(/(\d+)\.\.(\d+)/) { |a,b| - ($1.to_i..$2.to_i).collect.join("|") - }. - gsub(/\./, "\\."). - gsub(/\*/, "[0-9a-z\\-]+") + - "\\.?$") - elsif h.is_a?(String) and h =~ /^[0-9a-f\.:]+(\/\d+)?$/i - IPAddr.new(h) - elsif h.is_a?(String) or h.is_a?(Regexp) - h - else - logger.warn "Cannot add #{h.inspect} to source list #{@label} as it is not a string or regular expression." - nil - end - end.flatten.reject{|h| h.nil?} + do_parse_source(h) + end.flatten.compact arr.each do |source| ## @@ -182,13 +164,13 @@ module Mauve url_list = [] if @url - url_list_s = GenericHttpApiClient.do_get(@url) + url_list_s = do_get(@url) if url_list_s.is_a?(String) - url_list = url_list_s.split("\n").reject{|s| s.empty?} + url_list = url_list_s.split("\n").collect{|s| do_parse_source(s)}.flatten.compact end end - new_list = [url_list + @list].collect do |host| + new_list = (url_list + @list).collect do |host| if host.is_a?(String) [host] + MauveResolv.get_ips_for(host).collect{|i| IPAddr.new(i)} else @@ -196,10 +178,38 @@ module Mauve end end - @resolved_list = new_list.flatten end + private + + def do_parse_source(h) + # "*" means [^\.]+ + # "(\d+)\.\.(\d+)" is expanded to every integer between $1 and $2 + # joined by a pipe, e.g. 1..5 means 1|2|3|4|5 + # "." is literal, not a single-character match + if h.is_a?(String) and (h =~ /[\[\]\*]/ or h =~ /(\d+)\.\.(\d+)/) + Regexp.new( + h. + gsub(/(\d+)\.\.(\d+)/) { |a,b| + ($1.to_i..$2.to_i).collect.join("|") + }. + gsub(/\./, "\\."). + gsub(/\*/, "[0-9a-z\\-]+") + + "\\.?$") + elsif h.is_a?(String) and h =~ /^[0-9a-f\.:]+(\/\d+)?$/i + IPAddr.new(h) + elsif h.is_a?(String) and h =~ /^\/(.*)\/$/ + Regexp.new($1) + elsif h.is_a?(String) or h.is_a?(Regexp) + h + else + logger.warn "Cannot parse source line #{h.inspect} for source list #{@label}." + nil + end + + end + end end |