Moved http api requests into separate module. Added ability to poll a URL for

lists of IPs.

3 files changed, 131 insertions, 117 deletions

diff --git a/lib/mauve/calendar_interface.rb b/lib/mauve/calendar_interface.rb
index 5282d79..f67c70b 100644
--- a/lib/mauve/calendar_interface.rb
+++ b/lib/mauve/calendar_interface.rb
@@ -5,10 +5,18 @@ module Mauve
 
   # Interface to the Bytemark calendar.
   #
-  class CalendarInterface  < GenericHttpApiClient
+  class CalendarInterface  
     
     class << self
 
+      include GenericHttpApiClient
+
+      # return [Log4r::Logger]
+      def logger
+        @logger ||= Log4r::Logger.new(self.to_s)
+      end
+    
+
       def get_attendees(klass, at=Time.now)
         #
         # Returns nil if no calendar_url has been set.
@@ -63,11 +71,11 @@ module Mauve
 
       private
 
-      def get_yaml(url)
+      def do_get_yaml(url)
         resp = do_get(url)
 
         return (resp.is_a?(String) ? YAML.load(resp) : nil)
-      rescue StandardError => err
+      rescue StandardError => ex
         logger.error "Caught #{ex.class.to_s} (#{ex.to_s}) whilst querying #{url.to_s}."
         logger.debug err.backtrace.join("\n")
         nil
diff --git a/lib/mauve/generic_http_api_client.rb b/lib/mauve/generic_http_api_client.rb
index 31177e9..0883dce 100644
--- a/lib/mauve/generic_http_api_client.rb
+++ b/lib/mauve/generic_http_api_client.rb
@@ -9,122 +9,118 @@ module Mauve
   #
   # This is a generic client that can talk HTTP to other apps to get data.
   #
-  class GenericHttpApiClient
-    
-    class << self
+  module GenericHttpApiClient
 
-      # @return [Log4r::Logger]
-      def logger
-        @logger ||= Log4r::Logger.new(self.to_s)
+    # return [Log4r::Logger]
+    def logger
+      @logger ||= Log4r::Logger.new(self.to_s)
+    end
+    
+    private
+
+    # Grab a URL from the wide web.
+    #
+    # @todo boot this in its own class since list of ips will need it too.
+    #
+    # @param [String] uri -- a URL
+    # @return [String or nil] -- the contents of the URI or nil if an error has been encountered.
+    #
+    def do_get (uri, limit = 11)
+
+      if 0 > limit
+        logger.warn("HTTP redirect too deep for #{uri}.")
+        return nil
       end
+      
+      begin
+        uri = URI.parse(uri) unless uri.is_a?(URI::HTTP)
 
-      private
+        raise ArgumentError, "#{uri_str.inspect} doesn't look like an HTTP uri" unless uri.is_a?(URI::HTTP)
 
-      # Grab a URL from the wide web.
-      #
-      # @todo boot this in its own class since list of ips will need it too.
-      #
-      # @param [String] uri -- a URL
-      # @return [String or nil] -- the contents of the URI or nil if an error has been encountered.
-      #
-      def do_get (uri, limit = 11)
+        http = Net::HTTP.new(uri.host, uri.port)
 
-        if 0 > limit
-          logger.warn("HTTP redirect too deep for #{uri}.")
-          return nil
+        #
+        # Five second timeouts.
+        #
+        http.open_timeout = http.read_timeout = Configuration.current.remote_http_timeout || 5
+
+        if (uri.scheme == "https")
+          http.use_ssl     = true
+          http.ca_path     = "/etc/ssl/certs/" if File.directory?("/etc/ssl/certs")
+          http.verify_mode = Configuration.current.remote_https_verify_mode || OpenSSL::SSL::VERIFY_NONE
         end
-        
-        begin
-          uri = URI.parse(uri) unless uri.is_a?(URI::HTTP)
 
-          raise ArgumentError, "#{uri_str.inspect} doesn't look like an HTTP uri" unless uri.is_a?(URI::HTTP)
+        response = http.start { http.get(uri.request_uri()) }
 
-          http = Net::HTTP.new(uri.host, uri.port)
+        if response.is_a?(Net::HTTPOK)
+          #
+          # Parse the string as YAML.
+          #
+          result = (response.body.is_a?(String) ? response.body : nil)
 
+          return result
+        elsif response.is_a?(Net::HTTPRedirection) and response.key?('Location')
+          location = response['Location']
+          
           #
-          # Five second timeouts.
+          # Bodge locations..
           #
-          http.open_timeout = http.read_timeout = Configuration.current.remote_http_timeout || 5
- 
-          if (uri.scheme == "https")
-            http.use_ssl     = true
-            http.ca_path     = "/etc/ssl/certs/" if File.directory?("/etc/ssl/certs")
-            http.verify_mode = Configuration.current.remote_https_verify_mode || OpenSSL::SSL::VERIFY_NONE
+          if location =~ /^\//
+            location = uri.class.build([uri.userinfo, uri.host, uri.port, nil, nil, nil]).to_s + location
           end
 
-          response = http.start { http.get(uri.request_uri()) }
-
-          if response.is_a?(Net::HTTPOK)
-            #
-            # Parse the string as YAML.
-            #
-            result = (response.body.is_a?(String) ? response.body : nil)
-
-            return result
-          elsif response.is_a?(Net::HTTPRedirection) and response.key?('Location')
-            location = response['Location']
-            
-            #
-            # Bodge locations..
-            #
-            if location =~ /^\//
-              location = uri.class.build([uri.userinfo, uri.host, uri.port, nil, nil, nil]).to_s + location
-            end
-
-            return do_get(location, limit-1) 
-
-          else
-            logger.warn("Request to #{uri.to_s} returned #{response.code} #{response.message}.")
-            return nil
+          return do_get(location, limit-1) 
 
-          end
+        else
+          logger.warn("Request to #{uri.to_s} returned #{response.code} #{response.message}.")
+          return nil
 
-        rescue Timeout::Error => ex
-          logger.error("Timeout caught during fetch of #{uri.to_s}.")
+        end
 
-        rescue StandardError => ex
-          logger.error("#{ex.class} caught during fetch of #{uri.to_s}: #{ex.to_s}.")
-          logger.debug(ex.backtrace.join("\n"))
+      rescue Timeout::Error => ex
+        logger.error("Timeout caught during fetch of #{uri.to_s}.")
 
-        end
+      rescue StandardError => ex
+        logger.error("#{ex.class} caught during fetch of #{uri.to_s}: #{ex.to_s}.")
+        logger.debug(ex.backtrace.join("\n"))
 
-        return nil
       end
 
-      # This does HTTP fetches with a 5 minute cache
-      #
-      # @param [String] url
-      # @param [Time] cache_until
-      #
-      # @return [String or nil]
-      def do_get_with_cache(url, cache_until = Time.now + 5.minutes)
-        @cache ||= {}
-
-        if @cache.has_key?(url)
-          result, cached_until = @cache[url]
+      return nil
+    end
 
-          return result if cached_until > Time.now and not result.nil?
-        end
+    # This does HTTP fetches with a 5 minute cache
+    #
+    # @param [String] url
+    # @param [Time] cache_until
+    #
+    # @return [String or nil]
+    def do_get_with_cache(url, cache_until = Time.now + 5.minutes)
+      @cache ||= {}
 
-        result = do_get(url)
-        @cache[url] = [result, cache_until] unless result.nil?
+      if @cache.has_key?(url)
+        result, cached_until = @cache[url]
 
-        return result
+        return result if cached_until > Time.now and not result.nil?
       end
 
-      #
-      # This should get called periodically.
-      #
-      def clean_cache
+      result = do_get(url)
+      @cache[url] = [result, cache_until] unless result.nil?
 
-        @cache.keys.select do |url|
-          result, cached_until = @cache[url]
-          @cache.delete(url) if !cached_until.is_a?(Time) or cached_until <= Time.now
-        end
+      return result
+    end
+
+    #
+    # This should get called periodically.
+    #
+    def clean_cache
 
-        @cache 
+      @cache.keys.select do |url|
+        result, cached_until = @cache[url]
+        @cache.delete(url) if !cached_until.is_a?(Time) or cached_until <= Time.now
       end
 
+      @cache 
     end
 
   end
diff --git a/lib/mauve/source_list.rb b/lib/mauve/source_list.rb
index 8248758..0596177 100644
--- a/lib/mauve/source_list.rb
+++ b/lib/mauve/source_list.rb
@@ -18,7 +18,9 @@ module Mauve
   # addresses and only one of tbhose is included in the list, a match 
   # will occur.
   #
-  class SourceList 
+  class SourceList
+
+    include GenericHttpApiClient
 
     attr_reader :label, :last_resolved_at
 
@@ -55,28 +57,8 @@ module Mauve
     #
     def +(l)
       arr = [l].flatten.collect do |h|
-        # "*"              means [^\.]+
-        # "(\d+)\.\.(\d+)" is expanded to every integer between $1 and $2
-        #                  joined by a pipe, e.g. 1..5 means 1|2|3|4|5
-        #  "."              is literal, not a single-character match
-        if h.is_a?(String) and (h =~ /[\[\]\*]/ or h =~ /(\d+)\.\.(\d+)/)
-          Regexp.new(
-              h.
-              gsub(/(\d+)\.\.(\d+)/) { |a,b|
-                ($1.to_i..$2.to_i).collect.join("|")
-              }.
-              gsub(/\./, "\\.").
-              gsub(/\*/, "[0-9a-z\\-]+") +
-              "\\.?$")
-        elsif h.is_a?(String) and h =~ /^[0-9a-f\.:]+(\/\d+)?$/i
-          IPAddr.new(h)
-        elsif h.is_a?(String) or h.is_a?(Regexp)
-          h
-        else
-          logger.warn "Cannot add #{h.inspect} to source list #{@label} as it is not a string or regular expression."
-          nil
-        end
-      end.flatten.reject{|h| h.nil?}
+        do_parse_source(h)
+      end.flatten.compact
 
       arr.each do |source|
         ##
@@ -182,13 +164,13 @@ module Mauve
       
       url_list = []
       if @url
-        url_list_s = GenericHttpApiClient.do_get(@url)
+        url_list_s = do_get(@url)
         if url_list_s.is_a?(String)
-          url_list = url_list_s.split("\n").reject{|s| s.empty?}
+          url_list = url_list_s.split("\n").collect{|s| do_parse_source(s)}.flatten.compact
         end
       end
 
-      new_list = [url_list + @list].collect do |host| 
+      new_list = (url_list + @list).collect do |host| 
         if host.is_a?(String)
           [host] + MauveResolv.get_ips_for(host).collect{|i| IPAddr.new(i)}
         else
@@ -196,10 +178,38 @@ module Mauve
         end
       end
 
-
       @resolved_list = new_list.flatten
     end
 
+    private
+
+    def do_parse_source(h)
+      # "*"              means [^\.]+
+      # "(\d+)\.\.(\d+)" is expanded to every integer between $1 and $2
+      #                  joined by a pipe, e.g. 1..5 means 1|2|3|4|5
+      #  "."              is literal, not a single-character match
+      if h.is_a?(String) and (h =~ /[\[\]\*]/ or h =~ /(\d+)\.\.(\d+)/)
+        Regexp.new(
+            h.
+            gsub(/(\d+)\.\.(\d+)/) { |a,b|
+              ($1.to_i..$2.to_i).collect.join("|")
+            }.
+            gsub(/\./, "\\.").
+            gsub(/\*/, "[0-9a-z\\-]+") +
+            "\\.?$")
+      elsif h.is_a?(String) and h =~ /^[0-9a-f\.:]+(\/\d+)?$/i
+        IPAddr.new(h)
+      elsif h.is_a?(String) and h =~ /^\/(.*)\/$/
+        Regexp.new($1)
+      elsif h.is_a?(String) or h.is_a?(Regexp)
+        h
+      else
+        logger.warn "Cannot parse source line #{h.inspect} for source list #{@label}."
+        nil
+      end
+
+    end
+
   end
 
 end