Decouple the enqueuing from the parsing.

3 files changed, 103 insertions, 65 deletions

diff --git a/bin/custodian-enqueue b/bin/custodian-enqueue
index 23df794..c9d9232 100755
--- a/bin/custodian-enqueue
+++ b/bin/custodian-enqueue
@@ -50,9 +50,11 @@
 
 
 #
-#  Implementation of our parser.
+#  Modules we require
 #
+require 'beanstalk-client'
 require 'custodian/parser'
+require 'getoptlong'
 
 
 
@@ -63,11 +65,13 @@ if __FILE__ == $0 then
 
   $help   = false
   $manual = false
+  $SERVER = "127.0.0.1:11300"
 
   begin
     opts = GetoptLong.new(
                           [ "--dump",  "-d", GetoptLong::NO_ARGUMENT ],
                           [ "--test",   GetoptLong::NO_ARGUMENT ],
+                          [ "--server",   GetoptLong::REQUIRED_ARGUMENT ],
                           [ "--file",  "-f", GetoptLong::REQUIRED_ARGUMENT ],
                           [ "--help",  "-h", GetoptLong::NO_ARGUMENT ],
                           [ "--manual","-m", GetoptLong::NO_ARGUMENT ],
@@ -79,6 +83,8 @@ if __FILE__ == $0 then
           ENV["DUMP"] = "1"
       when "--test" then
           ENV["TEST"] = "1"
+      when "--server" then
+          $SERVER = arg
       when "--file" then
           ENV["FILE"] = arg
       when "--timeout" then
@@ -126,6 +132,16 @@ if __FILE__ == $0 then
 
 
   #
+  # Connected to the server
+  #
+  queue = Beanstalk::Pool.new([$SERVER])
+  if ( ! queue )
+    puts "Failed to connect to beanstalk server: #{$SERVER}"
+    exit 1
+  end
+
+
+  #
   # Create the parser
   #
   mon = MonitorConfig.new( ENV['FILE'] )
@@ -138,9 +154,19 @@ if __FILE__ == $0 then
   end
 
   #
-  # Run
+  # Run the parser - and get callbacks when a new job is added.
   #
-  mon.parse_file()
+  mon.parse_file do |test|
+
+    if ( ENV['TEST'] )
+      # nop
+    elsif ( ENV['DUMP'] )
+      puts test.to_json
+    else
+      queue.put( test.to_json )
+    end
+  end
+
 
   #
   # If there were errors then we'll not get this far.
diff --git a/lib/custodian/parser.rb b/lib/custodian/parser.rb
index b7b4279..4e6bfef 100644
--- a/lib/custodian/parser.rb
+++ b/lib/custodian/parser.rb
@@ -1,6 +1,4 @@
 
-require 'beanstalk-client'
-require 'getoptlong'
 require 'json'
 require 'net/http'
 require 'net/https'
@@ -34,11 +32,6 @@ class MonitorConfig
   attr_reader :MACROS
 
   #
-  # A handle to the beanstalkd queue.
-  #
-  attr_reader :queue
-
-  #
   # The filename that we're going to parse.
   #
   attr_reader :filename
@@ -48,6 +41,13 @@ class MonitorConfig
   #
   attr_reader :timeout
 
+  #
+  # An array of job-objects.
+  #
+  #  TODO:  This is just an array of hashes at the moment.
+  #
+  attr_reader :jobs
+
 
 
   #
@@ -57,7 +57,7 @@ class MonitorConfig
 
 
     @MACROS  = Hash.new()
-    @queue   = Beanstalk::Pool.new(['127.0.0.1:11300'])
+    @jobs    = Array.new()
     @file    = filename
     @timeout = 60
 
@@ -466,23 +466,10 @@ class MonitorConfig
           end
         end
 
-
-        #
-        #  Just testing syntax?  At this point we're done
-        #
-        next if ( ENV['TEST'] )
-
         #
-        # We've now parsed the line.  Either output the JSON to the console
-        # or add to the queue.
+        #  Add the job to the results.
         #
-        if ( !ENV['DUMP'].nil? )
-          puts ( test.to_json )
-        else
-          @queue.put( test.to_json )
-        end
-
-        ret.push( test.to_json )
+        ret.push( test )
       end
 
       ret
@@ -497,15 +484,61 @@ class MonitorConfig
   #
   # Parse the configuration file which was named in our constructor.
   #
-  def parse_file()
+  # This updates our @jobs array with the tests - and optionally
+  # invokes our callback.
+  #
+  def parse_file( &callback )
+
     #
     #  Parse the configuration file on the command line
     #
     File.open( @file, "r").each_line do |line|
-      parse_line( line)
+
+      ret = parse_line( line)
+
+      #
+      #  The return value from the parse_line method
+      # is either:
+      #
+      #  Array -> An array of test-objects.
+      #
+      #  nil   -> The line was a macro.
+      #         or
+      #           The line was a comment.
+      #
+      #
+      if ( ret.kind_of?( Array ) )
+        ret.each do |probe|
+          @jobs.push( probe )
+        end
+      end
+
+      #
+      # If there was an optional callback then invoke it
+      # with the newly added job/jobs.
+      #
+      if ( block_given? )
+        if ( ret.kind_of?( Array ) )
+          ret.each do |probe|
+            yield probe
+          end
+        end
+      end
     end
   end
 
 
 end
 
+
+
+if __FILE__ == $0 then
+  p = MonitorConfig.new( "/home/steve/hg/custodian/cfg/steve.cfg" );
+  p.parse_file do |test|
+    puts "Test: #{test} received"
+  end
+
+  p.jobs.each do |job|
+    puts "Job: #{job}"
+  end
+end
diff --git a/t/test-parser.rb b/t/test-parser.rb
index a3b7b43..898c8e4 100755
--- a/t/test-parser.rb
+++ b/t/test-parser.rb
@@ -1,7 +1,6 @@
 #!/usr/bin/ruby1.8 -I./lib/ -I../lib/
 
 
-require 'json'
 require 'test/unit'
 require 'custodian/parser'
 
@@ -362,7 +361,7 @@ class TestParser < Test::Unit::TestCase
     parser = MonitorConfig.new("/dev/null" )
 
     #
-    # Adding a test should return an array - an array of JSON strings.
+    # Adding a test should return an array - an array of hashes.
     #
     ret = parser.parse_line( "example.vm.bytemark must run ssh otherwise 'I hate you'." )
     assert_equal( ret.class.to_s, "Array" )
@@ -381,23 +380,24 @@ class TestParser < Test::Unit::TestCase
     ret = parser.parse_line( "MACRO must run ping otherwise 'ping failure'." )
 
     #
-    # The resulting array should contain three JSON strings.
+    # The resulting array should contain three entries.
     #
     assert_equal( ret.class.to_s, "Array" )
     assert_equal( ret.size(), 3 )
 
     #
-    # Ensure we look like valid JSON, and contains the correct hostnames.
+    # Ensure we look like a valid hash, and contains the correct hostnames.
     #
     ret.each do |test|
-      assert( test =~ /^\{/ )
-      assert( test =~ /(kvm1|kvm2|kvm3)\.vm.bytemark.co.uk/ )
+      assert( test.kind_of?(Hash) )
+      assert( test[:target_host]  =~ /(kvm1|kvm2|kvm3)\.vm.bytemark.co.uk/ )
     end
 
+
     #
     #  Now add more alerts, and ensure we find something sane:
     #
-    #   1.  The addition should be JSON.
+    #   1.  The addition should be arrays of hashes.
     #
     #   2.  The addition should have the correct test-type
     #
@@ -418,16 +418,11 @@ class TestParser < Test::Unit::TestCase
       ret.each do |test|
 
         #
-        #  Look for valid-seeming JSON with a string match
+        #  Look for valid-seeming hash for the test - and the right type
         #
-        assert( test =~ /^\{/ )
-        assert( test =~ /"test_type":"#{name}"/ )
+        assert( test.kind_of?(Hash) )
+        assert( test[:test_type]  = name )
 
-        #
-        #  Deserialize and look for a literal match
-        #
-        hash = JSON.parse( test )
-        assert( hash['test_type'] == name )
       end
     end
   end
@@ -465,7 +460,7 @@ class TestParser < Test::Unit::TestCase
     expected.each do |test,port|
 
       #
-      # Adding a test should return an array - an array of JSON strings.
+      # Adding a test should return an array - an array of hashes.
       #
       ret = parser.parse_line( "example.vm.bytemark must run #{test} otherwise 'fail'." )
       assert_equal( ret.class.to_s, "Array" )
@@ -477,15 +472,9 @@ class TestParser < Test::Unit::TestCase
       addition = ret[0]
 
       #
-      # Look for the correct port in our JSON.
-      #
-      assert( addition =~ /"test_port":#{port}/ )
-
-      #
-      # Deserialize and look for a literal match.
+      # Look for the correct port in our hash.
       #
-      hash = JSON.parse( addition )
-      assert( hash['test_port'] == port )
+      assert( addition[:test_port] == port )
     end
   end
 
@@ -585,26 +574,16 @@ class TestParser < Test::Unit::TestCase
     ret = parser.parse_line( "MACRO must run ping." )
 
     #
-    # The resulting array should contain three JSON strings.
+    # The resulting array should contain three hashes.
     #
     assert_equal( ret.class.to_s, "Array" )
     assert_equal( ret.size(), 3 )
 
     #
-    # Ensure we look like valid JSON, and contains the correct hostnames.
+    # Ensure we have the correct test_alert.
     #
     ret.each do |test|
-      #
-      #  Looks like JSON?
-      #
-      assert( test =~ /^\{/ )
-      assert( test =~ /(kvm1|kvm2|kvm3)\.vm.bytemark.co.uk/ )
-
-      #
-      #  Decode and look for $hostname in the alert text.
-      #
-      hash = JSON.parse( test )
-      assert( hash['test_alert'] =~ /kvm/ )
+      assert( test[:test_alert] =~ /kvm/ )
     end
   end