Updated the redis-alerter to store more useful-state.

This will make visualization more simple.

2 files changed, 114 insertions, 23 deletions

diff --git a/debian/changelog b/debian/changelog
index cc2aca7..6a1ba7d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+custodian (0.24.13) stable; urgency=low
+
+  * Updated the redis-alerter, to store more useful-state.
+
+ -- Steve Kemp <steve@bytemark.co.uk>  Tue, 16 Nov 2015 15:00:51 +0000
+
 custodian (0.24.12) stable; urgency=low
 
   * Explicitly force the parser to use UTF-8 when reading the configuration
diff --git a/lib/custodian/alerts/redis-state.rb b/lib/custodian/alerts/redis-state.rb
index 9c5903c..3abf2ad 100644
--- a/lib/custodian/alerts/redis-state.rb
+++ b/lib/custodian/alerts/redis-state.rb
@@ -1,8 +1,37 @@
 #
 #  The redis-alerter.
 #
-#  This doesn't raise/clear alerts in the traditional sense, instead
-# it just saves the results in a "recent tests" set inside Redis.
+# This alerter doesn't raise/clear alerts in the traditional sense,
+# instead it stores the state of the tests in a Redis store.
+#
+# We maintain several structures which are updated by raise/clear
+# messages.  We keep track of all the recent tests in the set
+# `known_tests`.
+#
+# `known_tests` contains an array of the tests which have been
+#  carried out.  For example:
+#
+#       [ "foo.vm must run ping ..",
+#         "bar.vm must run ssh .." ]
+#
+#  Then for each test we keep track of the state-transitions,
+# and do so based upon the SHA1hash of the test-string.
+#
+#  Assume we have the following test:
+#
+#     "http://google.com must run http with status 200"
+#
+#  This is hashed to :
+#
+#    71cf1735cd389732877177a757c45fdb5407f673
+#
+#  We then keep the single current-state in the key:
+#
+#    71cf1735cd389732877177a757c45fdb5407f673.current = pass|fail|unknown
+#
+#  We build up a history when the state-changes via members of the set
+#
+#    71cf1735cd389732877177a757c45fdb5407f673.history = [ ]
 #
 #
 module Custodian
@@ -54,17 +83,46 @@ module Custodian
 
         return unless @redis
 
-        tmp = {}
-        tmp['time']   = Time.now.to_i
-        tmp['type']   = @test.get_type
-        tmp['target'] = @test.target
-        tmp['result'] = 'RAISE'
-        tmp['reason'] = @test.error
-        tmp['test'] = @test.to_s
-        tmp['class'] = @test.class
-
-        @redis.lpush('recent-tests', tmp.to_json)
-        @redis.ltrim('recent-tests', 0, 100)
+        #
+        #  Make sure we know about this test.
+        #
+        test_s = @test.to_s
+
+        @redis.sadd( "known_tests", test_s )
+
+        #
+        #  Get the current state of this test - so that if the state
+        # has changed we can add that to our history.
+        #
+        #  We use SHA1hash to determine our key.
+        #
+        key = Digest::SHA1.hexdigest test_s
+
+        #
+        #  The current state
+        #
+        current = @redis.get( "#{key}.current" ) || "unknown"
+        @redis.set( "#{key}.current", "FAIL" )
+
+        count = @redis.get( "#{key}.count" ) || "0"
+        @redis.set( "#{key}.count", (count.to_i + 1))
+
+        #
+        #  Bump the execution count for this test.
+        #
+        if ( current != "FAIL" )
+
+          #
+          #  The state has changed to raise.
+          #
+          tmp = {}
+          tmp['time']   = Time.now.to_i
+          tmp['result'] = 'FAIL'
+          tmp['reason'] = @test.error
+
+          @redis.lpush( "#{key}.history", tmp.to_json)
+          @redis.ltrim('#{key}.history', 0, 100)
+        end
 
       end
 
@@ -77,18 +135,45 @@ module Custodian
 
         return unless @redis
 
+        #
+        #  Make sure we know about this test.
+        #
+        test_s = @test.to_s
+
+        @redis.sadd( "known_tests", test_s )
+
+        #
+        #  Get the current state of this test - so that if the state
+        # has changed we can add that to our history.
+        #
+        #  We use SHA1hash to determine our key.
+        #
+        key = Digest::SHA1.hexdigest test_s
 
-        tmp = {}
-        tmp['time']   = Time.now.to_i
-        tmp['type']   = @test.get_type
-        tmp['target'] = @test.target
-        tmp['result'] = 'OK'
-        tmp['reason'] = ''
-        tmp['test'] = @test.to_s
-        tmp['class'] = @test.class
+        puts( "Key is #{key}")
 
-        @redis.lpush('recent-tests', tmp.to_json)
-        @redis.ltrim('recent-tests', 0, 100)
+        #
+        #  The current state
+        #
+        current = @redis.get( "#{key}.current" ) || "unknown"
+        @redis.set( "#{key}.current", "OK" )
+
+        count = @redis.get( "#{key}.count" ) || "0"
+        @redis.set( "#{key}.count", (count.to_i + 1 ))
+
+        if ( current != "OK" )
+
+          #
+          #  The state has changed to raise.
+          #
+          tmp = {}
+          tmp['time']   = Time.now.to_i
+          tmp['result'] = 'OK'
+          tmp['reason'] = @test.error
+
+          @redis.lpush( "#{key}.history", tmp.to_json)
+          @redis.ltrim('#{key}.history', 0, 100)
+        end
       end
 
       register_alert_type 'redis'