Added alert suppression to the server code + protobuf.

2 files changed, 281 insertions, 3 deletions

diff --git a/test/tc_mauve_alert.rb b/test/tc_mauve_alert.rb
index 269bf4c..61bd117 100644
--- a/test/tc_mauve_alert.rb
+++ b/test/tc_mauve_alert.rb
@@ -173,6 +173,123 @@ EOF
     
   end
 
+  def test_recieve_update_with_suppression
+    Server.instance.setup
+
+    update = Proto::AlertUpdate.new
+    update.source = "test-host"
+    message = Proto::Alert.new
+    update.alert << message
+    message.id = "test_recieve_update"
+
+    #
+    # If we send the same message every 2 minutes, with the suppress time
+    # moving forward each time, it should get set once, and then stay the same
+    # after that.
+    #
+    suppress_until = nil
+
+    5.times do |i|
+      message.raise_time = Time.now.to_i
+      message.suppress_until = Time.now.to_i + 10.minutes
+      suppress_until = Time.now + 10.minutes if suppress_until.nil?
+
+      Alert.receive_update(update, Time.now, "127.0.0.1")
+      a = Alert.first(:alert_id => 'test_recieve_update')
+
+      assert(a.raised?, "Alert not raised after a raised message has been sent")
+      assert(a.suppressed?, "Alert has not been suppressed when the suppress_until value has been set")
+      assert_equal(suppress_until, a.suppress_until, "The suppress until time has been set incorrectly when being raised repeatedly")
+
+      Timecop.freeze(Time.now + 2.minute)
+    end
+    
+    #
+    # Ten minutes have passed == the suppression should have lapsed.
+    #
+    assert_equal(0, (Time.now - suppress_until).to_i, "Ten minutes have not elapsed!")
+    a = Alert.first(:alert_id => 'test_recieve_update')
+    assert(!a.suppressed?, "The alert is still suppressed past its suppress_until time")
+
+    #
+    # Try again -- the suppression has expired, but the alert is still raised.
+    # This should not re-set the suppress_until value.
+    #
+    5.times do
+      message.suppress_until = Time.now.to_i + 10.minutes
+      Alert.receive_update(update, Time.now, "127.0.0.1")
+      a = Alert.first(:alert_id => 'test_recieve_update')
+      assert(a.raised?)
+      assert_equal(suppress_until, a.suppress_until)
+      assert(!a.suppressed?)
+
+      Timecop.freeze(Time.now + 2.minute)
+    end
+
+    #
+    # Now on clear, we should be able to set the suppression time again.
+    #
+    suppress_until = Time.now + 10.minutes
+    message.raise_time = nil
+    message.clear_time = Time.now.to_i
+    message.suppress_until = suppress_until.to_i
+
+    Alert.receive_update(update, Time.now, "127.0.0.1")
+    a = Alert.first(:alert_id => 'test_recieve_update')
+    assert(a.cleared?)
+    assert_equal(suppress_until, a.suppress_until)
+
+    #
+    # Now move on two minutes, and raise the alert.  The suppression time should not move forward.
+    #
+    Timecop.freeze(Time.now + 2.minutes)
+
+    4.times do
+      message.raise_time = Time.now.to_i
+      message.suppress_until = Time.now.to_i + 10.minutes
+      Alert.receive_update(update, Time.now, "127.0.0.1")
+      a = Alert.first(:alert_id => 'test_recieve_update')
+
+      assert(a.raised?)
+      assert(a.suppressed?)
+      assert_equal(suppress_until, a.suppress_until)
+
+      Timecop.freeze(Time.now + 2.minute)
+    end
+
+    #
+    # 10 minutes have now passed, The alert should no longer be suppressed.
+    #
+    assert_equal(0, (Time.now - suppress_until).to_i, "Ten minutes have not elapsed!")
+    a = Alert.first(:alert_id => 'test_recieve_update')
+    assert(!a.suppressed?, "The alert is still suppressed past its suppress_until time")
+
+    #
+    # Try again -- the suppression has expired, but the alert is still raised.
+    # In this case the suppress_until time should not be moved on, until the
+    # alert has cleared.
+    #
+    message.suppress_until = Time.now.to_i + 10.minutes
+    Alert.receive_update(update, Time.now, "127.0.0.1")
+    a = Alert.first(:alert_id => 'test_recieve_update')
+    assert(a.raised?)
+    assert_equal(suppress_until, a.suppress_until)
+    assert(!a.suppressed?)
+
+    #
+    # Now clear, and the suppress time should be set back to nil.
+    #
+    message.raise_time = nil
+    message.clear_time = Time.now.to_i
+    message.suppress_until = nil
+    Alert.receive_update(update, Time.now, "127.0.0.1")
+    a = Alert.first(:alert_id => 'test_recieve_update')
+    assert(a.cleared?)
+    assert_equal(nil, a.suppress_until)
+    assert(!a.suppressed?)
+
+  end
+    
   def test_notify_if_needed
     Configuration.current = ConfigurationBuilder.parse(@test_config)
     Server.instance.setup
@@ -293,7 +410,7 @@ EOF
     Server.instance.setup
 
     alert = Alert.new(
-      :alert_id  => "test_no_notification_for_old_alerts",
+      :alert_id  => "test_destroy_history_on_destroy",
       :source    => "test",
       :subject   => "test"
     )
@@ -315,4 +432,47 @@ EOF
 
   end
 
+  def test_alert_suppression
+    #
+    # This is a bit of a duff test
+    #
+    Configuration.current = ConfigurationBuilder.parse(@test_config)
+    Server.instance.setup
+
+    alert = Alert.new(
+      :alert_id  => "test_alert_suppression1",
+      :source    => "test",
+      :subject   => "test",
+      :suppress_until => Time.now + 5.minutes
+    )
+    alert.save
+    alert.reload
+
+    assert_equal(Time.now + 5.minutes, alert.suppress_until)
+    assert(alert.suppressed?)
+    
+    alert = Alert.new(
+      :alert_id  => "test_alert_suppression2",
+      :source    => "test",
+      :subject   => "test",
+      :suppress_until => Time.now - 5.minutes
+    )
+    alert.save
+    alert.reload
+
+    assert_equal(Time.now - 5.minutes, alert.suppress_until)
+    assert(!alert.suppressed?,"Alert marked as suppressed when the suppression period has expired")
+
+    alert = Alert.new(
+      :alert_id  => "test_alert_suppressio3",
+      :source    => "test",
+      :subject   => "test"
+    )
+    alert.save
+    alert.reload
+
+    assert_equal(nil, alert.suppress_until)
+    assert(!alert.suppressed?,"Alert marked as suppressed when the suppression period was never set")
+  end
+
 end
diff --git a/test/tc_mauve_alert_group.rb b/test/tc_mauve_alert_group.rb
index 92a9d61..fc31702 100644
--- a/test/tc_mauve_alert_group.rb
+++ b/test/tc_mauve_alert_group.rb
@@ -106,11 +106,14 @@ EOF
     a = Alert.new(
       :alert_id  => "test",
       :source    => "test",
-      :subject   => "test" 
+      :subject   => "test",
+      :suppress_until => Time.now + 5.minutes 
     )
 
     a.raise!
-    assert_equal("test1@example.com", notification_buffer.pop[2])
+    #
+    # Should be suppressed.
+    #
     assert(notification_buffer.empty?)
 
     Timecop.freeze(Time.now + 5.minutes)
@@ -161,6 +164,121 @@ EOF
     assert(notification_buffer.empty?)
   end
 
+  def test_alert_suppression
+    config=<<EOF
+server {
+  database "sqlite3::memory:"
+  use_notification_buffer false
+}
+
+notification_method("email") {
+  debug!
+  deliver_to_queue []
+  disable_normal_delivery!
+}
+
+person ("test1") {
+  email "test1@example.com"
+  all { email }
+}
+
+alert_group("default") {
+  includes{ true }
+  notify("test1")  {
+    during{ true }
+    every 15.minutes
+  }
+}
+EOF
+    Configuration.current = ConfigurationBuilder.parse(config)
+    notification_buffer = Configuration.current.notification_methods["email"].deliver_to_queue
+    Server.instance.setup
+
+    a = Alert.new(
+      :alert_id  => "test",
+      :source    => "test",
+      :subject   => "test",
+      :suppress_until => Time.now + 5.minutes 
+    )
+
+    #
+    # No notifications should be sent for 5 minutes
+    #
+    a.raise!
+
+    5.times do
+      assert(0,notification_buffer.length)
+      Timecop.freeze(Time.now + 1.minutes)
+      a.poll
+      AlertChanged.all.each{|ac| ac.poll}
+    end
+
+    #
+    # After 5 minutes a notification should be sent, and a reminder set for 15 minutes afterwards.
+    #
+    assert(1,notification_buffer.length)
+    ac = a.changes.all(:remind_at.not => nil)
+    assert_equal(1,ac.length, "Only one reminder should be in place at end of suppression period")
+    assert_equal(Time.now+15.minutes, ac.first.remind_at, "Reminder not set for the correct time after suppression")
+
+    #
+    # Clear the alert.
+    #
+    a.clear!
+    assert(1, notification_buffer.length)
+    ac = a.changes.all(:remind_at.not => nil)
+    assert_equal(0,ac.length, "No reminders should be in place after a clear")
+
+
+    #####
+    #
+    # This covers a planned maintenance scenario, when an alert is suppressed
+    # whilst cleared.  Flapping should not cause any notifications.
+    #
+
+    #
+    # No notifications should be sent for 15 minutes
+    #
+    a.suppress_until = Time.now + 15.minutes
+    a.clear!
+
+    Timecop.freeze(Time.now + 3.minutes)
+
+    2.times do
+      5.times do
+        #
+        # Raise.  This should not cause any notifications for 10 minutes.
+        #
+        a.raise!
+        assert(0,notification_buffer.length)
+        Timecop.freeze(Time.now + 1.minutes)
+        a.poll
+        AlertChanged.all.each{|ac| ac.poll}
+      end
+
+      #
+      # This should not raise any alerts, and all reminders should be cleared.
+      #
+      a.clear!
+      assert(0,notification_buffer.length)
+
+      Timecop.freeze(Time.now + 1.minutes)
+      a.poll
+      AlertChanged.all.each{|ac| ac.poll}
+
+      ac = a.changes.all(:remind_at.not => nil)
+      assert_equal(0, ac.length, "Reminder in place at when the raised alert was cleared.")
+    end
+
+    # Now re-raise.
+    a.raise!
+    assert(1,notification_buffer.length)
+
+    ac = a.changes.all(:remind_at.not => nil)
+    assert_equal(1,ac.length, "Only one reminder should be in place at end of suppression period")
+    assert_equal(Time.now+15.minutes, ac.first.remind_at, "Reminder not set for the correct time after suppression")
+  end
+
 end