diff options
| -rw-r--r-- | TODO-PJC | 1 | ||||
| -rw-r--r-- | debian/control | 3 | ||||
| -rw-r--r-- | debian/mauvealert-server.init | 15 | ||||
| -rw-r--r-- | heartbeat_hammer.sh | 7 | ||||
| -rw-r--r-- | lib/mauve/alert.rb | 8 | ||||
| -rw-r--r-- | lib/mauve/alert_changed.rb | 8 | ||||
| -rw-r--r-- | lib/mauve/notifiers/email.rb | 9 | ||||
| -rw-r--r-- | lib/mauve/notifiers/sms_aql.rb | 18 | ||||
| -rw-r--r-- | lib/mauve/notifiers/templates/email.html.erb | 7 | ||||
| -rw-r--r-- | lib/mauve/notifiers/templates/email.txt.erb | 5 | ||||
| -rw-r--r-- | lib/mauve/notifiers/templates/xmpp.txt.erb | 8 | ||||
| -rw-r--r-- | lib/mauve/notifiers/xmpp.rb | 7 | ||||
| -rw-r--r-- | lib/mauve/person.rb | 63 | ||||
| -rw-r--r-- | lib/mauve/processor.rb | 2 |
14 files changed, 90 insertions, 71 deletions
@@ -1,2 +1,3 @@ * Javascript ack_time update if ajax synchronicity a bit shite * Template SMS bit better with the link URL to the actual alert. + * Add "too noisy" reminder to notifcations. diff --git a/debian/control b/debian/control index a8edc31..a78c5e7 100644 --- a/debian/control +++ b/debian/control @@ -21,8 +21,9 @@ Description: Mauve network alert system -- client Package: mauvealert-server Architecture: all +Pre-Depends: libjs-jquery Depends: mauvealert-common, - libjs-jquery, + adduser, ruby1.8, libhaml-ruby1.8, liblog4r-ruby1.8, diff --git a/debian/mauvealert-server.init b/debian/mauvealert-server.init index 225a9d4..accea6c 100644 --- a/debian/mauvealert-server.init +++ b/debian/mauvealert-server.init @@ -47,16 +47,25 @@ do_start() # 0 if daemon has been started # 1 if daemon was already running # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --chuid $RUNASUSER --test > /dev/null \ - || return 1 + + start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --chuid $RUNASUSER --test > /dev/null \ + || return 1 start-stop-daemon --start --quiet --pidfile $PIDFILE --make-pidfile --chuid $RUNASUSER --background --startas $DAEMON -- \ $DAEMON_ARGS \ || return 2 # Add code here, if necessary, that waits for the process to be ready # to handle requests from services started subsequently which depend # on this one. As a last resort, sleep for some time. + count=0 + + while [ $((count = count + 1)) -le 5 ] ; do + echo -n . + sleep 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --chuid $RUNASUSER --test > /dev/null \ + || return 0 + done - sleep 5 + return 2 } # diff --git a/heartbeat_hammer.sh b/heartbeat_hammer.sh index de530f2..f2d4671 100644 --- a/heartbeat_hammer.sh +++ b/heartbeat_hammer.sh @@ -2,6 +2,7 @@ PRE="ruby -I lib bin/mauveclient localhost" F=60 +S=10 n=$* _host () { @@ -12,9 +13,9 @@ _host () { if [ "$down" == "n" ] ; then $PRE -o $hostname -i heartbeat -r +$F -c now -s "heartbeat failed" --detail="<p>The heartbeat wasn't sent for this host</p><p>This indicates that the host might be down</p>" - sleep $((F - $RANDOM*5/32768 - 5)) + sleep $((F - $RANDOM*$S/32768 - $S)) else - sleep $((RANDOM*5/32768 + $F + 5)) + sleep $((RANDOM*$S/32768 + $F + $S)) fi if [ $RANDOM -gt 30000 ] ; then @@ -27,7 +28,7 @@ _host () { done } -for i in `seq 1 500` ; do +for i in `seq 1 100` ; do _host $i & sleep 0.2 done diff --git a/lib/mauve/alert.rb b/lib/mauve/alert.rb index 07aeb55..80e1485 100644 --- a/lib/mauve/alert.rb +++ b/lib/mauve/alert.rb @@ -85,9 +85,11 @@ module Mauve validates_with_method :check_dates - def to_s - "#<Alert:#{id} #{alert_id} from #{source} update_type #{update_type}>" + def inspect + "#<Alert #{id}, alert_id #{alert_id}, source #{source}>" end + + alias to_s inspect def check_dates bad_dates = self.attributes.find_all do |key, value| @@ -210,6 +212,8 @@ module Mauve end end + logger.info "#{self.inspect} updated." unless @attributes_before_save.empty? + true end diff --git a/lib/mauve/alert_changed.rb b/lib/mauve/alert_changed.rb index 4dcba8e..40bcad1 100644 --- a/lib/mauve/alert_changed.rb +++ b/lib/mauve/alert_changed.rb @@ -20,9 +20,11 @@ module Mauve property :updated_at, DateTime - def to_s - "#<AlertChanged:#{id} of #{alert_id} for #{person} update_type #{update_type}>" + def inspect + "#<AlertChanged #{id}: alert_id #{alert_id}, for #{person}, update_type #{update_type}>" end + + alias to_s inspect belongs_to :alert @@ -134,7 +136,7 @@ module Mauve # Only remind if the time is right. # if DuringRunner.new(Time.now, alert, ¬ification.during).now? - Configuration.current.people[np].remind(alert, level) + Configuration.current.people[np].send_alert(level, alert) end self.remind_at = notification.remind_at_next(alert) save diff --git a/lib/mauve/notifiers/email.rb b/lib/mauve/notifiers/email.rb index 77b10f1..b6a1e1b 100644 --- a/lib/mauve/notifiers/email.rb +++ b/lib/mauve/notifiers/email.rb @@ -35,7 +35,7 @@ module Mauve end - def send_alert(destination, alert, all_alerts, conditions = nil) + def send_alert(destination, alert, all_alerts, conditions = {}) message = prepare_message(destination, alert, all_alerts, conditions) args = [@server, @port] args += [@username, @password, @login_method.to_sym] if @login_method @@ -54,10 +54,9 @@ module Mauve protected - def prepare_message(destination, alert, all_alerts, conditions = nil) - if conditions - @suppressed_changed = conditions[:suppressed_changed] - end + def prepare_message(destination, alert, all_alerts, conditions = {}) + was_suppressed = conditions[:was_suppressed] || false + is_suppressed = conditions[:is_suppressed] || false m = RMail::Message.new diff --git a/lib/mauve/notifiers/sms_aql.rb b/lib/mauve/notifiers/sms_aql.rb index dbda229..d0cf630 100644 --- a/lib/mauve/notifiers/sms_aql.rb +++ b/lib/mauve/notifiers/sms_aql.rb @@ -19,9 +19,9 @@ module Mauve @name = name end - def send_alert(destination, alert, all_alerts, conditions = nil) + def send_alert(destination, alert, all_alerts, conditions = {}) uri = URI.parse(GATEWAY) - + opts_string = { :username => @username, :password => @password, @@ -52,17 +52,9 @@ module Mauve end protected - def prepare_message(destination, alert, all_alerts, conditions=nil) - if conditions - @suppressed_changed = conditions[:suppressed_changed] - end - - txt = case @suppressed_changed - when true then "TOO MUCH NOISE! Last notification: " - when false then "BACK TO NORMAL: " - else - "" - end + def prepare_message(destination, alert, all_alerts, conditions={}) + was_suppressed = conditions[:was_suppressed] || false + is_suppressed = conditions[:is_suppressed] || false template_file = File.join(File.dirname(__FILE__),"templates","sms.txt.erb") diff --git a/lib/mauve/notifiers/templates/email.html.erb b/lib/mauve/notifiers/templates/email.html.erb index bf5aed5..99f56d5 100644 --- a/lib/mauve/notifiers/templates/email.html.erb +++ b/lib/mauve/notifiers/templates/email.html.erb @@ -19,4 +19,9 @@ end <h2>Detail</h2> <div><%= RedCloth.new(alert.detail).to_html %></div> <hr /> -<address>--<br />Love mauve<br />xxx.</address></body></html> +<% if was_supressed and not is_suppressed %> +<p><strong>Better now. Your notifications have now restarted.</strong></p> +<% elsif is_supressed and not was_suppressed %> +<p><strong>Too noisy! Your notifications have been suppressed.</strong></p> +<% end %> +<address>-- <br />Love mauve<br />xxx.</address></body></html> diff --git a/lib/mauve/notifiers/templates/email.txt.erb b/lib/mauve/notifiers/templates/email.txt.erb index d112718..a08ec55 100644 --- a/lib/mauve/notifiers/templates/email.txt.erb +++ b/lib/mauve/notifiers/templates/email.txt.erb @@ -19,6 +19,11 @@ end ----------------------------------------------------------------------- +<% if was_supressed and not is_suppressed +%>Better now. Your notifications have now restarted.<% +elsif is_supressed and not was_suppressed +%>Too noisy! Your notifications have been suppressed.<% end %> + -- Love mauve. xx diff --git a/lib/mauve/notifiers/templates/xmpp.txt.erb b/lib/mauve/notifiers/templates/xmpp.txt.erb index 2f86cb6..d39500c 100644 --- a/lib/mauve/notifiers/templates/xmpp.txt.erb +++ b/lib/mauve/notifiers/templates/xmpp.txt.erb @@ -11,4 +11,10 @@ end if alert.source != alert.subject %> -- from <%= alert.source %><% end -%>. +%>.<% +if was_suppressed and not is_suppressed +%> Normal service has resumed.<% +elsif is_suppressed and not was_suppressed +%> Further alerts supressed until things calm down.<% +end +%> diff --git a/lib/mauve/notifiers/xmpp.rb b/lib/mauve/notifiers/xmpp.rb index 34cc815..7fe1e39 100644 --- a/lib/mauve/notifiers/xmpp.rb +++ b/lib/mauve/notifiers/xmpp.rb @@ -182,12 +182,11 @@ module Mauve # which checks whether the jid in question has a presence matching one # or more of the choices - see +check_jid_has_presence+ for options. - def send_alert(destination, alert, all_alerts, conditions = nil) + def send_alert(destination, alert, all_alerts, conditions = {}) destination_jid = JID.new(destination) - if conditions - @suppressed_changed = conditions[:suppressed_changed] - end + was_suppressed = conditions[:was_suppressed] || false + is_suppressed = conditions[:is_suppressed] || false if conditions && !check_alert_conditions(destination_jid, conditions) logger.info("Alert conditions not met, not sending XMPP alert to #{destination_jid}") diff --git a/lib/mauve/person.rb b/lib/mauve/person.rb index 199bc23..1a5f2c9 100644 --- a/lib/mauve/person.rb +++ b/lib/mauve/person.rb @@ -7,8 +7,11 @@ module Mauve attr_reader :notification_thresholds - def initialize(*args) - @notification_thresholds = { } # 60 => Array.new(10) } + def initialize(*args) + # + # By default send 10 thresholds in a minute maximum + # + @notification_thresholds = { 60 => Array.new(10) } @suppressed = false super(*args) end @@ -51,6 +54,8 @@ module Mauve if args.first.is_a?(Array) conditions = @base_conditions.merge(args[0]) + else + conditions = @base_conditions end notification_method = Configuration.current.notification_methods[name.to_s] @@ -66,7 +71,7 @@ module Mauve # # Log the result note = "#{@alert.update_type.capitalize} #{name} notification to #{@person.username} (#{destination}) " + (res ? "succeeded" : "failed" ) - logger.info note + logger.info note+" about #{@alert}." h = History.new(:alert_id => @alert.id, :type => "notification", :event => note) logger.error "Unable to save history due to #{h.errors.inspect}" if !h.save @@ -145,10 +150,6 @@ module Mauve send_alert(level, alert) if is_relevant # last_change.was_relevant_when_raised? end - - def remind(alert, level) - send_alert(level, alert) - end # # This just wraps send_alert by sending the job to a queue. @@ -156,44 +157,50 @@ module Mauve def send_alert(level, alert) Server.notification_push([self, level, alert]) end - + def do_send_alert(level, alert) now = MauveTime.now - threshold_breached = @notification_thresholds.any? do |period, previous_alert_times| - first = previous_alert_times.first - first.is_a?(MauveTime) and (now - first) < period - end - was_suppressed = self.suppressed? - if Server.instance.started_at > alert.updated_at.to_time and (Server.instance.started_at + Server.instance.initial_sleep) > MauveTime.now - logger.info("Alert last updated in prior run of mauve -- ignoring for initial sleep period.") - return true + @suppressed = @notification_thresholds.any? do |period, previous_alert_times| + # + # Choose the second one as the first. + # + first = previous_alert_times[1] + first.is_a?(MauveTime) and (now - first) < period end - if threshold_breached + if self.suppressed? logger.info("Suspending further notifications to #{username} until further notice.") unless was_suppressed - @suppressed = true else logger.info "Starting to send notifications again for #{username}." if was_suppressed - @suppressed = false end + if Server.instance.started_at > alert.updated_at.to_time and (Server.instance.started_at + Server.instance.initial_sleep) > MauveTime.now + logger.info("Alert last updated in prior run of mauve -- ignoring for initial sleep period.") + return true + end + # # We only suppress notifications if we were suppressed before we started, # and are still suppressed. # - return true if was_suppressed and self.suppressed? + if was_suppressed and self.suppressed? + note = "#{alert.update_type.capitalize} notification to #{self.username} suppressed" + logger.info note + " about #{alert}." + History.create(:alert_id => alert.id, :type => "notification", :event => note) + return true + end result = NotificationCaller.new( self, alert, current_alerts, - {:is_suppressed => @suppressed, - :was_suppressed => was_suppressed, } + {:is_suppressed => @suppressed, + :was_suppressed => was_suppressed, } ).instance_eval(&__send__(level)) if result @@ -225,18 +232,6 @@ module Mauve end protected - # Remembers that an alert has been sent so that we can later check whether - # too many alerts have been sent in a particular period. - # - def remember_alert(now=MauveTime.now) - end - - # Returns time period over which "too many" alerts have been sent, or nil - # if none. - # - def threshold_breached(now=MauveTime.now) - end - # Whether the person is on holiday or not. # # @return [Boolean] True if person on holiday, false otherwise. diff --git a/lib/mauve/processor.rb b/lib/mauve/processor.rb index f8416e0..15987c2 100644 --- a/lib/mauve/processor.rb +++ b/lib/mauve/processor.rb @@ -54,7 +54,7 @@ module Mauve next end - logger.info "Update #{update.transmission_id} sent at #{update.transmission_time} from "+ + logger.debug "Update #{update.transmission_id} sent at #{update.transmission_time} from "+ "'#{update.source}'@#{ip_source} alerts #{update.alert.length}" Alert.receive_update(update, received_at) |