aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPatrick J Cherry <patrick@bytemark.co.uk>2011-07-14 18:00:49 +0100
committerPatrick J Cherry <patrick@bytemark.co.uk>2011-07-14 18:00:49 +0100
commit18b0906630ab4165d1e8b4bc66d7446b17c31605 (patch)
tree359b6018b6c7327d3c72c23516cd9378ce01a866
parent494b083408b354f38da9b1fc5e8ffb7238d009b3 (diff)
* Improved logging.
* Updated message suppression
-rw-r--r--TODO-PJC1
-rw-r--r--debian/control3
-rw-r--r--debian/mauvealert-server.init15
-rw-r--r--heartbeat_hammer.sh7
-rw-r--r--lib/mauve/alert.rb8
-rw-r--r--lib/mauve/alert_changed.rb8
-rw-r--r--lib/mauve/notifiers/email.rb9
-rw-r--r--lib/mauve/notifiers/sms_aql.rb18
-rw-r--r--lib/mauve/notifiers/templates/email.html.erb7
-rw-r--r--lib/mauve/notifiers/templates/email.txt.erb5
-rw-r--r--lib/mauve/notifiers/templates/xmpp.txt.erb8
-rw-r--r--lib/mauve/notifiers/xmpp.rb7
-rw-r--r--lib/mauve/person.rb63
-rw-r--r--lib/mauve/processor.rb2
14 files changed, 90 insertions, 71 deletions
diff --git a/TODO-PJC b/TODO-PJC
index 15f04a2..ce45b90 100644
--- a/TODO-PJC
+++ b/TODO-PJC
@@ -1,2 +1,3 @@
* Javascript ack_time update if ajax synchronicity a bit shite
* Template SMS bit better with the link URL to the actual alert.
+ * Add "too noisy" reminder to notifcations.
diff --git a/debian/control b/debian/control
index a8edc31..a78c5e7 100644
--- a/debian/control
+++ b/debian/control
@@ -21,8 +21,9 @@ Description: Mauve network alert system -- client
Package: mauvealert-server
Architecture: all
+Pre-Depends: libjs-jquery
Depends: mauvealert-common,
- libjs-jquery,
+ adduser,
ruby1.8,
libhaml-ruby1.8,
liblog4r-ruby1.8,
diff --git a/debian/mauvealert-server.init b/debian/mauvealert-server.init
index 225a9d4..accea6c 100644
--- a/debian/mauvealert-server.init
+++ b/debian/mauvealert-server.init
@@ -47,16 +47,25 @@ do_start()
# 0 if daemon has been started
# 1 if daemon was already running
# 2 if daemon could not be started
- start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --chuid $RUNASUSER --test > /dev/null \
- || return 1
+
+ start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --chuid $RUNASUSER --test > /dev/null \
+ || return 1
start-stop-daemon --start --quiet --pidfile $PIDFILE --make-pidfile --chuid $RUNASUSER --background --startas $DAEMON -- \
$DAEMON_ARGS \
|| return 2
# Add code here, if necessary, that waits for the process to be ready
# to handle requests from services started subsequently which depend
# on this one. As a last resort, sleep for some time.
+ count=0
+
+ while [ $((count = count + 1)) -le 5 ] ; do
+ echo -n .
+ sleep 1
+ start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --chuid $RUNASUSER --test > /dev/null \
+ || return 0
+ done
- sleep 5
+ return 2
}
#
diff --git a/heartbeat_hammer.sh b/heartbeat_hammer.sh
index de530f2..f2d4671 100644
--- a/heartbeat_hammer.sh
+++ b/heartbeat_hammer.sh
@@ -2,6 +2,7 @@
PRE="ruby -I lib bin/mauveclient localhost"
F=60
+S=10
n=$*
_host () {
@@ -12,9 +13,9 @@ _host () {
if [ "$down" == "n" ] ; then
$PRE -o $hostname -i heartbeat -r +$F -c now -s "heartbeat failed" --detail="<p>The heartbeat wasn't sent for this host</p><p>This indicates that the host might be down</p>"
- sleep $((F - $RANDOM*5/32768 - 5))
+ sleep $((F - $RANDOM*$S/32768 - $S))
else
- sleep $((RANDOM*5/32768 + $F + 5))
+ sleep $((RANDOM*$S/32768 + $F + $S))
fi
if [ $RANDOM -gt 30000 ] ; then
@@ -27,7 +28,7 @@ _host () {
done
}
-for i in `seq 1 500` ; do
+for i in `seq 1 100` ; do
_host $i &
sleep 0.2
done
diff --git a/lib/mauve/alert.rb b/lib/mauve/alert.rb
index 07aeb55..80e1485 100644
--- a/lib/mauve/alert.rb
+++ b/lib/mauve/alert.rb
@@ -85,9 +85,11 @@ module Mauve
validates_with_method :check_dates
- def to_s
- "#<Alert:#{id} #{alert_id} from #{source} update_type #{update_type}>"
+ def inspect
+ "#<Alert #{id}, alert_id #{alert_id}, source #{source}>"
end
+
+ alias to_s inspect
def check_dates
bad_dates = self.attributes.find_all do |key, value|
@@ -210,6 +212,8 @@ module Mauve
end
end
+ logger.info "#{self.inspect} updated." unless @attributes_before_save.empty?
+
true
end
diff --git a/lib/mauve/alert_changed.rb b/lib/mauve/alert_changed.rb
index 4dcba8e..40bcad1 100644
--- a/lib/mauve/alert_changed.rb
+++ b/lib/mauve/alert_changed.rb
@@ -20,9 +20,11 @@ module Mauve
property :updated_at, DateTime
- def to_s
- "#<AlertChanged:#{id} of #{alert_id} for #{person} update_type #{update_type}>"
+ def inspect
+ "#<AlertChanged #{id}: alert_id #{alert_id}, for #{person}, update_type #{update_type}>"
end
+
+ alias to_s inspect
belongs_to :alert
@@ -134,7 +136,7 @@ module Mauve
# Only remind if the time is right.
#
if DuringRunner.new(Time.now, alert, &notification.during).now?
- Configuration.current.people[np].remind(alert, level)
+ Configuration.current.people[np].send_alert(level, alert)
end
self.remind_at = notification.remind_at_next(alert)
save
diff --git a/lib/mauve/notifiers/email.rb b/lib/mauve/notifiers/email.rb
index 77b10f1..b6a1e1b 100644
--- a/lib/mauve/notifiers/email.rb
+++ b/lib/mauve/notifiers/email.rb
@@ -35,7 +35,7 @@ module Mauve
end
- def send_alert(destination, alert, all_alerts, conditions = nil)
+ def send_alert(destination, alert, all_alerts, conditions = {})
message = prepare_message(destination, alert, all_alerts, conditions)
args = [@server, @port]
args += [@username, @password, @login_method.to_sym] if @login_method
@@ -54,10 +54,9 @@ module Mauve
protected
- def prepare_message(destination, alert, all_alerts, conditions = nil)
- if conditions
- @suppressed_changed = conditions[:suppressed_changed]
- end
+ def prepare_message(destination, alert, all_alerts, conditions = {})
+ was_suppressed = conditions[:was_suppressed] || false
+ is_suppressed = conditions[:is_suppressed] || false
m = RMail::Message.new
diff --git a/lib/mauve/notifiers/sms_aql.rb b/lib/mauve/notifiers/sms_aql.rb
index dbda229..d0cf630 100644
--- a/lib/mauve/notifiers/sms_aql.rb
+++ b/lib/mauve/notifiers/sms_aql.rb
@@ -19,9 +19,9 @@ module Mauve
@name = name
end
- def send_alert(destination, alert, all_alerts, conditions = nil)
+ def send_alert(destination, alert, all_alerts, conditions = {})
uri = URI.parse(GATEWAY)
-
+
opts_string = {
:username => @username,
:password => @password,
@@ -52,17 +52,9 @@ module Mauve
end
protected
- def prepare_message(destination, alert, all_alerts, conditions=nil)
- if conditions
- @suppressed_changed = conditions[:suppressed_changed]
- end
-
- txt = case @suppressed_changed
- when true then "TOO MUCH NOISE! Last notification: "
- when false then "BACK TO NORMAL: "
- else
- ""
- end
+ def prepare_message(destination, alert, all_alerts, conditions={})
+ was_suppressed = conditions[:was_suppressed] || false
+ is_suppressed = conditions[:is_suppressed] || false
template_file = File.join(File.dirname(__FILE__),"templates","sms.txt.erb")
diff --git a/lib/mauve/notifiers/templates/email.html.erb b/lib/mauve/notifiers/templates/email.html.erb
index bf5aed5..99f56d5 100644
--- a/lib/mauve/notifiers/templates/email.html.erb
+++ b/lib/mauve/notifiers/templates/email.html.erb
@@ -19,4 +19,9 @@ end
<h2>Detail</h2>
<div><%= RedCloth.new(alert.detail).to_html %></div>
<hr />
-<address>--<br />Love mauve<br />xxx.</address></body></html>
+<% if was_supressed and not is_suppressed %>
+<p><strong>Better now. Your notifications have now restarted.</strong></p>
+<% elsif is_supressed and not was_suppressed %>
+<p><strong>Too noisy! Your notifications have been suppressed.</strong></p>
+<% end %>
+<address>-- <br />Love mauve<br />xxx.</address></body></html>
diff --git a/lib/mauve/notifiers/templates/email.txt.erb b/lib/mauve/notifiers/templates/email.txt.erb
index d112718..a08ec55 100644
--- a/lib/mauve/notifiers/templates/email.txt.erb
+++ b/lib/mauve/notifiers/templates/email.txt.erb
@@ -19,6 +19,11 @@ end
-----------------------------------------------------------------------
+<% if was_supressed and not is_suppressed
+%>Better now. Your notifications have now restarted.<%
+elsif is_supressed and not was_suppressed
+%>Too noisy! Your notifications have been suppressed.<% end %>
+
--
Love mauve.
xx
diff --git a/lib/mauve/notifiers/templates/xmpp.txt.erb b/lib/mauve/notifiers/templates/xmpp.txt.erb
index 2f86cb6..d39500c 100644
--- a/lib/mauve/notifiers/templates/xmpp.txt.erb
+++ b/lib/mauve/notifiers/templates/xmpp.txt.erb
@@ -11,4 +11,10 @@ end
if alert.source != alert.subject
%> -- from <%= alert.source %><%
end
-%>.
+%>.<%
+if was_suppressed and not is_suppressed
+%> Normal service has resumed.<%
+elsif is_suppressed and not was_suppressed
+%> Further alerts supressed until things calm down.<%
+end
+%>
diff --git a/lib/mauve/notifiers/xmpp.rb b/lib/mauve/notifiers/xmpp.rb
index 34cc815..7fe1e39 100644
--- a/lib/mauve/notifiers/xmpp.rb
+++ b/lib/mauve/notifiers/xmpp.rb
@@ -182,12 +182,11 @@ module Mauve
# which checks whether the jid in question has a presence matching one
# or more of the choices - see +check_jid_has_presence+ for options.
- def send_alert(destination, alert, all_alerts, conditions = nil)
+ def send_alert(destination, alert, all_alerts, conditions = {})
destination_jid = JID.new(destination)
- if conditions
- @suppressed_changed = conditions[:suppressed_changed]
- end
+ was_suppressed = conditions[:was_suppressed] || false
+ is_suppressed = conditions[:is_suppressed] || false
if conditions && !check_alert_conditions(destination_jid, conditions)
logger.info("Alert conditions not met, not sending XMPP alert to #{destination_jid}")
diff --git a/lib/mauve/person.rb b/lib/mauve/person.rb
index 199bc23..1a5f2c9 100644
--- a/lib/mauve/person.rb
+++ b/lib/mauve/person.rb
@@ -7,8 +7,11 @@ module Mauve
attr_reader :notification_thresholds
- def initialize(*args)
- @notification_thresholds = { } # 60 => Array.new(10) }
+ def initialize(*args)
+ #
+ # By default send 10 thresholds in a minute maximum
+ #
+ @notification_thresholds = { 60 => Array.new(10) }
@suppressed = false
super(*args)
end
@@ -51,6 +54,8 @@ module Mauve
if args.first.is_a?(Array)
conditions = @base_conditions.merge(args[0])
+ else
+ conditions = @base_conditions
end
notification_method = Configuration.current.notification_methods[name.to_s]
@@ -66,7 +71,7 @@ module Mauve
#
# Log the result
note = "#{@alert.update_type.capitalize} #{name} notification to #{@person.username} (#{destination}) " + (res ? "succeeded" : "failed" )
- logger.info note
+ logger.info note+" about #{@alert}."
h = History.new(:alert_id => @alert.id, :type => "notification", :event => note)
logger.error "Unable to save history due to #{h.errors.inspect}" if !h.save
@@ -145,10 +150,6 @@ module Mauve
send_alert(level, alert) if is_relevant # last_change.was_relevant_when_raised?
end
-
- def remind(alert, level)
- send_alert(level, alert)
- end
#
# This just wraps send_alert by sending the job to a queue.
@@ -156,44 +157,50 @@ module Mauve
def send_alert(level, alert)
Server.notification_push([self, level, alert])
end
-
+
def do_send_alert(level, alert)
now = MauveTime.now
- threshold_breached = @notification_thresholds.any? do |period, previous_alert_times|
- first = previous_alert_times.first
- first.is_a?(MauveTime) and (now - first) < period
- end
-
was_suppressed = self.suppressed?
- if Server.instance.started_at > alert.updated_at.to_time and (Server.instance.started_at + Server.instance.initial_sleep) > MauveTime.now
- logger.info("Alert last updated in prior run of mauve -- ignoring for initial sleep period.")
- return true
+ @suppressed = @notification_thresholds.any? do |period, previous_alert_times|
+ #
+ # Choose the second one as the first.
+ #
+ first = previous_alert_times[1]
+ first.is_a?(MauveTime) and (now - first) < period
end
- if threshold_breached
+ if self.suppressed?
logger.info("Suspending further notifications to #{username} until further notice.") unless was_suppressed
- @suppressed = true
else
logger.info "Starting to send notifications again for #{username}." if was_suppressed
- @suppressed = false
end
+ if Server.instance.started_at > alert.updated_at.to_time and (Server.instance.started_at + Server.instance.initial_sleep) > MauveTime.now
+ logger.info("Alert last updated in prior run of mauve -- ignoring for initial sleep period.")
+ return true
+ end
+
#
# We only suppress notifications if we were suppressed before we started,
# and are still suppressed.
#
- return true if was_suppressed and self.suppressed?
+ if was_suppressed and self.suppressed?
+ note = "#{alert.update_type.capitalize} notification to #{self.username} suppressed"
+ logger.info note + " about #{alert}."
+ History.create(:alert_id => alert.id, :type => "notification", :event => note)
+ return true
+ end
result = NotificationCaller.new(
self,
alert,
current_alerts,
- {:is_suppressed => @suppressed,
- :was_suppressed => was_suppressed, }
+ {:is_suppressed => @suppressed,
+ :was_suppressed => was_suppressed, }
).instance_eval(&__send__(level))
if result
@@ -225,18 +232,6 @@ module Mauve
end
protected
- # Remembers that an alert has been sent so that we can later check whether
- # too many alerts have been sent in a particular period.
- #
- def remember_alert(now=MauveTime.now)
- end
-
- # Returns time period over which "too many" alerts have been sent, or nil
- # if none.
- #
- def threshold_breached(now=MauveTime.now)
- end
-
# Whether the person is on holiday or not.
#
# @return [Boolean] True if person on holiday, false otherwise.
diff --git a/lib/mauve/processor.rb b/lib/mauve/processor.rb
index f8416e0..15987c2 100644
--- a/lib/mauve/processor.rb
+++ b/lib/mauve/processor.rb
@@ -54,7 +54,7 @@ module Mauve
next
end
- logger.info "Update #{update.transmission_id} sent at #{update.transmission_time} from "+
+ logger.debug "Update #{update.transmission_id} sent at #{update.transmission_time} from "+
"'#{update.source}'@#{ip_source} alerts #{update.alert.length}"
Alert.receive_update(update, received_at)