diff options
| -rwxr-xr-x | bin/mauveclient | 6 | ||||
| -rwxr-xr-x | bin/mauveserver | 7 | ||||
| -rw-r--r-- | debian/mauvealert-server.init | 4 | ||||
| -rw-r--r-- | debian/mauvealert-server.install | 1 | ||||
| -rw-r--r-- | debian/mauvealert-server.logrotate | 2 | ||||
| -rw-r--r-- | heartbeat_hammer.sh | 30 | ||||
| -rw-r--r-- | lib/mauve/alert.rb | 26 | ||||
| -rw-r--r-- | lib/mauve/person.rb | 27 | ||||
| -rw-r--r-- | lib/mauve/processor.rb | 10 | ||||
| -rw-r--r-- | lib/mauve/udp_server.rb | 5 |
10 files changed, 69 insertions, 49 deletions
diff --git a/bin/mauveclient b/bin/mauveclient index 251b45b..e3bc453 100755 --- a/bin/mauveclient +++ b/bin/mauveclient @@ -183,9 +183,9 @@ def parse_time_spec(spec = "now") end multiplier *= case $3 - when ?m then 60 - when ?h then 3600 - when ?d then 86400 + when "m" then 60 + when "h" then 3600 + when "d" then 86400 else 1 end diff --git a/bin/mauveserver b/bin/mauveserver index a5cf7b5..4ac5a6d 100755 --- a/bin/mauveserver +++ b/bin/mauveserver @@ -68,8 +68,6 @@ end %w(HUP).each do |sig| trap(sig) do - # this blows up if you do it twice in quick succession, but don't really - # care about that case as it's only for log rotation. Mauve::Server.instance.logger.warn "#{sig} signal received. Restarting." Mauve::Server.instance.stop @@ -80,7 +78,7 @@ end new_config = Mauve::ConfigurationBuilder.load(configuration_file) Mauve::Configuration.current = new_config rescue BuildException => ex - Mauve::Server.instance.logger.error "Reconfiguration failed: #{ex}. Sticking with old one." + Mauve::Server.instance.logger.error "Reconfiguration failed: #{ex.to_s}. Sticking with old one." end Mauve::Server.instance.logger.warn "Restarting." Mauve::Server.instance.start @@ -90,6 +88,7 @@ end %w(USR1).each do |sig| trap(sig) do Mauve::Server.instance.logger.warn "#{sig} signal received. Re-opening logs." + Log4r::Outputter.each_outputter do |old| next unless old.is_a?(Log4r::FileOutputter) @@ -101,7 +100,7 @@ end Mauve::Server.instance.logger.outputters.delete(old) old.close - Mauve::Server.instance.logger.debug "Opened #{new.filename}." + Mauve::Server.instance.logger.info "Opened #{new.filename}." end end end diff --git a/debian/mauvealert-server.init b/debian/mauvealert-server.init index 15c287b..225a9d4 100644 --- a/debian/mauvealert-server.init +++ b/debian/mauvealert-server.init @@ -22,6 +22,7 @@ DAEMON=/usr/sbin/$NAME DAEMON_ARGS="/etc/mauvealert/$NAME.conf" PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME +RUNASUSER=mauvealert # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 @@ -135,6 +136,9 @@ case "$1" in do_reload log_end_msg $? ;; + # + # This is for use by logrotate, as the post-rotate method. + # reopen-logs) log_daemon_msg "Re-opening logs for $DESC" "$NAME" do_reopenlogs diff --git a/debian/mauvealert-server.install b/debian/mauvealert-server.install index c23596d..f5950da 100644 --- a/debian/mauvealert-server.install +++ b/debian/mauvealert-server.install @@ -8,6 +8,7 @@ lib/mauve/auth_bytemark.rb usr/lib/ruby/1.8/mauve/ lib/mauve/calendar_interface.rb usr/lib/ruby/1.8/mauve/ lib/mauve/configuration.rb usr/lib/ruby/1.8/mauve/ lib/mauve/datamapper.rb usr/lib/ruby/1.8/mauve/ +lib/mauve/history.rb usr/lib/ruby/1.8/mauve/ lib/mauve/http_server.rb usr/lib/ruby/1.8/mauve/ lib/mauve/mauve_thread.rb usr/lib/ruby/1.8/mauve/ lib/mauve/notification.rb usr/lib/ruby/1.8/mauve/ diff --git a/debian/mauvealert-server.logrotate b/debian/mauvealert-server.logrotate index 3885a35..9d15453 100644 --- a/debian/mauvealert-server.logrotate +++ b/debian/mauvealert-server.logrotate @@ -1,4 +1,4 @@ -/var/log/mauvealert.log { +/var/log/mauvealert/mauvealert.log { rotate 7 daily compress diff --git a/heartbeat_hammer.sh b/heartbeat_hammer.sh index 9b60171..de530f2 100644 --- a/heartbeat_hammer.sh +++ b/heartbeat_hammer.sh @@ -1,18 +1,34 @@ #!/bin/bash -PRE="ruby -I lib ./bin/mauveclient localhost" +PRE="ruby -I lib bin/mauveclient localhost" +F=60 +n=$* _host () { - hostname="imaginary-$i.bytemark.co.uk" + hostname="imaginary-$i.example.com" + down="n" + while( true ) ; do - sleep $((RANDOM/60)) - echo $hostname - $PRE -o $hostname -i heartbeat -r +2m -c now -s "heartbeat failed" --detail="<p>The heartbeat wasn't sent for this host</p><p>This indicates that the host might be down</p>" + + if [ "$down" == "n" ] ; then + $PRE -o $hostname -i heartbeat -r +$F -c now -s "heartbeat failed" --detail="<p>The heartbeat wasn't sent for this host</p><p>This indicates that the host might be down</p>" + sleep $((F - $RANDOM*5/32768 - 5)) + else + sleep $((RANDOM*5/32768 + $F + 5)) + fi + + if [ $RANDOM -gt 30000 ] ; then + [ "$down" == "n" ] && echo "Host $hostname down" + down="y" + else + [ "$down" == "y" ] && echo "Host $hostname up" + down="n" + fi done } - -for i in `seq 1 50` ; do +for i in `seq 1 500` ; do _host $i & + sleep 0.2 done diff --git a/lib/mauve/alert.rb b/lib/mauve/alert.rb index 147c7cb..07aeb55 100644 --- a/lib/mauve/alert.rb +++ b/lib/mauve/alert.rb @@ -162,9 +162,9 @@ module Mauve # This allows us to take a copy of the changes before we save. # def take_copy_of_changes - @changes_before_save = Hash.new + @attributes_before_save = Hash.new self.original_attributes.each do |k,v| - @changes_before_save[k.name] = v + @attributes_before_save[k.name] = v end end @@ -175,25 +175,30 @@ module Mauve # # Make sure we don't barf # - @changes_before_save ||= Hash.new + @attributes_before_save ||= Hash.new - is_a_change = [:subject, :summary].any?{|k| @changes_before_save.keys.include?(k)} + is_a_new_alert = @attributes_before_save.values.all?{|a| a.nil?} + # + # Do not alert about changes, for now. + # + is_a_change = false # [:subject, :summary].any?{|k| @attributes_before_save.keys.include?(k)} # - # We notify if the update type has changed, or if the update type is + # We notify if the update type has changed (but not from nil), or if the update type is # "raised", and the above is_a_change condition is true # - if @changes_before_save.has_key?(:update_type) or (self.update_type == "raised" and is_a_change) + if (@attributes_before_save.has_key?(:update_type) and !is_a_new_alert) or + (self.update_type == "raised" and (is_a_new_alert or is_a_change)) self.notify h = History.new(:alert_id => self.id, :type => "update") if self.update_type == "acknowledged" - h.event = "ACKNOWLEDGED by #{self.acknowledged_by} until #{self.will_unacknowledge_at}" + h.event = "ACKNOWLEDGED by #{self.acknowledged_by} until #{self.will_unacknowledge_at}" elsif is_a_change h.event = "CHANGED: " - h.event += @changes_before_save.keys.collect{|k| "#{k.to_s}: #{@changes_before_save[k]} -> #{self.__send__(k)}"}.join(", ") + h.event += @attributes_before_save.keys.collect{|k| "#{k.to_s}: #{@attributes_before_save[k]} -> #{self.__send__(k)}"}.join(", ") else h.event = self.update_type.upcase @@ -473,11 +478,6 @@ module Mauve alert_db.importance = alert.importance if alert.importance != 0 - # - # This will probably get overwritten below. - # - # alert_db.update_type = "changed" unless alert_db.update_type - alert_db.updated_at = reception_time if alert_db.raised? diff --git a/lib/mauve/person.rb b/lib/mauve/person.rb index cf29ff9..199bc23 100644 --- a/lib/mauve/person.rb +++ b/lib/mauve/person.rb @@ -24,11 +24,10 @@ module Mauve # class NotificationCaller - def initialize(person, alert, other_alerts, notification_methods, base_conditions={}) + def initialize(person, alert, other_alerts, base_conditions={}) @person = person @alert = alert @other_alerts = other_alerts - @notification_methods = notification_methods @base_conditions = base_conditions end @@ -66,7 +65,7 @@ module Mauve # # Log the result - note = "#{@alert.update_type.upcase}: notification " + (res ? "succeeded" : "failed" ) + " for #{@person.username} using notifier '#{name}' to '#{destination}'." + note = "#{@alert.update_type.capitalize} #{name} notification to #{@person.username} (#{destination}) " + (res ? "succeeded" : "failed" ) logger.info note h = History.new(:alert_id => @alert.id, :type => "notification", :event => note) logger.error "Unable to save history due to #{h.errors.inspect}" if !h.save @@ -155,6 +154,10 @@ module Mauve # This just wraps send_alert by sending the job to a queue. # def send_alert(level, alert) + Server.notification_push([self, level, alert]) + end + + def do_send_alert(level, alert) now = MauveTime.now threshold_breached = @notification_thresholds.any? do |period, previous_alert_times| @@ -162,15 +165,15 @@ module Mauve first.is_a?(MauveTime) and (now - first) < period end - was_suppressed = self.suppressed? + was_suppressed = self.suppressed? if Server.instance.started_at > alert.updated_at.to_time and (Server.instance.started_at + Server.instance.initial_sleep) > MauveTime.now logger.info("Alert last updated in prior run of mauve -- ignoring for initial sleep period.") - return + return true end if threshold_breached - logger.info("Suspending notifications to #{username} until further notice.") unless was_suppressed + logger.info("Suspending further notifications to #{username} until further notice.") unless was_suppressed @suppressed = true else @@ -183,18 +186,14 @@ module Mauve # We only suppress notifications if we were suppressed before we started, # and are still suppressed. # - return if was_suppressed and self.suppressed? - - Server.notification_push([self, level, alert, was_suppressed]) - end + return true if was_suppressed and self.suppressed? - def do_send_alert(level, alert, was_suppressed) result = NotificationCaller.new( self, alert, current_alerts, - Configuration.current.notification_methods, - :was_suppressed => was_suppressed + {:is_suppressed => @suppressed, + :was_suppressed => was_suppressed, } ).instance_eval(&__send__(level)) if result @@ -209,8 +208,10 @@ module Mauve @notification_thresholds[period].shift end true + else false + end end diff --git a/lib/mauve/processor.rb b/lib/mauve/processor.rb index a1c0896..f8416e0 100644 --- a/lib/mauve/processor.rb +++ b/lib/mauve/processor.rb @@ -25,10 +25,10 @@ module Mauve end def main_loop - # - # Only do the loop a maximum of 10 times every @sleep_interval seconds - # - 10.times do + + sz = Server.packet_buffer_size + + sz.times do data, client, received_at = Server.packet_pop # @@ -47,7 +47,7 @@ module Mauve update.parse_from_string(data) if @transmission_id_cache[update.transmission_id.to_s] - logger.info("Ignoring duplicate transmission id #{update.transmission_id}") + logger.debug("Ignoring duplicate transmission id #{update.transmission_id}") # # Continue with next packet. # diff --git a/lib/mauve/udp_server.rb b/lib/mauve/udp_server.rb index 8649efe..049ea09 100644 --- a/lib/mauve/udp_server.rb +++ b/lib/mauve/udp_server.rb @@ -50,7 +50,7 @@ module Mauve @socket.bind(@ip, @port) - logger.info("Successfully opened UDP socket on #{@ip}:#{@port}") + logger.info("Opened socket on #{@ip}:#{@port}") end def close_socket @@ -64,7 +64,7 @@ module Mauve logger.debug ex.backtrace.join("\n") end - logger.info("Successfully closed UDP socket") + logger.info("Closed socket") end def main_loop @@ -83,7 +83,6 @@ module Mauve packet = @socket.recvfrom(65535) received_at = MauveTime.now rescue Errno::EAGAIN, Errno::EWOULDBLOCK => ex - puts "#{i += 1} + #{ex}" IO.select([@socket]) retry unless self.should_stop? end |