Added last_state_changed thing for MauveThread. Closes #1836.

5 files changed, 46 insertions, 12 deletions

diff --git a/lib/mauve/alert.rb b/lib/mauve/alert.rb
index 22f6784..bf47d34 100644
--- a/lib/mauve/alert.rb
+++ b/lib/mauve/alert.rb
@@ -319,10 +319,10 @@ module Mauve
     end
     
     def poll
+      logger.debug("Polling #{self.to_s}")
       raise! if (will_unacknowledge_at and will_unacknowledge_at <= Time.now) or
         (will_raise_at and will_raise_at <= Time.now)
       clear! if will_clear_at && will_clear_at <= Time.now
-      logger.info("Polled #{self.to_s}")
     end
 
 
diff --git a/lib/mauve/alert_changed.rb b/lib/mauve/alert_changed.rb
index 2871aee..5cd8b2c 100644
--- a/lib/mauve/alert_changed.rb
+++ b/lib/mauve/alert_changed.rb
@@ -91,6 +91,7 @@ module Mauve
     end
     
     def poll # mimic interface from Alert
+      logger.debug("Polling #{self.to_s}")
       remind if remind_at.is_a?(Time) and remind_at <= Time.now
     end
 
diff --git a/lib/mauve/mauve_thread.rb b/lib/mauve/mauve_thread.rb
index 77df95e..7d5dcbe 100644
--- a/lib/mauve/mauve_thread.rb
+++ b/lib/mauve/mauve_thread.rb
@@ -5,11 +5,10 @@ module Mauve
 
   class MauveThread
 
-    attr_reader :state, :poll_every
+    attr_reader :poll_every
 
     def initialize
       @thread = nil
-      @state = :stopped
     end
 
     def logger
@@ -21,9 +20,9 @@ module Mauve
       # 
       # Set the minimum poll frequency.
       #
-      if i.to_f < 0.2
-        logger.debug "Increasing thread polling interval to 0.2s from #{i}"
-        i = 0.2 
+      if i.to_f < 0
+        logger.debug "Increasing thread polling interval to 0s from #{i}"
+        i = 0 
       end
 
       @poll_every = i
@@ -33,6 +32,7 @@ module Mauve
       #
       # Good to go.
       #
+      @thread = Thread.current
       self.state = :starting
 
       @poll_every ||= interval
@@ -76,15 +76,33 @@ module Mauve
       [:freezing, :stopping].include?(self.state)
     end
 
+    def state
+      if self.alive?
+        @thread.key?(:state) ?  @thread[:state] : :unknown
+      else
+        :stopped
+      end
+    end
+
     def state=(s)
       raise "Bad state for mauve_thread #{s.inspect}" unless [:stopped, :starting, :started, :freezing, :frozen, :stopping, :killing, :killed].include?(s)
+      raise "Thread not ready yet." unless @thread.is_a?(Thread)
 
-      unless @state == s
-        @state = s
+      unless @thread[:state] == s
+        @thread[:state] = s
+        @thread[:last_state_change] = Time.now
         logger.debug(s.to_s.capitalize) 
       end
 
-      @state
+      @thread[:state]
+    end
+
+    def last_state_change
+      if self.alive? and @thread.key?(:last_state_change)
+        return @thread[:last_state_change]
+      else
+        return nil
+      end
     end
 
     def freeze
@@ -101,13 +119,15 @@ module Mauve
 
     def run
       if self.alive? 
+        # Wake up if we're stopped.
         if self.stop?
           @thread.wakeup 
         end
       else
         @logger = nil
-        self.state = :starting
-        @thread = Thread.new{ self.run_thread { self.main_loop } }
+        Thread.new do
+          self.run_thread { self.main_loop } 
+        end
       end
     end
 
diff --git a/lib/mauve/server.rb b/lib/mauve/server.rb
index eb5872b..3d59049 100644
--- a/lib/mauve/server.rb
+++ b/lib/mauve/server.rb
@@ -103,7 +103,7 @@ module Mauve
     end
 
     def start
-      self.state = :starting
+#      self.state = :starting
 
       self.setup
       
@@ -123,6 +123,14 @@ module Mauve
         #
         thread_list.delete(klass.instance.thread)
 
+        #
+        # Make sure that if the thread is frozen, that we've not been frozen for too long.
+        #
+        if klass.instance.state != :started and klass.instance.last_state_change.is_a?(Time) and klass.instance.last_state_change < (Time.now - 2.minutes)
+          logger.warn "#{klass} has been #{klass.instance.state} since #{klass.instance.last_state_change}. Killing and restarting."
+          klass.instance.stop
+        end
+
         # 
         # Do nothing if we're frozen or supposed to be stopping or still alive!
         #
diff --git a/lib/mauve/timer.rb b/lib/mauve/timer.rb
index 00af255..02951b7 100644
--- a/lib/mauve/timer.rb
+++ b/lib/mauve/timer.rb
@@ -11,6 +11,11 @@ module Mauve
 
     include Singleton
 
+    def initialize
+      super
+      @poll_every = 0
+    end
+
     def main_loop
       #
       # Get the next alert.