add support for retrying failed attempts

Looks like this in syslog:
Jul 11 21:05:53 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 22"
Jul 11 21:05:53 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 23"
Jul 11 21:05:54 ytti oxidized[9820]: 10.10.10.10 status no_connection, retry attempt 1
Jul 11 21:05:54 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 22"
Jul 11 21:05:54 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 23"
Jul 11 21:05:55 ytti oxidized[9820]: 10.10.10.10 status no_connection, retry attempt 2
Jul 11 21:05:55 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 22"
Jul 11 21:05:55 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 23"
Jul 11 21:05:56 ytti oxidized[9820]: 10.10.10.10 status no_connection, retry attempt 3
Jul 11 21:05:56 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 22"
Jul 11 21:05:56 ytti oxidized[9820]: 10.10.10.10 raised Errno::ENETUNREACH with msg "Network is unreachable - connect(2) for "10.10.10.10" port 23"
Jul 11 21:05:57 ytti oxidized[9820]: 10.10.10.10 status no_connection, retries exhausted, giving up

1 files changed, 12 insertions, 3 deletions

diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb
index 58bf659..0e96212 100644
--- a/lib/oxidized/worker.rb
+++ b/lib/oxidized/worker.rb
@@ -29,17 +29,26 @@ module Oxidized
       node.last = job
       node.stats.add job
       @jobs.duration job.time
+      node.running = false
       if job.status == :success
         msg = "update #{node.name}"
         msg += " from #{node.from}" if node.from
         msg += " with message '#{node.msg}'" if node.msg
         node.output.new.store node.name, job.config,
                               :msg => msg, :user => node.user, :group => node.group
-        node.reset
       else
-        Log.warn "#{node.name} status #{job.status}"
+        msg = "#{node.name} status #{job.status}"
+        if node.retry < CFG.retries
+          node.retry += 1
+          msg += ", retry attempt #{node.retry}"
+          @nodes.next node.name
+        else
+          msg += ", retries exhausted, giving up"
+          node.retry = 0
+        end
+        Log.warn msg
       end
-      node.running = false
+      node.reset
     end
   end
 end