From 9c54ed631098db81a8cb2db60890af705c631541 Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Sat, 21 Feb 2015 16:29:24 +0200 Subject: Force new job if too long since last job MAX_INTER_JOB_GAP is now 300s, if latest job was started 300s ago, we add new job. Ratioanele is that if we want n jobs, and all these jobs are taking very very long, or perhaps hanging, then we are blocking everything else too. Consider you have use one job, because it's enough to meet your rotation interval quota. Then some one box is somehow taking tens of minutes or hours, we won't figure out new amount of workers until it finishes, so we're blocking all other jobs from spawning. I'm not super happy about this solution, not really sure what is the right wayt to tackle it. --- lib/oxidized/nodes.rb | 2 +- lib/oxidized/worker.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/oxidized/nodes.rb b/lib/oxidized/nodes.rb index 032118d..3586b97 100644 --- a/lib/oxidized/nodes.rb +++ b/lib/oxidized/nodes.rb @@ -1,6 +1,6 @@ module Oxidized - require 'oxidized/node' require 'ipaddr' + require 'oxidized/node' class Oxidized::NotSupported < OxidizedError; end class Oxidized::NodeNotFound < OxidizedError; end class Nodes < Array diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb index e274e1e..8fe7ab5 100644 --- a/lib/oxidized/worker.rb +++ b/lib/oxidized/worker.rb @@ -2,15 +2,18 @@ module Oxidized require 'oxidized/job' require 'oxidized/jobs' class Worker + MAX_INTER_JOB_GAP = 300 def initialize nodes @nodes = nodes @jobs = Jobs.new CFG.threads, CFG.interval, @nodes + @last = Time.now.utc Thread.abort_on_exception = true end def work ended = [] @jobs.delete_if { |job| ended << job if not job.alive? } ended.each { |job| process job } + @jobs.add_job if Time.now.utc - @last > MAX_INTER_JOB_GAP while @jobs.size < @jobs.want Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}" # ask for next node in queue non destructive way @@ -21,6 +24,7 @@ module Oxidized # shift nodes and get the next node node = @nodes.get node.running? ? next : node.running = true + @last = Time.now.utc @jobs.push Job.new node end end -- cgit v1.2.1