summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSaku Ytti <saku@ytti.fi>2015-02-22 13:05:37 +0200
committerSaku Ytti <saku@ytti.fi>2015-02-22 13:05:37 +0200
commitaea148d771dca0e10ca1a2f5505446fbc689dad2 (patch)
treeaaa17621cb03f2c63bd689310725059a8ba9e05d
parent33124f2570a4d8c4dfa153120dbc06b3c725a25c (diff)
refactor HOLB prevention
Still not sure we want this. But previous one might have caused infinite loop in #work. Consider we have just 1 node all together, and our rotation interval is more than our MAX_INTER_JOB_GAP, then we'd add @want to 2, instead of 1. Now we want more threads than we have nodes, and 'while @jobs.size < @jobs.want' will never be true
-rw-r--r--lib/oxidized/jobs.rb24
-rw-r--r--lib/oxidized/worker.rb6
2 files changed, 22 insertions, 8 deletions
diff --git a/lib/oxidized/jobs.rb b/lib/oxidized/jobs.rb
index 3342679..ff7f92b 100644
--- a/lib/oxidized/jobs.rb
+++ b/lib/oxidized/jobs.rb
@@ -1,28 +1,46 @@
module Oxidized
class Jobs < Array
- AVERAGE_DURATION = 5 # initially presume nodes take 5s to complete
+ AVERAGE_DURATION = 5 # initially presume nodes take 5s to complete
+ MAX_INTER_JOB_GAP = 300 # add job if more than X from last job started
attr_accessor :interval, :max, :want
+
def initialize max, interval, nodes
@max = max
@interval = interval
@nodes = nodes
+ @last = Time.now.utc
@durations = Array.new @nodes.size, AVERAGE_DURATION
duration AVERAGE_DURATION
super()
end
+
+ def push arg
+ @last = Time.now.utc
+ super
+ end
+
def duration last
@durations.push(last).shift
@duration = @durations.inject(:+).to_f / @nodes.size #rolling average
new_count
end
+
def new_count
@want = ((@nodes.size * @duration) / @interval).to_i
@want = 1 if @want < 1
@want = @nodes.size if @want > @nodes.size
@want = @max if @want > @max
end
- def add_job
- @want += 1
+
+ def work
+ # if a) we want less or same amount of threads as we now running
+ # and b) we want less threads running than the total amount of nodes
+ # and c) there is more than MAX_INTER_JOB_GAP since last one was started
+ # then we want one more thread (rationale is to fix hanging thread causing HOLB)
+ if @want <= size and @want < @nodes.size
+ @want +=1 if (Time.now.utc - @last) > MAX_INTER_JOB_GAP
+ end
end
+
end
end
diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb
index 99fc8b8..7ed70ac 100644
--- a/lib/oxidized/worker.rb
+++ b/lib/oxidized/worker.rb
@@ -2,12 +2,9 @@ module Oxidized
require 'oxidized/job'
require 'oxidized/jobs'
class Worker
- MAX_INTER_JOB_GAP = 300
-
def initialize nodes
@nodes = nodes
@jobs = Jobs.new CFG.threads, CFG.interval, @nodes
- @last = Time.now.utc
Thread.abort_on_exception = true
end
@@ -15,7 +12,7 @@ module Oxidized
ended = []
@jobs.delete_if { |job| ended << job if not job.alive? }
ended.each { |job| process job }
- @jobs.add_job if Time.now.utc - @last > MAX_INTER_JOB_GAP
+ @jobs.work
while @jobs.size < @jobs.want
Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}"
# ask for next node in queue non destructive way
@@ -26,7 +23,6 @@ module Oxidized
# shift nodes and get the next node
node = @nodes.get
node.running? ? next : node.running = true
- @last = Time.now.utc
@jobs.push Job.new node
end
end