summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFlorian Hibler <fhibler@users.noreply.github.com>2015-02-26 16:47:18 +0000
committerFlorian Hibler <fhibler@users.noreply.github.com>2015-02-26 16:47:18 +0000
commit1c85b3202033ed965abe16e15688f0b0025561ea (patch)
tree4ea9288b7a3dc774a60cdbd096761df4da147f8e
parent19de2ac51cd765e1c7b7fe731759fb50086675b5 (diff)
parentdb165b5e1ebb3091a49029d739b68a913bc75db6 (diff)
Merge pull request #1 from ytti/master
Merge to latest
-rw-r--r--CHANGELOG.md9
-rw-r--r--extra/nagios_check_failing_nodes.rb25
-rw-r--r--lib/oxidized/core.rb1
-rw-r--r--lib/oxidized/jobs.rb32
-rw-r--r--lib/oxidized/model/model.rb9
-rw-r--r--lib/oxidized/node.rb3
-rw-r--r--lib/oxidized/nodes.rb2
-rw-r--r--lib/oxidized/output/git.rb9
-rw-r--r--lib/oxidized/worker.rb6
9 files changed, 82 insertions, 14 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c1735c3..24b9cba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,12 @@
+# 0.4.0
+- FEATURE: allow setting IP address in addition to name in source (SQL/CSV)
+- FEATURE: approximate how long it takes to get node from larger view than 1
+- FEATURE: unconditionally start new job if too long has passed since previous start
+- BUGFIX: xos while using telnet (by @fhibler)
+- BUGFIX: ironware logout on some models (by @fhibler)
+- BUGFIX: allow node to be removed while it is being collected
+- BUGFIX: if model returns non string value, return empty string
+
# 0.3.0
- FEATURE: *FIXME* bunch of stuff I did for richih, docs needed
- FEATURE: ComWare model (by erJasp)
diff --git a/extra/nagios_check_failing_nodes.rb b/extra/nagios_check_failing_nodes.rb
new file mode 100644
index 0000000..1c81f66
--- /dev/null
+++ b/extra/nagios_check_failing_nodes.rb
@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+## contrib via https://github.com/ytti/oxidized/issues/67
+
+require 'open-uri'
+require 'json'
+
+critical = false
+critical_nodes = []
+
+json = JSON.load(open("http://localhost:8888/nodes.json"))
+json.each do |node|
+ if node['last']['status'] != 'success'
+ critical_nodes << node['name']
+ critical = true
+ end
+end
+
+if critical
+ puts 'Unable to backup: ' + critical_nodes.join(' ')
+ exit 2
+else
+ puts 'Backup of all nodes completed successfully.'
+ exit 0
+end
diff --git a/lib/oxidized/core.rb b/lib/oxidized/core.rb
index ba46f3a..4c67f4b 100644
--- a/lib/oxidized/core.rb
+++ b/lib/oxidized/core.rb
@@ -17,6 +17,7 @@ module Oxidized
Oxidized.mgr = Manager.new
nodes = Nodes.new
@worker = Worker.new nodes
+ trap 'HUP' { nodes.load }
if CFG.rest?
begin
require 'oxidized/web'
diff --git a/lib/oxidized/jobs.rb b/lib/oxidized/jobs.rb
index 6476744..ff7f92b 100644
--- a/lib/oxidized/jobs.rb
+++ b/lib/oxidized/jobs.rb
@@ -1,24 +1,46 @@
module Oxidized
class Jobs < Array
- attr_accessor :interval, :duration, :max, :want
+ AVERAGE_DURATION = 5 # initially presume nodes take 5s to complete
+ MAX_INTER_JOB_GAP = 300 # add job if more than X from last job started
+ attr_accessor :interval, :max, :want
+
def initialize max, interval, nodes
@max = max
- #@interval = interval * 60
@interval = interval
@nodes = nodes
- @duration = 4
- new_count
+ @last = Time.now.utc
+ @durations = Array.new @nodes.size, AVERAGE_DURATION
+ duration AVERAGE_DURATION
super()
end
+
+ def push arg
+ @last = Time.now.utc
+ super
+ end
+
def duration last
- @duration = (@duration + last) / 2
+ @durations.push(last).shift
+ @duration = @durations.inject(:+).to_f / @nodes.size #rolling average
new_count
end
+
def new_count
@want = ((@nodes.size * @duration) / @interval).to_i
@want = 1 if @want < 1
@want = @nodes.size if @want > @nodes.size
@want = @max if @want > @max
end
+
+ def work
+ # if a) we want less or same amount of threads as we now running
+ # and b) we want less threads running than the total amount of nodes
+ # and c) there is more than MAX_INTER_JOB_GAP since last one was started
+ # then we want one more thread (rationale is to fix hanging thread causing HOLB)
+ if @want <= size and @want < @nodes.size
+ @want +=1 if (Time.now.utc - @last) > MAX_INTER_JOB_GAP
+ end
+ end
+
end
end
diff --git a/lib/oxidized/model/model.rb b/lib/oxidized/model/model.rb
index d3f54b9..899b40a 100644
--- a/lib/oxidized/model/model.rb
+++ b/lib/oxidized/model/model.rb
@@ -133,10 +133,10 @@ module Oxidized
outputs << out
end
procs[:pre].each do |pre_proc|
- outputs.unshift Oxidized::String.new(instance_eval(&pre_proc))
+ outputs.unshift process_cmd_output(instance_eval(&pre_proc), '')
end
procs[:post].each do |post_proc|
- outputs << Oxidized::String.new(instance_eval(&post_proc))
+ outputs << process_cmd_output(instance_eval(&post_proc), '')
end
outputs
end
@@ -152,9 +152,8 @@ module Oxidized
private
def process_cmd_output output, name
- if output.class != Oxidized::String
- output = Oxidized::String.new output
- end
+ output = Oxidized::String.new output if ::String === output
+ output = Oxidized::String.new '' unless Oxidized::String === output
output.set_cmd(name)
output
end
diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb
index 6bc2b0f..2d41600 100644
--- a/lib/oxidized/node.rb
+++ b/lib/oxidized/node.rb
@@ -10,7 +10,8 @@ module Oxidized
alias :running? :running
def initialize opt
@name = opt[:name]
- @ip = Resolv.getaddress @name
+ @ip = IPAddr.new(opt[:ip]).to_s rescue nil
+ @ip ||= Resolv.getaddress @name
@group = opt[:group]
@input = resolve_input opt
@output = resolve_output opt
diff --git a/lib/oxidized/nodes.rb b/lib/oxidized/nodes.rb
index 032118d..3586b97 100644
--- a/lib/oxidized/nodes.rb
+++ b/lib/oxidized/nodes.rb
@@ -1,6 +1,6 @@
module Oxidized
- require 'oxidized/node'
require 'ipaddr'
+ require 'oxidized/node'
class Oxidized::NotSupported < OxidizedError; end
class Oxidized::NodeNotFound < OxidizedError; end
class Nodes < Array
diff --git a/lib/oxidized/output/git.rb b/lib/oxidized/output/git.rb
index 0c73638..d5eb8e7 100644
--- a/lib/oxidized/output/git.rb
+++ b/lib/oxidized/output/git.rb
@@ -1,5 +1,6 @@
module Oxidized
class Git < Output
+ class GitError < OxidizedError; end
begin
gem 'rugged', '~> 0.21.0'
require 'rugged'
@@ -71,8 +72,12 @@ class Git < Output
end
repo = Rugged::Repository.new repo
update_repo repo, file, data, @msg, @user, @email
- rescue Rugged::OSError, Rugged::RepositoryError
- Rugged::Repository.init_at repo, :bare
+ rescue Rugged::OSError, Rugged::RepositoryError => open_error
+ begin
+ Rugged::Repository.init_at repo, :bare
+ rescue => create_error
+ raise GitError, "first '#{open_error.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo"
+ end
retry
end
diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb
index e274e1e..7ed70ac 100644
--- a/lib/oxidized/worker.rb
+++ b/lib/oxidized/worker.rb
@@ -7,10 +7,12 @@ module Oxidized
@jobs = Jobs.new CFG.threads, CFG.interval, @nodes
Thread.abort_on_exception = true
end
+
def work
ended = []
@jobs.delete_if { |job| ended << job if not job.alive? }
ended.each { |job| process job }
+ @jobs.work
while @jobs.size < @jobs.want
Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}"
# ask for next node in queue non destructive way
@@ -24,6 +26,7 @@ module Oxidized
@jobs.push Job.new node
end
end
+
def process job
node = job.node
node.last = job
@@ -49,6 +52,9 @@ module Oxidized
end
Log.warn msg
end
+ rescue NodeNotFound
+ Log.warn "#{node.name} not found, removed while collecting?"
end
+
end
end