From 73b92861a46f1f1d3be560c12e0e9dd0eab00707 Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 11:02:24 +0300 Subject: enable SSH logging if input logging is true --- lib/oxidized/input/ssh.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index d321a11..0bc846f 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -45,9 +45,10 @@ module Oxidized ssh_opts[:proxy] = proxy end - ssh_opts[:keys] = vars(:ssh_keys).is_a?(Array) ? vars(:ssh_keys) : [vars(:ssh_keys)] if vars(:ssh_keys) - ssh_opts[:kex] = vars(:ssh_kex).split(/,\s*/) if vars(:ssh_kex) + ssh_opts[:keys] = [vars(:ssh_keys)].flatten if vars(:ssh_keys) + ssh_opts[:kex] = vars(:ssh_kex).split(/,\s*/) if vars(:ssh_kex) ssh_opts[:encryption] = vars(:ssh_encryption).split(/,\s*/) if vars(:ssh_encryption) + ssh_opts[:verbose] = Logger::DEBUG if Oxidized.config.input.debug? Oxidized.logger.debug "lib/oxidized/input/ssh.rb: Connecting to #{@node.name}" @ssh = Net::SSH.start(@node.ip, @node.auth[:username], ssh_opts) -- cgit v1.2.1 From ee2a575054e3d8f44b6ea92974a22ebb455dc770 Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 18:39:59 +0300 Subject: Remove resolve_repo smells --- lib/oxidized/node.rb | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb index 300221e..bc1a586 100644 --- a/lib/oxidized/node.rb +++ b/lib/oxidized/node.rb @@ -173,32 +173,18 @@ module Oxidized end def resolve_repo opt - if is_git? opt - remote_repo = Oxidized.config.output.git.repo - - if remote_repo.is_a?(::String) - if Oxidized.config.output.git.single_repo? || @group.nil? - remote_repo - else - File.join(File.dirname(remote_repo), @group + '.git') - end - else - remote_repo[@group] - end - elsif is_gitcrypt? opt - remote_repo = Oxidized.config.output.gitcrypt.repo - - if remote_repo.is_a?(::String) - if Oxidized.config.output.gitcrypt.single_repo? || @group.nil? - remote_repo - else - File.join(File.dirname(remote_repo), @group + '.git') - end + type = git_type opt + return nil unless type + + remote_repo = Oxidized.config.output.send(type).repo + if remote_repo.is_a?(::String) + if Oxidized.config.output.send(type).single_repo? || @group.nil? + remote_repo else - remote_repo[@group] + File.join(File.dirname(remote_repo), @group + '.git') end else - return + remote_repo[@group] end end @@ -237,12 +223,11 @@ module Oxidized value end - def is_git? opt - (opt[:output] || Oxidized.config.output.default) == 'git' + def git_type + type = opt[:output] || Oxidized.config.output.default + return nil unless type[0..2] == "git" + type end - def is_gitcrypt? opt - (opt[:output] || Oxidized.config.output.default) == 'gitcrypt' - end end end -- cgit v1.2.1 From c0cb572d98708716dc3241e0d73f91143f887b65 Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 18:45:59 +0300 Subject: remove SSH::connect smell --- lib/oxidized/input/ssh.rb | 59 +++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 28 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index 0bc846f..ef92ef6 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -21,37 +21,10 @@ module Oxidized @output = '' @pty_options = { term: "vt100" } @node.model.cfg['ssh'].each { |cb| instance_exec(&cb) } - secure = Oxidized.config.input.ssh.secure @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-ssh", 'w') if Oxidized.config.input.debug? - port = vars(:ssh_port) || 22 - - ssh_opts = { - port: port.to_i, - paranoid: secure, - keepalive: true, - password: @node.auth[:password], :timeout => Oxidized.config.timeout, - number_of_password_prompts: 0, - } - - auth_methods = vars(:auth_methods) || %w(none publickey password) - ssh_opts[:auth_methods] = auth_methods - Oxidized.logger.debug "AUTH METHODS::#{auth_methods}" - - if proxy_host = vars(:ssh_proxy) - proxy_command = "ssh " - proxy_command += "-o StrictHostKeyChecking=no " unless secure - proxy_command += "#{proxy_host} -W %h:%p" - proxy = Net::SSH::Proxy::Command.new(proxy_command) - ssh_opts[:proxy] = proxy - end - - ssh_opts[:keys] = [vars(:ssh_keys)].flatten if vars(:ssh_keys) - ssh_opts[:kex] = vars(:ssh_kex).split(/,\s*/) if vars(:ssh_kex) - ssh_opts[:encryption] = vars(:ssh_encryption).split(/,\s*/) if vars(:ssh_encryption) - ssh_opts[:verbose] = Logger::DEBUG if Oxidized.config.input.debug? Oxidized.logger.debug "lib/oxidized/input/ssh.rb: Connecting to #{@node.name}" - @ssh = Net::SSH.start(@node.ip, @node.auth[:username], ssh_opts) + @ssh = Net::SSH.start(@node.ip, @node.auth[:username], get_ssh_opts) unless @exec shell_open @ssh begin @@ -143,5 +116,35 @@ module Oxidized end end end + + def get_ssh_opts + port = vars(:ssh_port) || 22 + ssh_opts = { + port: port.to_i, + paranoid: secure, + keepalive: true, + password: @node.auth[:password], timeout: Oxidized.config.timeout, + number_of_password_prompts: 0 + } + + auth_methods = vars(:auth_methods) || %w(none publickey password) + ssh_opts[:auth_methods] = auth_methods + Oxidized.logger.debug "AUTH METHODS::#{auth_methods}" + + if proxy_host = vars(:ssh_proxy) + proxy_command = "ssh " + proxy_command += "-o StrictHostKeyChecking=no " unless Oxidized.config.input.ssh.secure? + proxy_command += "#{proxy_host} -W %h:%p" + proxy = Net::SSH::Proxy::Command.new(proxy_command) + ssh_opts[:proxy] = proxy + end + + ssh_opts[:keys] = [vars(:ssh_keys)].flatten if vars(:ssh_keys) + ssh_opts[:kex] = vars(:ssh_kex).split(/,\s*/) if vars(:ssh_kex) + ssh_opts[:encryption] = vars(:ssh_encryption).split(/,\s*/) if vars(:ssh_encryption) + ssh_opts[:verbose] = Logger::DEBUG if Oxidized.config.input.debug? + + ssh_opts + end end end -- cgit v1.2.1 From 483a9901280ea436db34f61ef220cc2f4d90b0da Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 18:49:46 +0300 Subject: remove source/http load smells --- lib/oxidized/source/http.rb | 46 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index bf1e74c..36fa764 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -17,28 +17,7 @@ module Oxidized def load node_want = nil nodes = [] - uri = URI.parse(@cfg.url) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true if uri.scheme == 'https' - http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure - - # map headers - headers = {} - @cfg.headers.each do |header, value| - headers[header] = value - end - - req_uri = uri.request_uri - if node_want - req_uri = "#{req_uri}/#{node_want}" - end - request = Net::HTTP::Get.new(req_uri, headers) - if (@cfg.user? && @cfg.pass?) - request.basic_auth(@cfg.user, @cfg.pass) - end - - response = http.request(request) - data = JSON.parse(response.body) + data = JSON.parse(get_http) data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? data.each do |node| next if node.empty? @@ -72,5 +51,28 @@ module Oxidized end object end + + def get_http + uri = URI.parse(@cfg.url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true if uri.scheme == 'https' + http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure + + # map headers + headers = {} + @cfg.headers.each do |header, value| + headers[header] = value + end + + req_uri = uri.request_uri + if node_want + req_uri = "#{req_uri}/#{node_want}" + end + request = Net::HTTP::Get.new(req_uri, headers) + if (@cfg.user? && @cfg.pass?) + request.basic_auth(@cfg.user, @cfg.pass) + end + http.request(request).body + end end end -- cgit v1.2.1 From 31b863f2eeff3ccbec8d0c0c35caf2c0e1d3e0ef Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 19:05:16 +0300 Subject: remove worker smell --- lib/oxidized/worker.rb | 76 ++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 34 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb index 74b046d..06e8689 100644 --- a/lib/oxidized/worker.rb +++ b/lib/oxidized/worker.rb @@ -36,48 +36,17 @@ module Oxidized Oxidized.logger.debug("lib/oxidized/worker.rb: #{@jobs.size} jobs running in parallel") unless @jobs.empty? end + def process job node = job.node node.last = job node.stats.add job @jobs.duration job.time node.running = false - if job.status == :success - @jobs_done += 1 # needed for :nodes_done hook - Oxidized.Hooks.handle :node_success, :node => node, - :job => job - msg = "update #{node.name}" - msg += " from #{node.from}" if node.from - msg += " with message '#{node.msg}'" if node.msg - output = node.output.new - if output.store node.name, job.config, - :msg => msg, :email => node.email, :user => node.user, :group => node.group - node.modified - Oxidized.logger.info "Configuration updated for #{node.group}/#{node.name}" - Oxidized.Hooks.handle :post_store, :node => node, - :job => job, - :commitref => output.commitref - end - node.reset + process_success node, job else - msg = "#{node.name} status #{job.status}" - if node.retry < Oxidized.config.retries - node.retry += 1 - msg += ", retry attempt #{node.retry}" - @nodes.next node.name - else - # Only increment the @jobs_done when we give up retries for a node (or success). - # As it would otherwise cause @jobs_done to be incremented with generic retries. - # This would cause :nodes_done hook to desync from running at the end of the nodelist and - # be fired when the @jobs_done > @nodes.count (could be mid-cycle on the next cycle). - @jobs_done += 1 - msg += ", retries exhausted, giving up" - node.retry = 0 - Oxidized.Hooks.handle :node_fail, :node => node, - :job => job - end - Oxidized.logger.warn msg + process_failure node, job end rescue NodeNotFound Oxidized.logger.warn "#{node.name} not found, removed while collecting?" @@ -85,6 +54,45 @@ module Oxidized private + def process_success node, job + @jobs_done += 1 # needed for :nodes_done hook + Oxidized.Hooks.handle :node_success, node: node, + job: job + msg = "update #{node.name}" + msg += " from #{node.from}" if node.from + msg += " with message '#{node.msg}'" if node.msg + output = node.output.new + if output.store node.name, job.config, + msg: msg, email: node.email, user: node.user, group: node.group + node.modified + Oxidized.logger.info "Configuration updated for #{node.group}/#{node.name}" + Oxidized.Hooks.handle :post_store, node: node, + job: job, + commitref: output.commitref + end + node.reset + end + + def process_failure node, job + msg = "#{node.name} status #{job.status}" + if node.retry < Oxidized.config.retries + node.retry += 1 + msg += ", retry attempt #{node.retry}" + @nodes.next node.name + else + # Only increment the @jobs_done when we give up retries for a node (or success). + # As it would otherwise cause @jobs_done to be incremented with generic retries. + # This would cause :nodes_done hook to desync from running at the end of the nodelist and + # be fired when the @jobs_done > @nodes.count (could be mid-cycle on the next cycle). + @jobs_done += 1 + msg += ", retries exhausted, giving up" + node.retry = 0 + Oxidized.Hooks.handle :node_fail, :node => node, + :job => job + end + Oxidized.logger.warn msg + end + def is_cycle_finished? if @jobs_done > @nodes.count true -- cgit v1.2.1 From 2f3cf7647742ce80b428d5e0aad75b45e0510bdc Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 19:08:50 +0300 Subject: remove csv smell --- lib/oxidized/source/csv.rb | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/source/csv.rb b/lib/oxidized/source/csv.rb index 7b771f6..769b1f7 100644 --- a/lib/oxidized/source/csv.rb +++ b/lib/oxidized/source/csv.rb @@ -20,14 +20,7 @@ module Oxidized def load _node_want = nil nodes = [] - file = File.expand_path(@cfg.file) - file = if @cfg.gpg? - crypto = GPGME::Crypto.new password: @cfg.gpg_password - file = crypto.decrypt(File.open(file)).to_s - else - open(file) - end - file.each_line do |line| + get_file.each_line do |line| next if line.match(/^\s*#/) data = line.chomp.split(@cfg.delimiter, -1) next if data.empty? @@ -49,5 +42,17 @@ module Oxidized end nodes end + + private + + def get_file + file = File.expand_path(@cfg.file) + if @cfg.gpg? + crypto = GPGME::Crypto.new password: @cfg.gpg_password + crypto.decrypt(File.open(file)).to_s + else + open(file) + end + end end end -- cgit v1.2.1 From c150be119d4869c74b790b6f7d920a7c8f72ae66 Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 19:16:21 +0300 Subject: reduce http output smell --- lib/oxidized/output/http.rb | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/output/http.rb b/lib/oxidized/output/http.rb index 0467261..756b9ad 100644 --- a/lib/oxidized/output/http.rb +++ b/lib/oxidized/output/http.rb @@ -14,28 +14,19 @@ module Oxidized raise NoConfig, 'no output http config, edit ~/.config/oxidized/config' end end + require "net/http" require "uri" require "json" + def store node, outputs, opt = {} @commitref = nil - json = JSON.pretty_generate( - { - 'msg' => opt[:msg], - 'user' => opt[:user], - 'email' => opt[:email], - 'group' => opt[:group], - 'node' => node, - 'config' => outputs.to_cfg, - # actually we need to also iterate outputs, for other types like in gitlab. But most people don't use 'type' functionality. - } - ) uri = URI.parse @cfg.url http = Net::HTTP.new uri.host, uri.port # http.use_ssl = true if uri.scheme = 'https' req = Net::HTTP::Post.new(uri.request_uri, initheader = { 'Content-Type' => 'application/json' }) req.basic_auth @cfg.user, @cfg.password - req.body = json + req.body = generate_json(node, outputs, opt) response = http.request req case response.code.to_i @@ -50,5 +41,21 @@ module Oxidized Oxidized.logger.info "Configuration http backup for #{node} failed status: #{response.body}" end end + + private + + def generate_json node, outputs, opt + JSON.pretty_generate( + { + 'msg' => opt[:msg], + 'user' => opt[:user], + 'email' => opt[:email], + 'group' => opt[:group], + 'node' => node, + 'config' => outputs.to_cfg, + # actually we need to also iterate outputs, for other types like in gitlab. But most people don't use 'type' functionality. + } + ) + end end end -- cgit v1.2.1 From 7a99cd83da26e21921311011fea63dacfd2999ae Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 19:21:00 +0300 Subject: ehh what can you do --- lib/oxidized/input/ssh.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index ef92ef6..82335f9 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -118,9 +118,8 @@ module Oxidized end def get_ssh_opts - port = vars(:ssh_port) || 22 ssh_opts = { - port: port.to_i, + port: (vars(:ssh_port) || 22).to_i, paranoid: secure, keepalive: true, password: @node.auth[:password], timeout: Oxidized.config.timeout, -- cgit v1.2.1 From 04ffb2c68ac7f069bd9f33f39c77095c979aaec1 Mon Sep 17 00:00:00 2001 From: Saku Ytti Date: Fri, 22 Jun 2018 19:31:12 +0300 Subject: rubycop fixes --- lib/oxidized/input/ssh.rb | 6 +++--- lib/oxidized/node.rb | 1 - lib/oxidized/source/csv.rb | 4 ++-- lib/oxidized/source/http.rb | 4 ++-- lib/oxidized/worker.rb | 1 - 5 files changed, 7 insertions(+), 9 deletions(-) (limited to 'lib/oxidized') diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index 82335f9..01f7063 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -24,7 +24,7 @@ module Oxidized @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-ssh", 'w') if Oxidized.config.input.debug? Oxidized.logger.debug "lib/oxidized/input/ssh.rb: Connecting to #{@node.name}" - @ssh = Net::SSH.start(@node.ip, @node.auth[:username], get_ssh_opts) + @ssh = Net::SSH.start(@node.ip, @node.auth[:username], make_ssh_opts) unless @exec shell_open @ssh begin @@ -117,13 +117,13 @@ module Oxidized end end - def get_ssh_opts + def make_ssh_opts ssh_opts = { port: (vars(:ssh_port) || 22).to_i, paranoid: secure, keepalive: true, password: @node.auth[:password], timeout: Oxidized.config.timeout, - number_of_password_prompts: 0 + number_of_password_prompts: 0 } auth_methods = vars(:auth_methods) || %w(none publickey password) diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb index bc1a586..c3a6885 100644 --- a/lib/oxidized/node.rb +++ b/lib/oxidized/node.rb @@ -228,6 +228,5 @@ module Oxidized return nil unless type[0..2] == "git" type end - end end diff --git a/lib/oxidized/source/csv.rb b/lib/oxidized/source/csv.rb index 769b1f7..5b43352 100644 --- a/lib/oxidized/source/csv.rb +++ b/lib/oxidized/source/csv.rb @@ -20,7 +20,7 @@ module Oxidized def load _node_want = nil nodes = [] - get_file.each_line do |line| + open_file.each_line do |line| next if line.match(/^\s*#/) data = line.chomp.split(@cfg.delimiter, -1) next if data.empty? @@ -45,7 +45,7 @@ module Oxidized private - def get_file + def open_file file = File.expand_path(@cfg.file) if @cfg.gpg? crypto = GPGME::Crypto.new password: @cfg.gpg_password diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 36fa764..26c9e05 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -17,7 +17,7 @@ module Oxidized def load node_want = nil nodes = [] - data = JSON.parse(get_http) + data = JSON.parse(read_http(node_want)) data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location? data.each do |node| next if node.empty? @@ -52,7 +52,7 @@ module Oxidized object end - def get_http + def read_http node_want uri = URI.parse(@cfg.url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = true if uri.scheme == 'https' diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb index 06e8689..4b9614b 100644 --- a/lib/oxidized/worker.rb +++ b/lib/oxidized/worker.rb @@ -36,7 +36,6 @@ module Oxidized Oxidized.logger.debug("lib/oxidized/worker.rb: #{@jobs.size} jobs running in parallel") unless @jobs.empty? end - def process job node = job.node node.last = job -- cgit v1.2.1