From f2da63369fcb754e3715091cc4fd2f93db42106e Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Tue, 27 Oct 2015 14:49:52 -0400 Subject: better test framework, more specs and Travis CI --- .travis.yml | 3 ++ Gemfile.lock | 41 +++++++++++++++++++++ README.md | 2 +- Rakefile | 49 ++++--------------------- bin/console | 9 +++++ lib/oxidized.rb | 15 ++++++-- lib/oxidized/cli.rb | 2 +- lib/oxidized/config.rb | 83 +++++++++++++++++++++++-------------------- lib/oxidized/config/vars.rb | 4 +-- lib/oxidized/core.rb | 7 ++-- lib/oxidized/hook.rb | 2 +- lib/oxidized/input/cli.rb | 3 +- lib/oxidized/input/ftp.rb | 4 +-- lib/oxidized/input/ssh.rb | 18 +++++----- lib/oxidized/input/telnet.rb | 4 +-- lib/oxidized/job.rb | 6 ++-- lib/oxidized/model/junos.rb | 14 ++++---- lib/oxidized/model/model.rb | 3 +- lib/oxidized/node.rb | 22 +++++------- lib/oxidized/nodes.rb | 2 +- lib/oxidized/output/file.rb | 6 ++-- lib/oxidized/output/git.rb | 10 +++--- lib/oxidized/source/csv.rb | 14 ++++---- lib/oxidized/source/http.rb | 2 +- lib/oxidized/source/source.rb | 2 +- lib/oxidized/source/sql.rb | 14 ++++---- lib/oxidized/string.rb | 2 +- lib/oxidized/version.rb | 3 ++ lib/oxidized/worker.rb | 6 ++-- oxidized.gemspec | 13 +++++-- spec/node_spec.rb | 44 +++++++++++++++++++++++ spec/nodes_spec.rb | 35 +++++++++++------- spec/spec_helper.rb | 14 ++++++++ 33 files changed, 285 insertions(+), 173 deletions(-) create mode 100644 .travis.yml create mode 100644 Gemfile.lock create mode 100755 bin/console create mode 100644 lib/oxidized/version.rb create mode 100644 spec/node_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3734051 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: ruby +rvm: + - 2.1.6 diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..bbdc4c8 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,41 @@ +PATH + remote: . + specs: + oxidized (0.8.1) + asetus (~> 0.1) + net-ssh (~> 2.8) + rugged (~> 0.21, >= 0.21.4) + slop (~> 3.5) + +GEM + remote: https://rubygems.org/ + specs: + asetus (0.3.0) + coderay (1.1.0) + metaclass (0.0.4) + method_source (0.8.2) + minitest (5.8.2) + mocha (1.1.0) + metaclass (~> 0.0.1) + net-ssh (2.9.2) + pry (0.10.3) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) + rake (10.4.2) + rugged (0.23.3) + slop (3.6.0) + +PLATFORMS + ruby + +DEPENDENCIES + bundler (~> 1.10) + minitest (~> 5.8) + mocha (~> 1.1) + oxidized! + pry (~> 0) + rake (~> 10.0) + +BUNDLED WITH + 1.10.6 diff --git a/README.md b/README.md index 97aa4ac..aea4326 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Oxidized +# Oxidized [![Build Status](https://travis-ci.org/Shopify/oxidized.svg)](https://travis-ci.org/Shopify/oxidized) [![Gem Version](https://badge.fury.io/rb/oxidized.svg)](http://badge.fury.io/rb/oxidized) diff --git a/Rakefile b/Rakefile index 3d85377..2dc5415 100644 --- a/Rakefile +++ b/Rakefile @@ -1,51 +1,14 @@ -begin - require 'rake/testtask' - require 'bundler' - # Bundler.setup -rescue LoadError - warn 'bundler missing' -end - -gemspec = eval(File.read(Dir['*.gemspec'].first)) -file = [gemspec.name, gemspec.version].join('-') + '.gem' - -desc 'Validate gemspec' -task :gemspec do - gemspec.validate -end +require 'bundler/gem_tasks' +require 'rake/testtask' desc 'Run minitest' task :test do Rake::TestTask.new do |t| - t.libs.push "lib" - t.test_files = FileList['spec/*_spec.rb'] + t.libs << 'spec' + t.test_files = FileList['spec/**/*_spec.rb'] + t.warning = true t.verbose = true end end -desc 'Build gem' -task :build do - system "gem build #{gemspec.name}.gemspec" - FileUtils.mkdir_p 'gems' - FileUtils.mv file, 'gems' -end - -desc 'Install gem' -task :install => :build do - system "sudo -Es sh -c \'umask 022; gem install gems/#{file}\'" -end - -desc 'Remove gems' -task :clean do - FileUtils.rm_rf 'gems' -end - -desc 'Tag the release' -task :tag do - system "git tag #{gemspec.version}" -end - -desc 'Push to rubygems' -task :push => :tag do - system "gem push gems/#{file}" -end +task default: :test diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..0419027 --- /dev/null +++ b/bin/console @@ -0,0 +1,9 @@ +#!/usr/bin/env ruby + +lib = File.expand_path('../../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) + +require 'oxidized' +require 'pry' + +Pry.start diff --git a/lib/oxidized.rb b/lib/oxidized.rb index 61948ff..df30602 100644 --- a/lib/oxidized.rb +++ b/lib/oxidized.rb @@ -1,6 +1,17 @@ module Oxidized class OxidizedError < StandardError; end - Encoding.default_external = 'UTF-8' - Directory = File.expand_path File.join File.dirname(__FILE__), '../' + Directory = File.expand_path File.join File.dirname(__FILE__), '../' require 'oxidized/core' + + def self.asetus + @@asetus + end + + def self.asetus=(val) + @@asetus = val + end + + def self.config + asetus.cfg + end end diff --git a/lib/oxidized/cli.rb b/lib/oxidized/cli.rb index c66ec8d..00ea4b6 100644 --- a/lib/oxidized/cli.rb +++ b/lib/oxidized/cli.rb @@ -18,7 +18,7 @@ module Oxidized def initialize Log.info "Oxidized starting, running as pid #{$$}" _args, @opts = parse_opts - CFG.debug = true if @opts[:debug] + Oxidized.config.debug = true if @opts[:debug] end def crash error diff --git a/lib/oxidized/config.rb b/lib/oxidized/config.rb index f45004a..36c1a8f 100644 --- a/lib/oxidized/config.rb +++ b/lib/oxidized/config.rb @@ -11,47 +11,52 @@ module Oxidized SourceDir = File.join Directory, %w(lib oxidized source) HookDir = File.join Directory, %w(lib oxidized hook) Sleep = 1 + + def self.load + asetus = Asetus.new(name: 'oxidized', load: false, key_to_s: true) + Oxidized.asetus = asetus + + asetus.default.username = 'username' + asetus.default.password = 'password' + asetus.default.model = 'junos' + asetus.default.interval = 3600 + asetus.default.log = File.join Config::Root, 'log' + asetus.default.debug = false + asetus.default.threads = 30 + asetus.default.timeout = 20 + asetus.default.retries = 3 + asetus.default.prompt = /^([\w.@-]+[#>]\s?)$/ + asetus.default.rest = '127.0.0.1:8888' # or false to disable + asetus.default.vars = {} # could be 'enable'=>'enablePW' + asetus.default.groups = {} # group level configuration + + asetus.default.input.default = 'ssh, telnet' + asetus.default.input.debug = false # or String for session log file + asetus.default.input.ssh.secure = false # complain about changed certs + + asetus.default.output.default = 'file' # file, git + asetus.default.source.default = 'csv' # csv, sql + + asetus.default.model_map = { + 'cisco' => 'ios', + 'juniper' => 'junos', + } + + begin + asetus.load # load system+user configs, merge to Config.cfg + rescue => error + raise InvalidConfig, "Error loading config: #{error.message}" + end + + Log.level = Logger::INFO unless asetus.cfg.debug + raise NoConfig, 'edit ~/.config/oxidized/config' if asetus.create + Log.file = asetus.cfg.log if asetus.cfg.log + + asetus + end end + class << self attr_accessor :mgr, :Hooks end - CFGS = Asetus.new :name=>'oxidized', :load=>false, :key_to_s=>true - CFGS.default.username = 'username' - CFGS.default.password = 'password' - CFGS.default.model = 'junos' - CFGS.default.interval = 3600 - CFGS.default.log = File.join Config::Root, 'log' - CFGS.default.debug = false - CFGS.default.threads = 30 - CFGS.default.timeout = 20 - CFGS.default.retries = 3 - CFGS.default.prompt = /^([\w.@-]+[#>]\s?)$/ - CFGS.default.rest = '127.0.0.1:8888' # or false to disable - CFGS.default.vars = {} # could be 'enable'=>'enablePW' - CFGS.default.groups = {} # group level configuration - - CFGS.default.input.default = 'ssh, telnet' - CFGS.default.input.debug = false # or String for session log file - CFGS.default.input.ssh.secure = false # complain about changed certs - - CFGS.default.output.default = 'file' # file, git - CFGS.default.source.default = 'csv' # csv, sql - - CFGS.default.model_map = { - 'cisco' => 'ios', - 'juniper' => 'junos', - } - - begin - CFGS.load # load system+user configs, merge to Config.cfg - rescue => error - raise InvalidConfig, "Error loading config: #{error.message}" - ensure - CFG = CFGS.cfg # convenienence, instead of Config.cfg.password, CFG.password - end - - Log.level = Logger::INFO unless CFG.debug - raise NoConfig, 'edit ~/.config/oxidized/config' if CFGS.create - Log.file = CFG.log if CFG.log - end diff --git a/lib/oxidized/config/vars.rb b/lib/oxidized/config/vars.rb index b39e7fe..10f39c3 100644 --- a/lib/oxidized/config/vars.rb +++ b/lib/oxidized/config/vars.rb @@ -3,8 +3,8 @@ module Oxidized::Config::Vars # nil values will be ignored def vars name r = @node.vars[name] unless @node.vars.nil? - r ||= Oxidized::CFG.groups[@node.group].vars[name.to_s] if Oxidized::CFG.groups.has_key?(@node.group) - r ||= Oxidized::CFG.vars[name.to_s] if Oxidized::CFG.vars.has_key?(name.to_s) + r ||= Oxidized.config.groups[@node.group].vars[name.to_s] if Oxidized.config.groups.has_key?(@node.group) + r ||= Oxidized.config.vars[name.to_s] if Oxidized.config.vars.has_key?(name.to_s) r end end diff --git a/lib/oxidized/core.rb b/lib/oxidized/core.rb index 6e7a352..1b918d1 100644 --- a/lib/oxidized/core.rb +++ b/lib/oxidized/core.rb @@ -17,19 +17,20 @@ module Oxidized class NoNodesFound < OxidizedError; end def initialize args + Config.load Oxidized.mgr = Manager.new - Oxidized.Hooks = HookManager.from_config CFG + Oxidized.Hooks = HookManager.from_config(Oxidized.config) nodes = Nodes.new raise NoNodesFound, 'source returns no usable nodes' if nodes.size == 0 @worker = Worker.new nodes trap('HUP') { nodes.load } - if CFG.rest? + if Oxidized.config.rest? begin require 'oxidized/web' rescue LoadError raise OxidizedError, 'oxidized-web not found: sudo gem install oxidized-web - or disable web support by setting "rest: false" in your configuration' end - @rest = API::Web.new nodes, CFG.rest + @rest = API::Web.new nodes, Oxidized.config.rest @rest.run end run diff --git a/lib/oxidized/hook.rb b/lib/oxidized/hook.rb index 7f1942b..d774653 100644 --- a/lib/oxidized/hook.rb +++ b/lib/oxidized/hook.rb @@ -66,7 +66,7 @@ end # Hook abstract base class class Hook - attr_accessor :cfg + attr_reader :cfg def initialize end diff --git a/lib/oxidized/input/cli.rb b/lib/oxidized/input/cli.rb index 22d188c..731b459 100644 --- a/lib/oxidized/input/cli.rb +++ b/lib/oxidized/input/cli.rb @@ -1,6 +1,7 @@ module Oxidized class Input module CLI + attr_reader :node def initialize @post_login = [] @@ -10,7 +11,7 @@ module Oxidized def get connect_cli - d = @node.model.get + d = node.model.get disconnect d end diff --git a/lib/oxidized/input/ftp.rb b/lib/oxidized/input/ftp.rb index 70db60c..a3d7ac8 100644 --- a/lib/oxidized/input/ftp.rb +++ b/lib/oxidized/input/ftp.rb @@ -18,7 +18,7 @@ module Oxidized def connect node @node = node @node.model.cfg['ftp'].each { |cb| instance_exec(&cb) } - @log = File.open(Oxidized::Config::Crash + "-#{@node.ip}-ftp", 'w') if CFG.input.debug? + @log = File.open(Oxidized::Config::Crash + "-#{@node.ip}-ftp", 'w') if Oxidized.config.input.debug? @ftp = Net::FTP.new @node.ip, @node.auth[:username], @node.auth[:password] connected? end @@ -47,7 +47,7 @@ module Oxidized @ftp.close #rescue Errno::ECONNRESET, IOError ensure - @log.close if CFG.input.debug? + @log.close if Oxidized.config.input.debug? end end diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index 21fb02c..e45adb6 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -19,11 +19,11 @@ module Oxidized @node = node @output = '' @node.model.cfg['ssh'].each { |cb| instance_exec(&cb) } - secure = CFG.input.ssh.secure - @log = File.open(Oxidized::Config::Crash + "-#{@node.ip}-ssh", 'w') if CFG.input.debug? + secure = Oxidized.config.input.ssh.secure + @log = File.open(Oxidized::Config::Crash + "-#{@node.ip}-ssh", 'w') if Oxidized.config.input.debug? port = vars(:ssh_port) || 22 @ssh = Net::SSH.start @node.ip, @node.auth[:username], :port => port.to_i, - :password => @node.auth[:password], :timeout => CFG.timeout, + :password => @node.auth[:password], :timeout => Oxidized.config.timeout, :paranoid => secure, :auth_methods => %w(none publickey password keyboard-interactive), :number_of_password_prompts => 0 @@ -42,8 +42,8 @@ module Oxidized @ssh and not @ssh.closed? end - def cmd cmd, expect=@node.prompt - Log.debug "SSH: #{cmd} @ #{@node.name}" + def cmd cmd, expect=node.prompt + Log.debug "SSH: #{cmd} @ #{node.name}" if @exec @ssh.exec! cmd else @@ -64,17 +64,17 @@ module Oxidized def disconnect disconnect_cli # if disconnect does not disconnect us, give up after timeout - Timeout::timeout(CFG.timeout) { @ssh.loop } + Timeout::timeout(Oxidized.config.timeout) { @ssh.loop } rescue Errno::ECONNRESET, Net::SSH::Disconnect, IOError ensure - @log.close if CFG.input.debug? + @log.close if Oxidized.config.input.debug? (@ssh.close rescue true) unless @ssh.closed? end def shell_open ssh @ses = ssh.open_channel do |ch| ch.on_data do |_ch, data| - @log.print data if CFG.input.debug? + @log.print data if Oxidized.config.input.debug? @output << data @output = @node.model.expects @output end @@ -109,7 +109,7 @@ module Oxidized end def expect regexp - Timeout::timeout(CFG.timeout) do + Timeout::timeout(Oxidized.config.timeout) do @ssh.loop(0.1) do sleep 0.1 not @output.match regexp diff --git a/lib/oxidized/input/telnet.rb b/lib/oxidized/input/telnet.rb index bf0140c..b92d703 100644 --- a/lib/oxidized/input/telnet.rb +++ b/lib/oxidized/input/telnet.rb @@ -8,7 +8,7 @@ module Oxidized def connect node @node = node - @timeout = CFG.timeout + @timeout = Oxidized.config.timeout @node.model.cfg['telnet'].each { |cb| instance_exec(&cb) } port = vars(:telnet_port) || 23 @@ -16,7 +16,7 @@ module Oxidized 'Port' => port.to_i, 'Timeout' => @timeout, 'Model' => @node.model } - opt['Output_log'] = Oxidized::Config::Crash + "-#{@node.ip}-telnet" if CFG.input.debug? + opt['Output_log'] = Oxidized::Config::Crash + "-#{@node.ip}-telnet" if Oxidized.config.input.debug? @telnet = Net::Telnet.new opt if @node.auth[:username] and @node.auth[:username].length > 0 diff --git a/lib/oxidized/job.rb b/lib/oxidized/job.rb index 6fb60a8..0a4a24b 100644 --- a/lib/oxidized/job.rb +++ b/lib/oxidized/job.rb @@ -1,11 +1,11 @@ module Oxidized class Job < Thread attr_reader :start, :end, :status, :time, :node, :config - def initialize node + def initialize(node) @node = node @start = Time.now.utc - super do |node| - @status, @config = node.run + super do + @status, @config = @node.run @end = Time.now.utc @time = @end - @start end diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index 0e921d2..da3af72 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -3,7 +3,7 @@ class JunOS < Oxidized::Model comment '# ' def telnet - @input.class.to_s.match /Telnet/ + @input.class.to_s.match(/Telnet/) end cmd :all do |cfg| @@ -12,16 +12,16 @@ class JunOS < Oxidized::Model cfg.lines.map { |line| line.rstrip }.join("\n") + "\n" end - cmd :secret do |cfg| - cfg.gsub! /encrypted-password (\S+).*/, '' - cfg.gsub! /community (\S+) {/, 'community {' + cmd :secret do |cfg| + cfg.gsub!(/encrypted-password (\S+).*/, '') + cfg.gsub!(/community (\S+) {/, 'community {') cfg end cmd 'show configuration | display omit' cmd 'show version' do |cfg| - @model = $1 if cfg.match /^Model: (\S+)/ + @model = $1 if cfg.match(/^Model: (\S+)/) comment cfg end @@ -39,8 +39,8 @@ class JunOS < Oxidized::Model end cfg :telnet do - username /^login:/ - password /^Password:/ + username(/^login:/) + password(/^Password:/) end cfg :ssh do diff --git a/lib/oxidized/model/model.rb b/lib/oxidized/model/model.rb index e0e3941..c95cc67 100644 --- a/lib/oxidized/model/model.rb +++ b/lib/oxidized/model/model.rb @@ -11,7 +11,8 @@ module Oxidized klass.instance_variable_set '@cfg', Hash.new { |h,k| h[k] = [] } klass.instance_variable_set '@procs', Hash.new { |h,k| h[k] = [] } klass.instance_variable_set '@expect', [] - klass.const_set :CFG, CFG + klass.instance_variable_set '@comment', nil + klass.instance_variable_set '@prompt', nil end def comment _comment='# ' return @comment if @comment diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb index d50317f..1bbf6dc 100644 --- a/lib/oxidized/node.rb +++ b/lib/oxidized/node.rb @@ -54,9 +54,7 @@ module Oxidized end end begin - if input.connect self - input.get - end + input.connect(self) and input.get rescue *rescue_fail.keys => err resc = '' if not level = rescue_fail[err.class] @@ -122,19 +120,17 @@ module Oxidized private def resolve_prompt opt - prompt = opt[:prompt] - prompt ||= @model.prompt - prompt ||= CFG.prompt + opt[:prompt] || @model.prompt || Oxidized.config.prompt end def resolve_auth opt # Resolve configured username/password, give priority to group level configuration # TODO: refactor to use revised behaviour of Asetus cfg_username, cfg_password = - if CFG.groups.has_key?(@group) and ['username', 'password'].all? {|e| CFG.groups[@group].has_key?(e)} - [CFG.groups[@group].username, CFG.groups[@group].password] - elsif ['username', 'password'].all? {|e| CFG.has_key?(e)} - [CFG.username, CFG.password] + if Oxidized.config.groups.has_key?(@group) and ['username', 'password'].all? {|e| Oxidized.config.groups[@group].has_key?(e)} + [Oxidized.config.groups[@group].username, Oxidized.config.groups[@group].password] + elsif ['username', 'password'].all? {|e| Oxidized.config.has_key?(e)} + [Oxidized.config.username, Oxidized.config.password] else [nil, nil] end @@ -145,7 +141,7 @@ module Oxidized end def resolve_input opt - inputs = (opt[:input] or CFG.input.default) + inputs = (opt[:input] or Oxidized.config.input.default) inputs.split(/\s*,\s*/).map do |input| if not Oxidized.mgr.input[input] Oxidized.mgr.add_input input or raise MethodNotFound, "#{input} not found for node #{ip}" @@ -155,7 +151,7 @@ module Oxidized end def resolve_output opt - output = (opt[:output] or CFG.output.default) + output = (opt[:output] or Oxidized.config.output.default) if not Oxidized.mgr.output[output] Oxidized.mgr.add_output output or raise MethodNotFound, "#{output} not found for node #{ip}" end @@ -163,7 +159,7 @@ module Oxidized end def resolve_model opt - model = (opt[:model] or CFG.model) + model = (opt[:model] or Oxidized.config.model) if not Oxidized.mgr.model[model] Oxidized.mgr.add_model model or raise ModelNotFound, "#{model} not found for node #{ip}" end diff --git a/lib/oxidized/nodes.rb b/lib/oxidized/nodes.rb index cb2ce7b..f0c5d56 100644 --- a/lib/oxidized/nodes.rb +++ b/lib/oxidized/nodes.rb @@ -9,7 +9,7 @@ module Oxidized def load node_want=nil with_lock do new = [] - @source = CFG.source.default + @source = Oxidized.config.source.default Oxidized.mgr.add_source @source Oxidized.mgr.source[@source].new.load.each do |node| # we want to load specific node(s), not all of them diff --git a/lib/oxidized/output/file.rb b/lib/oxidized/output/file.rb index 38c9917..ba08683 100644 --- a/lib/oxidized/output/file.rb +++ b/lib/oxidized/output/file.rb @@ -3,13 +3,13 @@ class OxidizedFile < Output require 'fileutils' def initialize - @cfg = CFG.output.file + @cfg = Oxidized.config.output.file end def setup if @cfg.empty? - CFGS.user.output.file.directory = File.join(Config::Root, 'configs') - CFGS.save :user + Oxidized.asetus.user.output.file.directory = File.join(Config::Root, 'configs') + Oxidized.asetus.save :user raise NoConfig, 'no output file config, edit ~/.config/oxidized/config' end end diff --git a/lib/oxidized/output/git.rb b/lib/oxidized/output/git.rb index 3757cfc..eedf51d 100644 --- a/lib/oxidized/output/git.rb +++ b/lib/oxidized/output/git.rb @@ -8,15 +8,15 @@ class Git < Output end def initialize - @cfg = CFG.output.git + @cfg = Oxidized.config.output.git end def setup if @cfg.empty? - CFGS.user.output.git.user = 'Oxidized' - CFGS.user.output.git.email = 'o@example.com' - CFGS.user.output.git.repo = File.join(Config::Root, 'oxidized.git') - CFGS.save :user + Oxidized.asetus.user.output.git.user = 'Oxidized' + Oxidized.asetus.user.output.git.email = 'o@example.com' + Oxidized.asetus.user.output.git.repo = File.join(Config::Root, 'oxidized.git') + Oxidized.asetus.save :user raise NoConfig, 'no output git config, edit ~/.config/oxidized/config' end @cfg.repo = File.expand_path @cfg.repo diff --git a/lib/oxidized/source/csv.rb b/lib/oxidized/source/csv.rb index 5064e5e..a0ce848 100644 --- a/lib/oxidized/source/csv.rb +++ b/lib/oxidized/source/csv.rb @@ -1,17 +1,17 @@ module Oxidized class CSV < Source def initialize - @cfg = CFG.source.csv + @cfg = Oxidized.config.source.csv super end def setup if @cfg.empty? - CFGS.user.source.csv.file = File.join(Config::Root, 'router.db') - CFGS.user.source.csv.delimiter = /:/ - CFGS.user.source.csv.map.name = 0 - CFGS.user.source.csv.map.model = 1 - CFGS.save :user + Oxidized.asetus.user.source.csv.file = File.join(Config::Root, 'router.db') + Oxidized.asetus.user.source.csv.delimiter = /:/ + Oxidized.asetus.user.source.csv.map.name = 0 + Oxidized.asetus.user.source.csv.map.model = 1 + Oxidized.asetus.save :user raise NoConfig, 'no source csv config, edit ~/.config/oxidized/config' end end @@ -19,7 +19,7 @@ class CSV < Source def load nodes = [] open(File.expand_path @cfg.file).each_line do |line| - next if line.match /^\s*#/ + next if line.match(/^\s*#/) data = line.chomp.split @cfg.delimiter next if data.empty? # map node parameters diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index d2e3ea6..6e765cf 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -1,7 +1,7 @@ module Oxidized class HTTP < Source def initialize - @cfg = CFG.source.http + @cfg = Oxidized.config.source.http super end diff --git a/lib/oxidized/source/source.rb b/lib/oxidized/source/source.rb index 3c1f255..7862dd1 100644 --- a/lib/oxidized/source/source.rb +++ b/lib/oxidized/source/source.rb @@ -2,7 +2,7 @@ module Oxidized class Source class NoConfig < OxidizedError; end def initialize - @map = (CFG.model_map or {}) + @map = (Oxidized.config.model_map or {}) end def map_model model @map.has_key?(model) ? @map[model] : model diff --git a/lib/oxidized/source/sql.rb b/lib/oxidized/source/sql.rb index cd9ef4a..fc1caa8 100644 --- a/lib/oxidized/source/sql.rb +++ b/lib/oxidized/source/sql.rb @@ -8,12 +8,12 @@ class SQL < Source def setup if @cfg.empty? - CFGS.user.source.sql.adapter = 'sqlite' - CFGS.user.source.sql.database = File.join(Config::Root, 'sqlite.db') - CFGS.user.source.sql.table = 'devices' - CFGS.user.source.sql.map.name = 'name' - CFGS.user.source.sql.map.model = 'rancid' - CFGS.save :user + Oxidized.asetus.user.source.sql.adapter = 'sqlite' + Oxidized.asetus.user.source.sql.database = File.join(Config::Root, 'sqlite.db') + Oxidized.asetus.user.source.sql.table = 'devices' + Oxidized.asetus.user.source.sql.map.name = 'name' + Oxidized.asetus.user.source.sql.map.model = 'rancid' + Oxidized.asetus.save :user raise NoConfig, 'no source sql config, edit ~/.config/oxidized/config' end end @@ -44,7 +44,7 @@ class SQL < Source def initialize super - @cfg = CFG.source.sql + @cfg = Oxidized.config.source.sql end def connect diff --git a/lib/oxidized/string.rb b/lib/oxidized/string.rb index 4bdfbf2..8bcb082 100644 --- a/lib/oxidized/string.rb +++ b/lib/oxidized/string.rb @@ -16,7 +16,7 @@ module Oxidized # sets @cmd and @name unless @name is already set def set_cmd command @cmd = command - @name = @cmd.strip.gsub(/\s+/, '_') if @name == nil + @name ||= @cmd.strip.gsub(/\s+/, '_') end def initialize str='' diff --git a/lib/oxidized/version.rb b/lib/oxidized/version.rb new file mode 100644 index 0000000..c2a0c0e --- /dev/null +++ b/lib/oxidized/version.rb @@ -0,0 +1,3 @@ +module Oxidized + VERSION = '0.8.1' +end diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb index eea747e..b36929f 100644 --- a/lib/oxidized/worker.rb +++ b/lib/oxidized/worker.rb @@ -4,7 +4,7 @@ module Oxidized class Worker def initialize nodes @nodes = nodes - @jobs = Jobs.new CFG.threads, CFG.interval, @nodes + @jobs = Jobs.new(Oxidized.config.threads, Oxidized.config.interval, @nodes) Thread.abort_on_exception = true end @@ -18,7 +18,7 @@ module Oxidized # ask for next node in queue non destructive way nextnode = @nodes.first unless nextnode.last.nil? - break if nextnode.last.end + CFG.interval > Time.now.utc + break if nextnode.last.end + Oxidized.config.interval > Time.now.utc end # shift nodes and get the next node node = @nodes.get @@ -48,7 +48,7 @@ module Oxidized node.reset else msg = "#{node.name} status #{job.status}" - if node.retry < CFG.retries + if node.retry < Oxidized.config.retries node.retry += 1 msg += ", retry attempt #{node.retry}" @nodes.next node.name diff --git a/oxidized.gemspec b/oxidized.gemspec index d2956b1..a5d481e 100644 --- a/oxidized.gemspec +++ b/oxidized.gemspec @@ -1,6 +1,11 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'oxidized/version' + Gem::Specification.new do |s| s.name = 'oxidized' - s.version = '0.8.1' + s.version = Oxidized::VERSION s.licenses = %w( Apache-2.0 ) s.platform = Gem::Platform::RUBY s.authors = [ 'Saku Ytti', 'Samer Abdel-Hafez', 'Anton Aksola' ] @@ -9,7 +14,7 @@ Gem::Specification.new do |s| s.summary = 'feeble attempt at rancid' s.description = 'software to fetch configuration from network devices and store them' s.rubyforge_project = s.name - s.files = `git ls-files`.split("\n") + s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } s.executables = %w( oxidized ) s.require_path = 'lib' @@ -19,4 +24,8 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'net-ssh', '~> 2.8' s.add_runtime_dependency 'rugged', '~> 0.21', '>= 0.21.4' s.add_development_dependency 'pry', '~> 0' + s.add_development_dependency 'bundler', '~> 1.10' + s.add_development_dependency 'rake', '~> 10.0' + s.add_development_dependency 'minitest', '~> 5.8' + s.add_development_dependency 'mocha', '~> 1.1' end diff --git a/spec/node_spec.rb b/spec/node_spec.rb new file mode 100644 index 0000000..c568463 --- /dev/null +++ b/spec/node_spec.rb @@ -0,0 +1,44 @@ +require 'spec_helper' + +describe Oxidized::Node do + before(:each) do + Oxidized.stubs(:asetus).returns(Asetus.new) + + Oxidized::Node.any_instance.stubs(:resolve_output) + @node = Oxidized::Node.new(name: 'example.com', + input: 'ssh', + output: 'git', + model: 'junos', + username: 'alma', + password: 'armud', + prompt: 'test_prompt') + + end + + describe '#new' do + it 'should resolve input' do + @node.input[0].to_s.split('::')[1].must_equal 'SSH' + end + it 'should resolve model' do + @node.model.class.must_equal JunOS + end + it 'should resolve username' do + @node.auth[:username].must_equal 'alma' + end + it 'should resolve password' do + @node.auth[:password].must_equal 'armud' + end + it 'should require prompt' do + @node.prompt.must_equal 'test_prompt' + end + end + + describe '#run' do + it 'should fetch the configuration' do + stub_oxidized_ssh + + status, _ = @node.run + status.must_equal :success + end + end +end diff --git a/spec/nodes_spec.rb b/spec/nodes_spec.rb index 80ed300..5f2ef95 100644 --- a/spec/nodes_spec.rb +++ b/spec/nodes_spec.rb @@ -1,33 +1,44 @@ -require 'oxidized' -Oxidized.mgr = Oxidized::Manager.new +require 'spec_helper' describe Oxidized::Nodes do before(:each) do + Resolv.any_instance.stubs(:getaddress) + Oxidized.stubs(:asetus).returns(Asetus.new) + opts = { + input: 'ssh', + output: 'git', + model: 'junos', + username: 'alma', + password: 'armud', + prompt: 'test_prompt' + } + + Oxidized::Node.any_instance.stubs(:resolve_output) @nodes_org = %w(ltt-pe1.hel kes2-rr1.tku tor-peer1.oul - hal-p2.tre sav-gr1-sw1.kuo psl-sec-pe1.hel).map { |e| Oxidized::Node.new(:name=>e) } - @nodes = Oxidized::Nodes.new @nodes_org.dup + hal-p2.tre sav-gr1-sw1.kuo psl-sec-pe1.hel).map { |e| Oxidized::Node.new(opts.merge(name: e)) } + @node = @nodes_org.delete_at(0) + @nodes = Oxidized::Nodes.new(nodes: @nodes_org.dup) end describe '#put' do it 'adds node to top of queue' do - node = Oxidized::Node.new(:name=>'kst-p1.sto') - @nodes.put node - expect(@nodes).to eq [node] + @nodes_org + @nodes.put @node + @nodes.must_equal [@node] + @nodes_org end end describe '#get' do it 'returns node from top of queue' do - expect(@nodes.get).to eq @nodes_org.first + @nodes.get.must_equal @nodes_org.first end it 'moves node from top to bottom' do @nodes.get - expect(@nodes).to end_with [@nodes_org.first] + @nodes.last.must_equal @nodes_org.first end it 'does not change node count' do before = @nodes.size @nodes.get - expect(before).to eq @nodes.size + before.must_equal @nodes.size end end @@ -35,12 +46,12 @@ describe Oxidized::Nodes do it 'moves node to top of queue' do node = @nodes[3] @nodes.next node.name - expect(@nodes).to start_with [node] + @nodes.first.must_equal node end it 'does not change node count' do before = @nodes.size @nodes.next @nodes[3].name - expect(before).to eq @nodes.size + before.must_equal @nodes.size end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..28eb9d4 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,14 @@ +require 'minitest/autorun' +require 'mocha/mini_test' +require 'oxidized' + +Oxidized.mgr = Oxidized::Manager.new + +def stub_oxidized_ssh + Oxidized::SSH.any_instance.stubs(:connect).returns(true) + Oxidized::SSH.any_instance.stubs(:node).returns(@node) + Oxidized::SSH.any_instance.expects(:cmd).at_least(1).returns("this is a command output\nModel: mx960") + Oxidized::SSH.any_instance.stubs(:connect_cli).returns(true) + Oxidized::SSH.any_instance.stubs(:disconnect).returns(true) + Oxidized::SSH.any_instance.stubs(:disconnect_cli).returns(true) +end -- cgit v1.2.1 From 8c61bf35408c0b47eb79d2d9d135b5b479abfecb Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Fri, 23 Oct 2015 15:18:58 -0400 Subject: a callback to push config changes to a remote repository --- lib/oxidized/hook/githubrepo.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 lib/oxidized/hook/githubrepo.rb diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb new file mode 100644 index 0000000..f2ffa2a --- /dev/null +++ b/lib/oxidized/hook/githubrepo.rb @@ -0,0 +1,16 @@ +class GithubRepo < Oxidized::Hook + def validate_cfg! + cfg.has_key?('remote_repo') or raise 'remote_repo is required' + end + + def run_hook(ctx) + credentials = Rugged::Credentials::SshKeyFromAgent.new(username: 'git') + repo = Rugged::Repository.new(Oxidized::CFG.output.git.repo) + log "Pushing local repository(#{repo.path})..." + remote = repo.remotes['origin'] || repo.remotes.create('origin', cfg.remote_repo) + log "to remote: #{remote.url}" + remote.push(['refs/heads/master'], credentials: credentials) + rescue Exception => e + log e.backtrace.join('\n') + end +end -- cgit v1.2.1 From c96c52ab7153ffd85906550544df627c6c80aac8 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 26 Oct 2015 12:48:48 -0400 Subject: raise more specific exception when required key is not present --- lib/oxidized/hook/githubrepo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index f2ffa2a..efa536b 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -1,6 +1,6 @@ class GithubRepo < Oxidized::Hook def validate_cfg! - cfg.has_key?('remote_repo') or raise 'remote_repo is required' + cfg.has_key?('remote_repo') or raise KeyError, 'remote_repo is required' end def run_hook(ctx) -- cgit v1.2.1 From bd7d0b1660be8bd4a32828ea5ca83c16c1797045 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 26 Oct 2015 13:33:53 -0400 Subject: do not rescue general exception --- lib/oxidized/hook/githubrepo.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index efa536b..a17d6b9 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -10,7 +10,5 @@ class GithubRepo < Oxidized::Hook remote = repo.remotes['origin'] || repo.remotes.create('origin', cfg.remote_repo) log "to remote: #{remote.url}" remote.push(['refs/heads/master'], credentials: credentials) - rescue Exception => e - log e.backtrace.join('\n') end end -- cgit v1.2.1 From 817f33a2a1e8f78720f5e73a10ee45384e886ae9 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Fri, 13 Nov 2015 16:31:02 -0500 Subject: githubrepo hook spec --- lib/oxidized/hook/githubrepo.rb | 2 +- spec/githubrepo_spec.rb | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 spec/githubrepo_spec.rb diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index a17d6b9..85adaab 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -5,7 +5,7 @@ class GithubRepo < Oxidized::Hook def run_hook(ctx) credentials = Rugged::Credentials::SshKeyFromAgent.new(username: 'git') - repo = Rugged::Repository.new(Oxidized::CFG.output.git.repo) + repo = Rugged::Repository.new(Oxidized.config.output.git.repo) log "Pushing local repository(#{repo.path})..." remote = repo.remotes['origin'] || repo.remotes.create('origin', cfg.remote_repo) log "to remote: #{remote.url}" diff --git a/spec/githubrepo_spec.rb b/spec/githubrepo_spec.rb new file mode 100644 index 0000000..c98244d --- /dev/null +++ b/spec/githubrepo_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' +require 'rugged' +require 'oxidized/hook/githubrepo' + +describe Oxidized::Node do + before(:each) do + asetus = Asetus.new + asetus.cfg.output.git.repo = 'foo.git' + Oxidized.stubs(:asetus).returns(asetus) + repo = mock() + remote = mock() + remote.expects(:url).returns('github.com/foo.git') + remote.expects(:push).returns(true) + repo.expects(:remotes).returns({'origin' => remote}) + repo.expects(:path).returns('foo.git') + Rugged::Repository.expects(:new).with('foo.git').returns(repo) + end + + describe "#run_hook" do + it "will push to the remote repository" do + gr = GithubRepo.new + gr.run_hook(nil).must_equal true + end + end +end -- cgit v1.2.1 From a54574ee01060097259adc22413efe1853b83bbb Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 26 Oct 2015 14:30:48 -0400 Subject: include health and hardware info into config --- lib/oxidized/model/junos.rb | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index da3af72..2572b93 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -20,7 +20,7 @@ class JunOS < Oxidized::Model cmd 'show configuration | display omit' - cmd 'show version' do |cfg| + cmd 'show version detail' do |cfg| @model = $1 if cfg.match(/^Model: (\S+)/) comment cfg end @@ -30,13 +30,26 @@ class JunOS < Oxidized::Model case @model when 'mx960' out << cmd('show chassis fabric reachability') { |cfg| comment cfg } + when 'mx480' + out << cmd('show chassis scb') { |cfg| comment cfg } + out << cmd('show chassis sfm detail') { |cfg| comment cfg } + out << cmd('show chassis ssb') { |cfg| comment cfg } + out << cmd('show chassis feb detail') { |cfg| comment cfg } + out << cmd('show chassis feb') { |cfg| comment cfg } + out << cmd('show chassis cfeb') { |cfg| comment cfg } end out end - cmd 'show chassis hardware' do |cfg| - comment cfg - end + cmd('show chassis environment') { |cfg| comment cfg } + cmd('show chassis firmware') { |cfg| comment cfg } + cmd('show chassis fpc detail') { |cfg| comment cfg } + cmd('show chassis hardware detail') { |cfg| comment cfg } + cmd('show chassis routing-engine') { |cfg| comment cfg } + cmd('show chassis alarms') { |cfg| comment cfg } + cmd('show system license') { |cfg| comment cfg } + cmd('show system boot-messages') { |cfg| comment cfg } + cmd('show system core-dumps') { |cfg| comment cfg } cfg :telnet do username(/^login:/) -- cgit v1.2.1 From 13d89b7e1a4f91728e528a5290779bcf4b645a6a Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 26 Oct 2015 15:02:09 -0400 Subject: strip temprature and speed measurement from chassis environment output --- lib/oxidized/model/junos.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index 2572b93..6ba18ad 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -41,7 +41,11 @@ class JunOS < Oxidized::Model out end - cmd('show chassis environment') { |cfg| comment cfg } + cmd('show chassis environment') do |cfg| + cfg.gsub! /\d+ degrees.* F/, '' + cfg.gsub! /Spinning at \S+/, '' + comment cfg + end cmd('show chassis firmware') { |cfg| comment cfg } cmd('show chassis fpc detail') { |cfg| comment cfg } cmd('show chassis hardware detail') { |cfg| comment cfg } -- cgit v1.2.1 From 7312e8ef0d10d8846c69d6962e98a450e6950bc9 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 23 Nov 2015 11:23:59 -0500 Subject: use paranthesis for method call --- lib/oxidized/model/junos.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index 6ba18ad..949a757 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -42,8 +42,8 @@ class JunOS < Oxidized::Model end cmd('show chassis environment') do |cfg| - cfg.gsub! /\d+ degrees.* F/, '' - cfg.gsub! /Spinning at \S+/, '' + cfg.gsub!(/\d+ degrees.* F/, '') + cfg.gsub!(/Spinning at \S+/, '') comment cfg end cmd('show chassis firmware') { |cfg| comment cfg } -- cgit v1.2.1 From 26b02bf064c76d6ef7943016c798629810986092 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 23 Nov 2015 12:41:50 -0500 Subject: include executed command into output --- lib/oxidized/model/model.rb | 2 +- oxidized-0.8.1.gem | Bin 0 -> 41472 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 oxidized-0.8.1.gem diff --git a/lib/oxidized/model/model.rb b/lib/oxidized/model/model.rb index c95cc67..ca851e3 100644 --- a/lib/oxidized/model/model.rb +++ b/lib/oxidized/model/model.rb @@ -79,7 +79,7 @@ module Oxidized attr_accessor :input, :node def cmd string, &block - out = @input.cmd string + out = "====================== #{string} ======================\n" + @input.cmd(string) return false unless out self.class.cmds[:all].each do |all_block| out = instance_exec Oxidized::String.new(out), string, &all_block diff --git a/oxidized-0.8.1.gem b/oxidized-0.8.1.gem new file mode 100644 index 0000000..c778eb7 Binary files /dev/null and b/oxidized-0.8.1.gem differ -- cgit v1.2.1 From 6b47e7122ea9e8f470c30335d4bcb588ed279a18 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 23 Nov 2015 13:32:17 -0500 Subject: ignore frequently changing monitoring data --- lib/oxidized/model/junos.rb | 13 +++++++++++-- oxidized-0.8.1.gem | Bin 41472 -> 0 bytes 2 files changed, 11 insertions(+), 2 deletions(-) delete mode 100644 oxidized-0.8.1.gem diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index 949a757..a02d588 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -47,9 +47,18 @@ class JunOS < Oxidized::Model comment cfg end cmd('show chassis firmware') { |cfg| comment cfg } - cmd('show chassis fpc detail') { |cfg| comment cfg } + cmd('show chassis fpc detail') do |cfg| + cfg.gsub!(/(Temperature\s+)(\d+)(.+)/, '\1\3') + cfg.gsub!(/(\s+\d+ days,)(.+seconds)/, '\1 \3') + comment cfg + end cmd('show chassis hardware detail') { |cfg| comment cfg } - cmd('show chassis routing-engine') { |cfg| comment cfg } + cmd('show chassis routing-engine') do |cfg| + cfg.gsub!(/(\S+\s+)(\d+)( percent)/, '\1\3') + cfg.gsub!(/(\s+\d+ days,)(.+seconds)/, '\1 \3') + cfg.gsub!(/(\s+)(\d+\.\d+)(\s+)(\d+\.\d+)(\s+)(\d+\.\d+)/, '\1\3\5') + comment cfg + end cmd('show chassis alarms') { |cfg| comment cfg } cmd('show system license') { |cfg| comment cfg } cmd('show system boot-messages') { |cfg| comment cfg } diff --git a/oxidized-0.8.1.gem b/oxidized-0.8.1.gem deleted file mode 100644 index c778eb7..0000000 Binary files a/oxidized-0.8.1.gem and /dev/null differ -- cgit v1.2.1 From 98a9d26be543e49ea21586c7b48e49387b8e0c09 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Thu, 3 Dec 2015 10:28:46 -0500 Subject: do not do syslog --- lib/oxidized/log.rb | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/oxidized/log.rb b/lib/oxidized/log.rb index e9ae1b4..04a2d65 100644 --- a/lib/oxidized/log.rb +++ b/lib/oxidized/log.rb @@ -1,22 +1,15 @@ module Oxidized + require 'logger' - begin - require 'syslog/logger' - Log = Syslog::Logger.new 'oxidized' - Log.define_singleton_method(:file=){|arg|} - rescue LoadError - # 1.9.3 has no love for syslog - require 'logger' - class Logger < Logger - def initialize target=STDOUT - super target - end - def file= target - FileUtils.mkdir_p File.dirname(target) - @logdev = LogDevice.new target - end + class Logger < Logger + def initialize target=STDOUT + super target + end + def file= target + FileUtils.mkdir_p File.dirname(target) + @logdev = LogDevice.new target end - Log = Logger.new end + Log = Logger.new end -- cgit v1.2.1 From bb1d59b78fd197c992e2a59dd3b56b0ffe0d3320 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Fri, 4 Dec 2015 00:13:35 -0500 Subject: user password username for github api --- lib/oxidized/hook/githubrepo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index 85adaab..fe73015 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -4,7 +4,7 @@ class GithubRepo < Oxidized::Hook end def run_hook(ctx) - credentials = Rugged::Credentials::SshKeyFromAgent.new(username: 'git') + credentials = Rugged::Credentials::UserPassword.new(username: cfg.username, password: cfg.password) repo = Rugged::Repository.new(Oxidized.config.output.git.repo) log "Pushing local repository(#{repo.path})..." remote = repo.remotes['origin'] || repo.remotes.create('origin', cfg.remote_repo) -- cgit v1.2.1 From 71f38cc19411c482c3cd932976a5183e089226eb Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Fri, 4 Dec 2015 00:27:55 -0500 Subject: fix the spec --- spec/githubrepo_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/githubrepo_spec.rb b/spec/githubrepo_spec.rb index c98244d..a193cd6 100644 --- a/spec/githubrepo_spec.rb +++ b/spec/githubrepo_spec.rb @@ -6,6 +6,10 @@ describe Oxidized::Node do before(:each) do asetus = Asetus.new asetus.cfg.output.git.repo = 'foo.git' + asetus.cfg.hooks.github_repo_hook.remote_repo = 'https://github.com/blah/blah.git' + asetus.cfg.hooks.github_repo_hook.username = 'username' + asetus.cfg.hooks.github_repo_hook.password = 'password' + GithubRepo.any_instance.stubs(:cfg).returns(asetus.cfg.hooks.github_repo_hook) Oxidized.stubs(:asetus).returns(asetus) repo = mock() remote = mock() -- cgit v1.2.1 From eead03c1557117882b42759de2de4114e3a4b5cd Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Fri, 4 Dec 2015 03:51:12 -0500 Subject: consider local repo main --- lib/oxidized/hook/githubrepo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index fe73015..bf33b89 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -9,6 +9,6 @@ class GithubRepo < Oxidized::Hook log "Pushing local repository(#{repo.path})..." remote = repo.remotes['origin'] || repo.remotes.create('origin', cfg.remote_repo) log "to remote: #{remote.url}" - remote.push(['refs/heads/master'], credentials: credentials) + remote.push(['+refs/heads/master'], credentials: credentials) end end -- cgit v1.2.1 From e73de9397d110a1442e3717823605d3020016b17 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Fri, 4 Dec 2015 13:39:10 -0500 Subject: do not start new instance when there is one running --- lib/oxidized/cli.rb | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/lib/oxidized/cli.rb b/lib/oxidized/cli.rb index 00ea4b6..0c87d53 100644 --- a/lib/oxidized/cli.rb +++ b/lib/oxidized/cli.rb @@ -4,7 +4,9 @@ module Oxidized require 'slop' def run + check_pid Process.daemon if @opts[:daemonize] + write_pid begin Oxidized.new rescue => error @@ -19,6 +21,7 @@ module Oxidized Log.info "Oxidized starting, running as pid #{$$}" _args, @opts = parse_opts Oxidized.config.debug = true if @opts[:debug] + @pidfile = File.expand_path("pid") end def crash error @@ -40,5 +43,49 @@ module Oxidized end [opts.parse!, opts] end + + def pidfile + @pidfile + end + + def pidfile? + !!pidfile + end + + def write_pid + if pidfile? + begin + File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY){|f| f.write("#{Process.pid}") } + at_exit { File.delete(pidfile) if File.exists?(pidfile) } + rescue Errno::EEXIST + check_pid + retry + end + end + end + + def check_pid + if pidfile? + case pid_status(pidfile) + when :running, :not_owned + puts "A server is already running. Check #{pidfile}" + exit(1) + when :dead + File.delete(pidfile) + end + end + end + + def pid_status(pidfile) + return :exited unless File.exists?(pidfile) + pid = ::File.read(pidfile).to_i + return :dead if pid == 0 + Process.kill(0, pid) + :running + rescue Errno::ESRCH + :dead + rescue Errno::EPERM + :not_owned + end end end -- cgit v1.2.1 From d0736c757645694ed255e060c5eab45c21f9ffa6 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 7 Dec 2015 09:22:36 -0500 Subject: bug fix --- lib/oxidized/cli.rb | 3 ++- lib/oxidized/model/junos.rb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/oxidized/cli.rb b/lib/oxidized/cli.rb index 0c87d53..fd766e3 100644 --- a/lib/oxidized/cli.rb +++ b/lib/oxidized/cli.rb @@ -18,10 +18,11 @@ module Oxidized private def initialize - Log.info "Oxidized starting, running as pid #{$$}" + Log.info "Oxidized starting, running as pid #{$$} by #{ENV['USER']}" _args, @opts = parse_opts Oxidized.config.debug = true if @opts[:debug] @pidfile = File.expand_path("pid") + Log.debug "pidfile: #{@pidfile}" end def crash error diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index a02d588..1b641ca 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -54,7 +54,8 @@ class JunOS < Oxidized::Model end cmd('show chassis hardware detail') { |cfg| comment cfg } cmd('show chassis routing-engine') do |cfg| - cfg.gsub!(/(\S+\s+)(\d+)( percent)/, '\1\3') + cfg.gsub!(/(\S+\s+)(\d{2})( percent)/, '\1\3') + cfg.gsub!(/(\S+\s+)(\s\d{1})( percent)/, '\1\3') cfg.gsub!(/(\s+\d+ days,)(.+seconds)/, '\1 \3') cfg.gsub!(/(\s+)(\d+\.\d+)(\s+)(\d+\.\d+)(\s+)(\d+\.\d+)/, '\1\3\5') comment cfg -- cgit v1.2.1 From d2e8c0e96460b34371bbb72f0ecf878321fbf8e3 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 7 Dec 2015 14:46:40 -0500 Subject: pull origin/master before pushing --- lib/oxidized/hook/githubrepo.rb | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index bf33b89..83e1078 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -9,6 +9,24 @@ class GithubRepo < Oxidized::Hook log "Pushing local repository(#{repo.path})..." remote = repo.remotes['origin'] || repo.remotes.create('origin', cfg.remote_repo) log "to remote: #{remote.url}" - remote.push(['+refs/heads/master'], credentials: credentials) + + fetch_and_merge_remote(repo, credentials) + + remote.push([repo.head.name], credentials: credentials) + end + + def fetch_and_merge_remote(repo, credentials) + their_branch = repo.branches["origin/master"] or return + + repo.fetch('origin', [repo.head.name], credentials: credentials) + + merge_index = repo.merge_commits(repo.head.target_id, their_branch.target_id) + + Rugged::Commit.create(repo, { + parents: [repo.head.target, their_branch.target], + tree: merge_index.write_tree(repo), + message: "Merge remote-tracking branch '#{their_branch.name}'", + update_ref: "HEAD" + }) end end -- cgit v1.2.1 From 68dd88766604145a8c8d16b22a68a9850ce92564 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 7 Dec 2015 15:27:42 -0500 Subject: fix test --- spec/githubrepo_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spec/githubrepo_spec.rb b/spec/githubrepo_spec.rb index a193cd6..9b509e4 100644 --- a/spec/githubrepo_spec.rb +++ b/spec/githubrepo_spec.rb @@ -17,6 +17,10 @@ describe Oxidized::Node do remote.expects(:push).returns(true) repo.expects(:remotes).returns({'origin' => remote}) repo.expects(:path).returns('foo.git') + repo_head = mock() + repo_head.expects(:name).returns('origin/master') + repo.expects(:head).returns(repo_head) + repo.expects(:branches).returns({}) Rugged::Repository.expects(:new).with('foo.git').returns(repo) end -- cgit v1.2.1 From 2eef353c6ef98662c3092a5a28ee6f167b1d1774 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Tue, 8 Dec 2015 18:34:16 -0500 Subject: warn when there is a conflict --- lib/oxidized/hook/githubrepo.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index 83e1078..25b2cd0 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -16,12 +16,14 @@ class GithubRepo < Oxidized::Hook end def fetch_and_merge_remote(repo, credentials) - their_branch = repo.branches["origin/master"] or return - repo.fetch('origin', [repo.head.name], credentials: credentials) + their_branch = repo.branches["origin/master"] or return + merge_index = repo.merge_commits(repo.head.target_id, their_branch.target_id) + log("Conflicts detected", :warn) if merge_index.conflicts? + Rugged::Commit.create(repo, { parents: [repo.head.target, their_branch.target], tree: merge_index.write_tree(repo), -- cgit v1.2.1 From b6a9c3ae528de0b577ff3f7c601b4ca5b641185d Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Tue, 8 Dec 2015 18:43:18 -0500 Subject: fix spec --- spec/githubrepo_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spec/githubrepo_spec.rb b/spec/githubrepo_spec.rb index 9b509e4..07296f3 100644 --- a/spec/githubrepo_spec.rb +++ b/spec/githubrepo_spec.rb @@ -18,8 +18,9 @@ describe Oxidized::Node do repo.expects(:remotes).returns({'origin' => remote}) repo.expects(:path).returns('foo.git') repo_head = mock() - repo_head.expects(:name).returns('origin/master') - repo.expects(:head).returns(repo_head) + repo_head.expects(:name).twice.returns('origin/master') + repo.expects(:head).twice.returns(repo_head) + repo.expects(:fetch).returns(true) repo.expects(:branches).returns({}) Rugged::Repository.expects(:new).with('foo.git').returns(repo) end -- cgit v1.2.1 From f79e596b571b8b5913c2b4a9dc79968a712dab7b Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Wed, 9 Dec 2015 22:34:47 -0500 Subject: keep the config output clean --- lib/oxidized/model/junos.rb | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index 1b641ca..bb56481 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -20,7 +20,7 @@ class JunOS < Oxidized::Model cmd 'show configuration | display omit' - cmd 'show version detail' do |cfg| + cmd 'show version' do |cfg| @model = $1 if cfg.match(/^Model: (\S+)/) comment cfg end @@ -30,40 +30,11 @@ class JunOS < Oxidized::Model case @model when 'mx960' out << cmd('show chassis fabric reachability') { |cfg| comment cfg } - when 'mx480' - out << cmd('show chassis scb') { |cfg| comment cfg } - out << cmd('show chassis sfm detail') { |cfg| comment cfg } - out << cmd('show chassis ssb') { |cfg| comment cfg } - out << cmd('show chassis feb detail') { |cfg| comment cfg } - out << cmd('show chassis feb') { |cfg| comment cfg } - out << cmd('show chassis cfeb') { |cfg| comment cfg } end out end - cmd('show chassis environment') do |cfg| - cfg.gsub!(/\d+ degrees.* F/, '') - cfg.gsub!(/Spinning at \S+/, '') - comment cfg - end - cmd('show chassis firmware') { |cfg| comment cfg } - cmd('show chassis fpc detail') do |cfg| - cfg.gsub!(/(Temperature\s+)(\d+)(.+)/, '\1\3') - cfg.gsub!(/(\s+\d+ days,)(.+seconds)/, '\1 \3') - comment cfg - end - cmd('show chassis hardware detail') { |cfg| comment cfg } - cmd('show chassis routing-engine') do |cfg| - cfg.gsub!(/(\S+\s+)(\d{2})( percent)/, '\1\3') - cfg.gsub!(/(\S+\s+)(\s\d{1})( percent)/, '\1\3') - cfg.gsub!(/(\s+\d+ days,)(.+seconds)/, '\1 \3') - cfg.gsub!(/(\s+)(\d+\.\d+)(\s+)(\d+\.\d+)(\s+)(\d+\.\d+)/, '\1\3\5') - comment cfg - end - cmd('show chassis alarms') { |cfg| comment cfg } - cmd('show system license') { |cfg| comment cfg } - cmd('show system boot-messages') { |cfg| comment cfg } - cmd('show system core-dumps') { |cfg| comment cfg } + cmd('show chassis hardware') { |cfg| comment cfg } cfg :telnet do username(/^login:/) -- cgit v1.2.1 From fa3b93c5046f23ebbb2e82ac4944357683f8c5d8 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Wed, 9 Dec 2015 22:53:16 -0500 Subject: let user configure which auth method to use --- lib/oxidized/hook/githubrepo.rb | 18 ++++++++++++--- spec/githubrepo_spec.rb | 50 ++++++++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index 25b2cd0..22ee8ea 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -4,18 +4,17 @@ class GithubRepo < Oxidized::Hook end def run_hook(ctx) - credentials = Rugged::Credentials::UserPassword.new(username: cfg.username, password: cfg.password) repo = Rugged::Repository.new(Oxidized.config.output.git.repo) log "Pushing local repository(#{repo.path})..." remote = repo.remotes['origin'] || repo.remotes.create('origin', cfg.remote_repo) log "to remote: #{remote.url}" - fetch_and_merge_remote(repo, credentials) + fetch_and_merge_remote(repo) remote.push([repo.head.name], credentials: credentials) end - def fetch_and_merge_remote(repo, credentials) + def fetch_and_merge_remote(repo) repo.fetch('origin', [repo.head.name], credentials: credentials) their_branch = repo.branches["origin/master"] or return @@ -31,4 +30,17 @@ class GithubRepo < Oxidized::Hook update_ref: "HEAD" }) end + + private + + def credentials + @credentials ||= if cfg.has_key?('username') && cfg.has_key?('password') + log "Using https auth", :debug + Rugged::Credentials::UserPassword.new(username: cfg.username, password: cfg.password) + else + log "Using ssh auth", :debug + Rugged::Credentials::SshKeyFromAgent.new(username: 'git') + end + end + end diff --git a/spec/githubrepo_spec.rb b/spec/githubrepo_spec.rb index 07296f3..a3b4953 100644 --- a/spec/githubrepo_spec.rb +++ b/spec/githubrepo_spec.rb @@ -4,30 +4,50 @@ require 'oxidized/hook/githubrepo' describe Oxidized::Node do before(:each) do - asetus = Asetus.new - asetus.cfg.output.git.repo = 'foo.git' - asetus.cfg.hooks.github_repo_hook.remote_repo = 'https://github.com/blah/blah.git' - asetus.cfg.hooks.github_repo_hook.username = 'username' - asetus.cfg.hooks.github_repo_hook.password = 'password' - GithubRepo.any_instance.stubs(:cfg).returns(asetus.cfg.hooks.github_repo_hook) - Oxidized.stubs(:asetus).returns(asetus) - repo = mock() + Oxidized.asetus = Asetus.new + Oxidized.config.output.git.repo = 'foo.git' + + @credentials = mock() + remote = mock() - remote.expects(:url).returns('github.com/foo.git') - remote.expects(:push).returns(true) - repo.expects(:remotes).returns({'origin' => remote}) - repo.expects(:path).returns('foo.git') + remote.expects(:url).returns('https://github.com/username/foo.git') + remote.expects(:push).with(['refs/heads/master'], credentials: @credentials).returns(true) + repo_head = mock() - repo_head.expects(:name).twice.returns('origin/master') + repo_head.expects(:name).twice.returns('refs/heads/master') + + repo = mock() + repo.expects(:path).returns('foo.git') + repo.expects(:remotes).returns({'origin' => remote}) repo.expects(:head).twice.returns(repo_head) - repo.expects(:fetch).returns(true) + repo.expects(:fetch).with('origin', ['refs/heads/master'], credentials: @credentials).returns(true) repo.expects(:branches).returns({}) + Rugged::Repository.expects(:new).with('foo.git').returns(repo) end describe "#run_hook" do - it "will push to the remote repository" do + it "will push to the remote repository using https" do + Oxidized.config.hooks.github_repo_hook.remote_repo = 'https://github.com/username/foo.git' + Oxidized.config.hooks.github_repo_hook.username = 'username' + Oxidized.config.hooks.github_repo_hook.password = 'password' + + Rugged::Credentials::UserPassword.expects(:new).with(username: 'username', password: 'password').returns(@credentials) + + gr = GithubRepo.new + gr.cfg = Oxidized.config.hooks.github_repo_hook + + gr.run_hook(nil).must_equal true + end + + it "will push to the remote repository using ssh" do + Oxidized.config.hooks.github_repo_hook.remote_repo = 'git@github.com:username/foo.git' + + Rugged::Credentials::SshKeyFromAgent.expects(:new).with(username: 'git').returns(@credentials) + gr = GithubRepo.new + gr.cfg = Oxidized.config.hooks.github_repo_hook + gr.run_hook(nil).must_equal true end end -- cgit v1.2.1 From 1c4f886e883f947b649613a21955929fa6d7e94c Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Wed, 9 Dec 2015 23:43:32 -0500 Subject: refactor and fix bugs introduced with new changes --- README.md | 2 +- bin/oxidized | 2 +- lib/oxidized.rb | 34 +++++++++++++++++++++++++++++++++- lib/oxidized/cli.rb | 12 +++++++----- lib/oxidized/config.rb | 9 +++++---- lib/oxidized/core.rb | 12 ++---------- lib/oxidized/hook.rb | 6 +++--- lib/oxidized/input/ftp.rb | 2 +- lib/oxidized/input/ssh.rb | 2 +- lib/oxidized/input/telnet.rb | 2 +- lib/oxidized/log.rb | 15 --------------- lib/oxidized/node.rb | 4 ++-- lib/oxidized/nodes.rb | 22 +++++++++++----------- lib/oxidized/worker.rb | 8 ++++---- spec/githubrepo_spec.rb | 1 + 15 files changed, 73 insertions(+), 60 deletions(-) delete mode 100644 lib/oxidized/log.rb diff --git a/README.md b/README.md index aea4326..3b7083f 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Oxidized supports ```CSV```, ```SQLite``` and ```HTTP``` as source backends. The Possible outputs are either ```file``` or ```git```. The file backend takes a destination directory as argument and will keep a file per device, with most recent running version of a device. The GIT backend (recommended) will initialize an empty GIT repository in the specified path and create a new commit on every configuration change. Take a look at the [Cookbook](#cookbook) for more details. -Maps define how to map a model's fields to model [model fields](https://github.com/ytti/oxidized/tree/master/lib/oxidized/model). Most of the settings should be self explanatory, log is ignored if Syslog::Logger exists (>=2.0) and syslog is used instead. +Maps define how to map a model's fields to model [model fields](https://github.com/ytti/oxidized/tree/master/lib/oxidized/model). Most of the settings should be self explanatory, log is ignored if `use_syslog`(requires Ruby >= 2.0) is set to `true`. First create the directory where the CSV ```output``` is going to store device configs and start Oxidized once. ``` diff --git a/bin/oxidized b/bin/oxidized index e5d9bec..2695a52 100755 --- a/bin/oxidized +++ b/bin/oxidized @@ -9,5 +9,5 @@ begin Oxidized::CLI.new.run rescue => error warn "#{error}" - raise if Oxidized::CFG.debug + raise if Oxidized.config.debug end diff --git a/lib/oxidized.rb b/lib/oxidized.rb index df30602..e92224a 100644 --- a/lib/oxidized.rb +++ b/lib/oxidized.rb @@ -1,6 +1,15 @@ module Oxidized class OxidizedError < StandardError; end - Directory = File.expand_path File.join File.dirname(__FILE__), '../' + + Directory = File.expand_path(File.join(File.dirname(__FILE__), '../')) + + require 'oxidized/string' + require 'oxidized/config' + require 'oxidized/config/vars' + require 'oxidized/worker' + require 'oxidized/nodes' + require 'oxidized/manager' + require 'oxidized/hook' require 'oxidized/core' def self.asetus @@ -14,4 +23,27 @@ module Oxidized def self.config asetus.cfg end + + def self.logger + @@logger + end + + def self.logger=(val) + @@logger = val + end + + def self.setup_logger + self.logger = if config.has_key?('use_syslog') && config.use_syslog + require 'syslog/logger' + Syslog::Logger.new('oxidized') + else + require 'logger' + if config.has_key?('log') + Logger.new(config.log) + else + Logger.new(STDERR) + end + end + logger.level = Logger::INFO unless config.debug + end end diff --git a/lib/oxidized/cli.rb b/lib/oxidized/cli.rb index fd766e3..15d20c5 100644 --- a/lib/oxidized/cli.rb +++ b/lib/oxidized/cli.rb @@ -1,13 +1,14 @@ module Oxidized class CLI - require 'oxidized' require 'slop' + require 'oxidized' def run check_pid Process.daemon if @opts[:daemonize] write_pid begin + Oxidized.logger.info "Oxidized starting, running as pid #{$$}" Oxidized.new rescue => error crash error @@ -18,15 +19,16 @@ module Oxidized private def initialize - Log.info "Oxidized starting, running as pid #{$$} by #{ENV['USER']}" _args, @opts = parse_opts - Oxidized.config.debug = true if @opts[:debug] + + Config.load(@opts) + Oxidized.setup_logger + @pidfile = File.expand_path("pid") - Log.debug "pidfile: #{@pidfile}" end def crash error - Log.fatal "Oxidized crashed, crashfile written in #{Config::Crash}" + Oxidized.logger.fatal "Oxidized crashed, crashfile written in #{Config::Crash}" open Config::Crash, 'w' do |file| file.puts '-' * 50 file.puts Time.now.utc diff --git a/lib/oxidized/config.rb b/lib/oxidized/config.rb index 36c1a8f..c850059 100644 --- a/lib/oxidized/config.rb +++ b/lib/oxidized/config.rb @@ -12,7 +12,7 @@ module Oxidized HookDir = File.join Directory, %w(lib oxidized hook) Sleep = 1 - def self.load + def self.load(cmd_opts={}) asetus = Asetus.new(name: 'oxidized', load: false, key_to_s: true) Oxidized.asetus = asetus @@ -20,7 +20,7 @@ module Oxidized asetus.default.password = 'password' asetus.default.model = 'junos' asetus.default.interval = 3600 - asetus.default.log = File.join Config::Root, 'log' + asetus.default.use_syslog = false asetus.default.debug = false asetus.default.threads = 30 asetus.default.timeout = 20 @@ -48,9 +48,10 @@ module Oxidized raise InvalidConfig, "Error loading config: #{error.message}" end - Log.level = Logger::INFO unless asetus.cfg.debug raise NoConfig, 'edit ~/.config/oxidized/config' if asetus.create - Log.file = asetus.cfg.log if asetus.cfg.log + + # override if comand line flag given + asetus.cfg.debug = cmd_opts[:debug] if cmd_opts[:debug] asetus end diff --git a/lib/oxidized/core.rb b/lib/oxidized/core.rb index 1b918d1..d57c5cb 100644 --- a/lib/oxidized/core.rb +++ b/lib/oxidized/core.rb @@ -1,12 +1,4 @@ module Oxidized - require 'oxidized/log' - require 'oxidized/string' - require 'oxidized/config' - require 'oxidized/config/vars' - require 'oxidized/worker' - require 'oxidized/nodes' - require 'oxidized/manager' - require 'oxidized/hook' class << self def new *args Core.new args @@ -17,7 +9,6 @@ module Oxidized class NoNodesFound < OxidizedError; end def initialize args - Config.load Oxidized.mgr = Manager.new Oxidized.Hooks = HookManager.from_config(Oxidized.config) nodes = Nodes.new @@ -28,7 +19,8 @@ module Oxidized begin require 'oxidized/web' rescue LoadError - raise OxidizedError, 'oxidized-web not found: sudo gem install oxidized-web - or disable web support by setting "rest: false" in your configuration' + raise OxidizedError, 'oxidized-web not found: sudo gem install oxidized-web - \ + or disable web support by setting "rest: false" in your configuration' end @rest = API::Web.new nodes, Oxidized.config.rest @rest.run diff --git a/lib/oxidized/hook.rb b/lib/oxidized/hook.rb index d774653..029688d 100644 --- a/lib/oxidized/hook.rb +++ b/lib/oxidized/hook.rb @@ -46,7 +46,7 @@ class HookManager hook.cfg = cfg @registered_hooks[event] << RegisteredHook.new(name, hook) - Log.debug "Hook #{name.inspect} registered #{hook.class} for event #{event.inspect}" + Oxidized.logger.debug "Hook #{name.inspect} registered #{hook.class} for event #{event.inspect}" end def handle event, ctx_params={} @@ -57,7 +57,7 @@ class HookManager begin r_hook.hook.run_hook ctx rescue => e - Log.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " + + Oxidized.logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " + "(#{e.inspect}) for event #{event.inspect}" end end @@ -81,7 +81,7 @@ class Hook end def log(msg, level=:info) - Log.send(level, "#{self.class.name}: #{msg}") + Oxidized.logger.send(level, "#{self.class.name}: #{msg}") end end diff --git a/lib/oxidized/input/ftp.rb b/lib/oxidized/input/ftp.rb index a3d7ac8..93cdb38 100644 --- a/lib/oxidized/input/ftp.rb +++ b/lib/oxidized/input/ftp.rb @@ -28,7 +28,7 @@ module Oxidized end def cmd file - Log.debug "FTP: #{file} @ #{@node.name}" + Oxidized.logger.debug "FTP: #{file} @ #{@node.name}" @ftp.getbinaryfile file, nil end diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index e45adb6..476a786 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -43,7 +43,7 @@ module Oxidized end def cmd cmd, expect=node.prompt - Log.debug "SSH: #{cmd} @ #{node.name}" + Oxidized.logger.debug "SSH: #{cmd} @ #{node.name}" if @exec @ssh.exec! cmd else diff --git a/lib/oxidized/input/telnet.rb b/lib/oxidized/input/telnet.rb index b92d703..e9fd7d9 100644 --- a/lib/oxidized/input/telnet.rb +++ b/lib/oxidized/input/telnet.rb @@ -37,7 +37,7 @@ module Oxidized end def cmd cmd, expect=@node.prompt - Log.debug "Telnet: #{cmd} @#{@node.name}" + Oxidized.logger.debug "Telnet: #{cmd} @#{@node.name}" args = { 'String' => cmd } args.merge!({ 'Match' => expect, 'Timeout' => @timeout }) if expect @telnet.cmd args diff --git a/lib/oxidized/log.rb b/lib/oxidized/log.rb deleted file mode 100644 index 04a2d65..0000000 --- a/lib/oxidized/log.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Oxidized - require 'logger' - - class Logger < Logger - def initialize target=STDOUT - super target - end - def file= target - FileUtils.mkdir_p File.dirname(target) - @logdev = LogDevice.new target - end - end - - Log = Logger.new -end diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb index 1bbf6dc..c16e3d0 100644 --- a/lib/oxidized/node.rb +++ b/lib/oxidized/node.rb @@ -62,7 +62,7 @@ module Oxidized level = rescue_fail[resc] resc = " (rescued #{resc})" end - Log.send(level, '%s raised %s%s with msg "%s"' % [self.ip, err.class, resc, err.message]) + Oxidized.logger.send(level, '%s raised %s%s with msg "%s"' % [self.ip, err.class, resc, err.message]) return false rescue => err file = Oxidized::Config::Crash + '.' + self.ip.to_s @@ -72,7 +72,7 @@ module Oxidized fh.puts '-' * 50 fh.puts err.backtrace end - Log.error '%s raised %s with msg "%s", %s saved' % [self.ip, err.class, err.message, file] + Oxidized.logger.error '%s raised %s with msg "%s", %s saved' % [self.ip, err.class, err.message, file] return false end end diff --git a/lib/oxidized/nodes.rb b/lib/oxidized/nodes.rb index f0c5d56..0c02d7a 100644 --- a/lib/oxidized/nodes.rb +++ b/lib/oxidized/nodes.rb @@ -1,8 +1,8 @@ module Oxidized - require 'ipaddr' - require 'oxidized/node' - class Oxidized::NotSupported < OxidizedError; end - class Oxidized::NodeNotFound < OxidizedError; end + require 'ipaddr' + require 'oxidized/node' + class Oxidized::NotSupported < OxidizedError; end + class Oxidized::NodeNotFound < OxidizedError; end class Nodes < Array attr_accessor :source alias :put :unshift @@ -18,13 +18,13 @@ module Oxidized _node = Node.new node new.push _node rescue ModelNotFound => err - Log.error "node %s raised %s with message '%s'" % [node, err.class, err.message] + Oxidized.logger.error "node %s raised %s with message '%s'" % [node, err.class, err.message] rescue Resolv::ResolvError => err - Log.error "node %s is not resolvable, raised %s with message '%s'" % [node, err.class, err.message] + Oxidized.logger.error "node %s is not resolvable, raised %s with message '%s'" % [node, err.class, err.message] end end size == 0 ? replace(new) : update_nodes(new) - Log.info "Loaded #{size} nodes" + Oxidized.logger.info "Loaded #{size} nodes" end end @@ -148,9 +148,9 @@ module Oxidized end end end - + public - + def version node, group with_lock do i = find_node_index node @@ -159,7 +159,7 @@ module Oxidized output.version node, group end end - + def get_version node, group, oid with_lock do i = find_node_index node @@ -168,7 +168,7 @@ module Oxidized output.get_version node, group, oid end end - + def get_diff node, group, oid1, oid2 with_lock do i = find_node_index node diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb index b36929f..15977e2 100644 --- a/lib/oxidized/worker.rb +++ b/lib/oxidized/worker.rb @@ -14,7 +14,7 @@ module Oxidized ended.each { |job| process job } @jobs.work while @jobs.size < @jobs.want - Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}" + Oxidized.logger.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}" # ask for next node in queue non destructive way nextnode = @nodes.first unless nextnode.last.nil? @@ -41,7 +41,7 @@ module Oxidized msg += " with message '#{node.msg}'" if node.msg if node.output.new.store node.name, job.config, :msg => msg, :user => node.user, :group => node.group - Log.info "Configuration updated for #{node.group}/#{node.name}" + Oxidized.logger.info "Configuration updated for #{node.group}/#{node.name}" Oxidized.Hooks.handle :post_store, :node => node, :job => job end @@ -58,10 +58,10 @@ module Oxidized Oxidized.Hooks.handle :node_fail, :node => node, :job => job end - Log.warn msg + Oxidized.logger.warn msg end rescue NodeNotFound - Log.warn "#{node.name} not found, removed while collecting?" + Oxidized.logger.warn "#{node.name} not found, removed while collecting?" end end diff --git a/spec/githubrepo_spec.rb b/spec/githubrepo_spec.rb index a3b4953..5786d73 100644 --- a/spec/githubrepo_spec.rb +++ b/spec/githubrepo_spec.rb @@ -6,6 +6,7 @@ describe Oxidized::Node do before(:each) do Oxidized.asetus = Asetus.new Oxidized.config.output.git.repo = 'foo.git' + Oxidized.setup_logger @credentials = mock() -- cgit v1.2.1 From c2a52593a70a89582202d629842189b4a1448a15 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Sun, 13 Dec 2015 22:12:24 -0500 Subject: do not commit when there is no diff --- lib/oxidized/hook/githubrepo.rb | 17 ++++++-- spec/githubrepo_spec.rb | 89 ++++++++++++++++++++++++++++++----------- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/lib/oxidized/hook/githubrepo.rb b/lib/oxidized/hook/githubrepo.rb index 22ee8ea..d10b51e 100644 --- a/lib/oxidized/hook/githubrepo.rb +++ b/lib/oxidized/hook/githubrepo.rb @@ -15,13 +15,24 @@ class GithubRepo < Oxidized::Hook end def fetch_and_merge_remote(repo) - repo.fetch('origin', [repo.head.name], credentials: credentials) + result = repo.fetch('origin', [repo.head.name], credentials: credentials) + log result.inspect, :debug - their_branch = repo.branches["origin/master"] or return + unless result[:total_deltas] > 0 + log "nothing recieved after fetch", :debug + return + end + + their_branch = repo.branches["origin/master"] + + log "merging fetched branch #{their_branch.name}", :debug merge_index = repo.merge_commits(repo.head.target_id, their_branch.target_id) - log("Conflicts detected", :warn) if merge_index.conflicts? + if merge_index.conflicts? + log("Conflicts detected, skipping Rugged::Commit.create", :warn) + return + end Rugged::Commit.create(repo, { parents: [repo.head.target, their_branch.target], diff --git a/spec/githubrepo_spec.rb b/spec/githubrepo_spec.rb index 5786d73..9ad43e9 100644 --- a/spec/githubrepo_spec.rb +++ b/spec/githubrepo_spec.rb @@ -3,52 +3,95 @@ require 'rugged' require 'oxidized/hook/githubrepo' describe Oxidized::Node do + let(:credentials) { mock() } + let(:remote) { mock() } + let(:repo_head) { mock() } + let(:repo) { mock() } + let(:gr) { GithubRepo.new } + before(:each) do Oxidized.asetus = Asetus.new Oxidized.config.output.git.repo = 'foo.git' Oxidized.setup_logger + end - @credentials = mock() + describe "#fetch_and_merge_remote" do + before(:each) do + Oxidized.config.hooks.github_repo_hook.remote_repo = 'git@github.com:username/foo.git' + Rugged::Credentials::SshKeyFromAgent.expects(:new).with(username: 'git').returns(credentials) + repo_head.expects(:name).returns('refs/heads/master') + gr.cfg = Oxidized.config.hooks.github_repo_hook + end - remote = mock() - remote.expects(:url).returns('https://github.com/username/foo.git') - remote.expects(:push).with(['refs/heads/master'], credentials: @credentials).returns(true) + it "should not try to merge when there is no update in remote branch" do + repo.expects(:fetch).with('origin', ['refs/heads/master'], credentials: credentials).returns(Hash.new(0)) + repo.expects(:branches).never + repo.expects(:head).returns(repo_head) + gr.fetch_and_merge_remote(repo).must_equal nil + end + describe "when there is update considering conflicts" do + let(:merge_index) { mock() } + let(:their_branch) { mock() } - repo_head = mock() - repo_head.expects(:name).twice.returns('refs/heads/master') + before(:each) do + repo.expects(:fetch).with('origin', ['refs/heads/master'], credentials: credentials).returns({total_deltas: 1}) + their_branch.expects(:target_id).returns(1) + repo_head.expects(:target_id).returns(2) + repo.expects(:merge_commits).with(2, 1).returns(merge_index) + repo.expects(:branches).returns({"origin/master" => their_branch}) + end - repo = mock() - repo.expects(:path).returns('foo.git') - repo.expects(:remotes).returns({'origin' => remote}) - repo.expects(:head).twice.returns(repo_head) - repo.expects(:fetch).with('origin', ['refs/heads/master'], credentials: @credentials).returns(true) - repo.expects(:branches).returns({}) + it "should not try merging when there's conflict" do + repo.expects(:head).twice.returns(repo_head) + their_branch.expects(:name).returns("origin/master") + merge_index.expects(:conflicts?).returns(true) + Rugged::Commit.expects(:create).never + gr.fetch_and_merge_remote(repo).must_equal nil + end - Rugged::Repository.expects(:new).with('foo.git').returns(repo) + it "should merge when there is no conflict" do + repo.expects(:head).times(3).returns(repo_head) + their_branch.expects(:target).returns("their_target") + their_branch.expects(:name).twice.returns("origin/master") + repo_head.expects(:target).returns("our_target") + merge_index.expects(:write_tree).with(repo).returns("tree") + merge_index.expects(:conflicts?).returns(false) + Rugged::Commit.expects(:create).with(repo, { + parents: ["our_target", "their_target"], + tree: "tree", + message: "Merge remote-tracking branch 'origin/master'", + update_ref: "HEAD" + }).returns(1) + gr.fetch_and_merge_remote(repo).must_equal 1 + end + end end describe "#run_hook" do + before(:each) do + remote.expects(:url).returns('https://github.com/username/foo.git') + remote.expects(:push).with(['refs/heads/master'], credentials: credentials).returns(true) + repo_head.expects(:name).twice.returns('refs/heads/master') + repo.expects(:head).twice.returns(repo_head) + repo.expects(:path).returns('foo.git') + repo.expects(:remotes).returns({'origin' => remote}) + repo.expects(:fetch).with('origin', ['refs/heads/master'], credentials: credentials).returns(Hash.new(0)) + Rugged::Repository.expects(:new).with('foo.git').returns(repo) + end + it "will push to the remote repository using https" do Oxidized.config.hooks.github_repo_hook.remote_repo = 'https://github.com/username/foo.git' Oxidized.config.hooks.github_repo_hook.username = 'username' Oxidized.config.hooks.github_repo_hook.password = 'password' - - Rugged::Credentials::UserPassword.expects(:new).with(username: 'username', password: 'password').returns(@credentials) - - gr = GithubRepo.new + Rugged::Credentials::UserPassword.expects(:new).with(username: 'username', password: 'password').returns(credentials) gr.cfg = Oxidized.config.hooks.github_repo_hook - gr.run_hook(nil).must_equal true end it "will push to the remote repository using ssh" do Oxidized.config.hooks.github_repo_hook.remote_repo = 'git@github.com:username/foo.git' - - Rugged::Credentials::SshKeyFromAgent.expects(:new).with(username: 'git').returns(@credentials) - - gr = GithubRepo.new + Rugged::Credentials::SshKeyFromAgent.expects(:new).with(username: 'git').returns(credentials) gr.cfg = Oxidized.config.hooks.github_repo_hook - gr.run_hook(nil).must_equal true end end -- cgit v1.2.1 From 1ee130b83a28f822cdc3379c24d4965ceee86034 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 4 Jan 2016 10:24:46 -0500 Subject: merge version --- lib/oxidized/version.rb | 2 +- oxidized.gemspec | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/oxidized/version.rb b/lib/oxidized/version.rb index c2a0c0e..38772b7 100644 --- a/lib/oxidized/version.rb +++ b/lib/oxidized/version.rb @@ -1,3 +1,3 @@ module Oxidized - VERSION = '0.8.1' + VERSION = '0.9.0' end diff --git a/oxidized.gemspec b/oxidized.gemspec index 127a2de..83e31cd 100644 --- a/oxidized.gemspec +++ b/oxidized.gemspec @@ -5,11 +5,7 @@ require 'oxidized/version' Gem::Specification.new do |s| s.name = 'oxidized' -<<<<<<< HEAD s.version = Oxidized::VERSION -======= - s.version = '0.9.0' ->>>>>>> upstream/master s.licenses = %w( Apache-2.0 ) s.platform = Gem::Platform::RUBY s.authors = [ 'Saku Ytti', 'Samer Abdel-Hafez', 'Anton Aksola' ] -- cgit v1.2.1 From 52804b9f96a441c66cb957d001e71f48cc6a0811 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 4 Jan 2016 10:47:29 -0500 Subject: return back build related tasks --- Rakefile | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Rakefile b/Rakefile index 2dc5415..3a1be5e 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,14 @@ require 'bundler/gem_tasks' require 'rake/testtask' +gemspec = eval(File.read(Dir['*.gemspec'].first)) +file = [gemspec.name, gemspec.version].join('-') + '.gem' + +desc 'Validate gemspec' +task :gemspec do + gemspec.validate +end + desc 'Run minitest' task :test do Rake::TestTask.new do |t| @@ -11,4 +19,31 @@ task :test do end end +desc 'Build gem' +task :build do + system "gem build #{gemspec.name}.gemspec" + FileUtils.mkdir_p 'gems' + FileUtils.mv file, 'gems' +end + +desc 'Install gem' +task :install => :build do + system "sudo -Es sh -c \'umask 022; gem install gems/#{file}\'" +end + +desc 'Remove gems' +task :clean do + FileUtils.rm_rf 'gems' +end + +desc 'Tag the release' +task :tag do + system "git tag #{gemspec.version}" +end + +desc 'Push to rubygems' +task :push => :tag do + system "gem push gems/#{file}" +end + task default: :test -- cgit v1.2.1 From f339170c877ca296987d66c0c44223a8cad1d338 Mon Sep 17 00:00:00 2001 From: Elvin Efendi Date: Mon, 4 Jan 2016 10:55:27 -0500 Subject: use latest net-ssh --- Gemfile.lock | 10 +++++----- oxidized.gemspec | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bbdc4c8..b46ca2f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,9 @@ PATH remote: . specs: - oxidized (0.8.1) + oxidized (0.9.0) asetus (~> 0.1) - net-ssh (~> 2.8) + net-ssh (~> 3.0, >= 3.0.2) rugged (~> 0.21, >= 0.21.4) slop (~> 3.5) @@ -14,10 +14,10 @@ GEM coderay (1.1.0) metaclass (0.0.4) method_source (0.8.2) - minitest (5.8.2) + minitest (5.8.3) mocha (1.1.0) metaclass (~> 0.0.1) - net-ssh (2.9.2) + net-ssh (3.0.2) pry (0.10.3) coderay (~> 1.1.0) method_source (~> 0.8.1) @@ -38,4 +38,4 @@ DEPENDENCIES rake (~> 10.0) BUNDLED WITH - 1.10.6 + 1.11.2 diff --git a/oxidized.gemspec b/oxidized.gemspec index 83e31cd..808d508 100644 --- a/oxidized.gemspec +++ b/oxidized.gemspec @@ -21,7 +21,7 @@ Gem::Specification.new do |s| s.required_ruby_version = '>= 1.9.3' s.add_runtime_dependency 'asetus', '~> 0.1' s.add_runtime_dependency 'slop', '~> 3.5' - s.add_runtime_dependency 'net-ssh', '~> 2.9', '>= 2.9.3' + s.add_runtime_dependency 'net-ssh', '~> 3.0', '>= 3.0.2' s.add_runtime_dependency 'rugged', '~> 0.21', '>= 0.21.4' s.add_development_dependency 'pry', '~> 0' s.add_development_dependency 'bundler', '~> 1.10' -- cgit v1.2.1