diff options
| -rw-r--r-- | CHANGELOG.md | 19 | ||||
| -rw-r--r-- | README.md | 32 | ||||
| -rw-r--r-- | extra/oxidized.service | 12 | ||||
| -rw-r--r-- | lib/oxidized/input/ssh.rb | 4 | ||||
| -rw-r--r-- | lib/oxidized/model/panos.rb | 26 | ||||
| -rw-r--r-- | lib/oxidized/model/screenos.rb | 9 | ||||
| -rw-r--r-- | lib/oxidized/nodes.rb | 29 | ||||
| -rw-r--r-- | lib/oxidized/output/git.rb | 102 | ||||
| -rw-r--r-- | lib/oxidized/source/csv.rb | 2 | ||||
| -rw-r--r-- | lib/oxidized/source/http.rb | 54 | ||||
| -rw-r--r-- | lib/oxidized/worker.rb | 4 | ||||
| -rw-r--r-- | oxidized.gemspec | 4 |
12 files changed, 275 insertions, 22 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed1b85..296c67d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +# 0.7.0 +- FEATURE: support http source (by @laf) +- FEATURE: support Palo Alto PANOS (by @rixxxx) +- BUGFIX: screenos fixes (by @rixxxx) +- BUGFIX: allow 'none' auth in ssh (spotted by @SaldoorMike, needed by ciscosmb+aireos) + +# 0.6.0 +- FEATURE: support cumulus linux (by @FlorianDoublet) +- FEATURE: support HP Comware SMB siwtches (by @sid3windr) +- FEATURE: remove secret additions (by @rodecker) +- FEATURE: option to put all groups in single repo (by @ytti) +- FEATURE: expand path in source: csv: (so that ~/foo/bar works) (by @ytti) +- BUGFIX: screenos fixes (by @rixxxx) +- BUGFIX: ironware fixes (by @FlorianDoublet) +- BUGFIX: powerconnect fixes (by @sid3windr) +- BUGFIX: don't ask interactive password in new net/ssh (by @ytti) + # 0.5.0 - FEATURE: Mikrotik RouterOS model (by @emjemj) - FEATURE: add support for Cisco VSS (by @MrRJ45) @@ -22,7 +39,7 @@ - BUGFIX: allow node to be removed while it is being collected - BUGFIX: if model returns non string value, return empty string - BUGFIX: better prompt for Arista EOS model (by @rodecker) -- BUGFIX: improved configuration handling for Arista EOS model (by @rodecker) +- BUGFIX: improved configuration handling for Arista EOS model (by @rodecker) # 0.3.0 - FEATURE: *FIXME* bunch of stuff I did for richih, docs needed @@ -2,7 +2,7 @@ [](http://badge.fury.io/rb/oxidized) -Oxidized is a network device configuration backup tool. It's a RANCID replacment! +Oxidized is a network device configuration backup tool. It's a RANCID replacement! * automatically adds/removes threads to meet configured retrieval interval * restful API to move node immediately to head-of-queue (GET/POST /node/next/[NODE]) @@ -27,6 +27,7 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacment * [Privileged mode](#privileged-mode) * [Source: CSV](#source-csv) * [Source: SQLite](#source-sqlite) + * [Source: HTTP](#source-http) * [Output: GIT](#output-git) * [Output: File](#output-file) * [Advanced Configuration](#advanced-configuration) @@ -67,6 +68,7 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacment * Juniper ScreenOS (Netscreen) * Mikrotik RouterOS * Ubiquiti AirOS + * Palo Alto PAN-OS # Installation @@ -99,7 +101,7 @@ To initialize a default configuration in your home directory ```~/.config/oxidiz ## Source -Oxidized supports ```CSV``` and ```SQLite``` as source backends. The CSV backend reads nodes from a rancid compatible router.db file. The SQLite backend will fire queries against a database and map certain fields to model items. Take a look at the [Cookbook](#cookbook) for more details. +Oxidized supports ```CSV```, ```SQLite``` and ```HTTP``` as source backends. The CSV backend reads nodes from a rancid compatible router.db file. The SQLite backend will fire queries against a database and map certain fields to model items. The HTTP backend will fire queries against a http/https url. Take a look at the [Cookbook](#cookbook) for more details. ## Outputs @@ -113,7 +115,7 @@ mkdir ~/.config/oxidized/configs oxidized ``` -Now tell Oxidized where it finds a list of network devices to backup configuration from. You can either use CSV or SQLite as source. To create a CVS source add the following snippet: +Now tell Oxidized where it finds a list of network devices to backup configuration from. You can either use CSV or SQLite as source. To create a CSV source add the following snippet: ``` source: @@ -220,6 +222,28 @@ source: enable: enable ``` +### Source: HTTP + +One object per device. + +``` +source: + default: http + http: + url: https://url/api + scheme: https + delimiter: !ruby/regexp /:/ + map: + name: hostname + model: os + username: username + password: password + vars_map: + enable: enable + headers: + X-Auth-Token: 'somerandomstring' +``` + ### Output: File Parent directory needs to be created manually, one file per device, with most recent running config. @@ -313,7 +337,7 @@ The following objects exist in Oxidized. * input - method to acquire config, loaded dynamically as needed (Also default in config file) * output - method to store config, loaded dynamically as needed (Also default in config file) * prompt - prompt used for node (Also default in config file, can be specified in model too) - * 'sql' and 'csv' (supports any format with single entry per line, like router.db) + * 'sql', 'csv' and 'http' (supports any format with single entry per line, like router.db) ## Model * lists commands to gather from given device model diff --git a/extra/oxidized.service b/extra/oxidized.service new file mode 100644 index 0000000..65063b7 --- /dev/null +++ b/extra/oxidized.service @@ -0,0 +1,12 @@ +#For debian 8 put it in /lib/systemd/system/ +#and call it with systemctl start oxidized.service + +[Unit] +Description=Oxidized - Network Device Configuration Backup Tool + +[Service] +ExecStart=/usr/local/bin/oxidized +User=root + +[Install] +WantedBy=multi-user.target diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index ec33d37..b1f109b 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -23,7 +23,9 @@ module Oxidized @log = File.open(CFG.input.debug?.to_s + '-ssh', 'w') if CFG.input.debug? @ssh = Net::SSH.start @node.ip, @node.auth[:username], :password => @node.auth[:password], :timeout => CFG.timeout, - :paranoid => secure + :paranoid => secure, + :auth_methods => %w(none publickey password keyboard-interactive), + :number_of_password_prompts => 0 unless @exec shell_open @ssh begin diff --git a/lib/oxidized/model/panos.rb b/lib/oxidized/model/panos.rb new file mode 100644 index 0000000..35624b7 --- /dev/null +++ b/lib/oxidized/model/panos.rb @@ -0,0 +1,26 @@ +class PanOS < Oxidized::Model + + # PaloAlto PAN-OS model # + + comment '! ' + + prompt /^[\w.\@:\(\)-]+>\s?$/ + + cmd :all do |cfg| + cfg.each_line.to_a[2..-3].join + end + + cmd 'show system info' do |cfg| + cfg.gsub! /^(up)?time:\ .*\n/, '' + comment cfg + end + + cmd 'show config running' do |cfg| + cfg + end + + cfg :ssh do + post_login 'set cli pager off' + pre_logout 'exit' + end +end diff --git a/lib/oxidized/model/screenos.rb b/lib/oxidized/model/screenos.rb index 922f401..7ee9d9f 100644 --- a/lib/oxidized/model/screenos.rb +++ b/lib/oxidized/model/screenos.rb @@ -4,7 +4,7 @@ class ScreenOS < Oxidized::Model comment '! ' - prompt '/^([\w.-\(\)]+->\s?)$/' + prompt /^[\w.:\(\)-]+->\s?$/ cmd :all do |cfg| cfg.each_line.to_a[2..-2].join @@ -18,6 +18,8 @@ class ScreenOS < Oxidized::Model end cmd 'get system' do |cfg| + cfg.gsub! /^Date\ .*\n/, '' + cfg.gsub! /^Up\ .*\n/, '' comment cfg end @@ -33,7 +35,10 @@ class ScreenOS < Oxidized::Model cfg :telnet, :ssh do post_login 'set console page 0' - pre_logout 'exit' + pre_logout do + send "exit\n" + send "n" + end end end diff --git a/lib/oxidized/nodes.rb b/lib/oxidized/nodes.rb index cb2fbc5..d25fbe7 100644 --- a/lib/oxidized/nodes.rb +++ b/lib/oxidized/nodes.rb @@ -148,6 +148,35 @@ module Oxidized end end end + + public + + def version node, group + with_lock do + i = find_node_index node + output = self[i].output.new + raise Oxidized::NotSupported unless output.respond_to? :fetch + output.version node, group + end + end + + def get_version node, group, oid + with_lock do + i = find_node_index node + output = self[i].output.new + raise Oxidized::NotSupported unless output.respond_to? :fetch + output.get_version node, group, oid + end + end + + def get_diff node, group, oid1, oid2 + with_lock do + i = find_node_index node + output = self[i].output.new + raise Oxidized::NotSupported unless output.respond_to? :fetch + output.get_diff node, group, oid1, oid2 + end + end end end diff --git a/lib/oxidized/output/git.rb b/lib/oxidized/output/git.rb index e9256e8..46e748a 100644 --- a/lib/oxidized/output/git.rb +++ b/lib/oxidized/output/git.rb @@ -2,7 +2,6 @@ module Oxidized class Git < Output class GitError < OxidizedError; end begin - gem 'rugged', '~> 0.21.0' require 'rugged' rescue LoadError raise OxidizedError, 'rugged not found: sudo gem install rugged' @@ -62,23 +61,104 @@ class Git < Output 'node not found' end end + + #give a hash of all oid revision for the givin node, and the date of the commit + def version node, group + begin + repo = @cfg.repo + if group + repo = File.join File.dirname(repo), group + '.git' + end + repo = Rugged::Repository.new repo + walker = Rugged::Walker.new(repo) + walker.sorting(Rugged::SORT_DATE) + walker.push(repo.head.target) + i = -1 + tab = [] + walker.each do |commit| + if commit.diff(paths: [node]).size > 0 + hash = {} + hash[:date] = commit.time.to_s + hash[:oid] = commit.oid + tab[i += 1] = hash + end + end + walker.reset + tab + rescue + 'node not found' + end + end + + #give the blob of a specific revision + def get_version node, group, oid + begin + repo = @cfg.repo + if group && group != '' + repo = File.join File.dirname(repo), group + '.git' + end + repo = Rugged::Repository.new repo + repo.blob_at(oid,node).content + rescue + 'version not found' + end + end + + #give a hash with the patch of a diff between 2 revision and the stats (added and deleted lines) + def get_diff node, group, oid1, oid2 + begin + repo = @cfg.repo + diff_commits = nil + if group && group != '' + repo = File.join File.dirname(repo), group + '.git' + end + repo = Rugged::Repository.new repo + commit = repo.lookup(oid1) + #if the second revision is precised + if oid2 + commit_old = repo.lookup(oid2) + diff = repo.diff(commit_old, commit) + diff.each do |patch| + if /#{node}\s+/.match(patch.to_s.lines.first) + diff_commits = {:patch => patch.to_s, :stat => patch.stat} + break + end + end + #else gives the diffs between the first oid and his first parrent + else + stat = commit.parents[0].diff(commit).stat + stat = [stat[1],stat[2]] + patch = commit.parents[0].diff(commit).patch + diff_commits = {:patch => patch, :stat => stat} + end + diff_commits + rescue + 'no diffs' + end + end private def update repo, file, data return if data.empty? if @opt[:group] - repo = File.join File.dirname(repo), @opt[:group] + '.git' + if @cfg.single_repo? + file = File.join @opt[:group], file + else + repo = File.join File.dirname(repo), @opt[:group] + '.git' + end end - repo = Rugged::Repository.new repo - update_repo repo, file, data, @msg, @user, @email - rescue Rugged::OSError, Rugged::RepositoryError => open_error begin - Rugged::Repository.init_at repo, :bare - rescue => create_error - raise GitError, "first '#{open_error.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo" + repo = Rugged::Repository.new repo + update_repo repo, file, data, @msg, @user, @email + rescue Rugged::OSError, Rugged::RepositoryError => open_error + begin + Rugged::Repository.init_at repo, :bare + rescue => create_error + raise GitError, "first '#{open_error.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo" + end + retry end - retry end def update_repo repo, file, data, msg, user, email @@ -99,8 +179,10 @@ class Git < Output :parents => repo.empty? ? [] : [repo.head.target].compact, :update_ref => 'HEAD', ) + index.write + true end end end -end +end
\ No newline at end of file diff --git a/lib/oxidized/source/csv.rb b/lib/oxidized/source/csv.rb index 3b66e65..5064e5e 100644 --- a/lib/oxidized/source/csv.rb +++ b/lib/oxidized/source/csv.rb @@ -18,7 +18,7 @@ class CSV < Source def load nodes = [] - open(@cfg.file).each_line do |line| + open(File.expand_path @cfg.file).each_line do |line| next if line.match /^\s*#/ data = line.chomp.split @cfg.delimiter next if data.empty? diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb new file mode 100644 index 0000000..d2e3ea6 --- /dev/null +++ b/lib/oxidized/source/http.rb @@ -0,0 +1,54 @@ +module Oxidized +class HTTP < Source + def initialize + @cfg = CFG.source.http + super + end + + def setup + if @cfg.url.empty? + raise NoConfig, 'no source http url config, edit ~/.config/oxidized/config' + end + end + + require "net/http" + require "uri" + require "json" + + def load + nodes = [] + uri = URI.parse(@cfg.url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true if uri.scheme == 'https' + + # map headers + headers = {} + @cfg.headers.each do |header, value| + headers[header] = value + end + + request = Net::HTTP::Get.new(uri.request_uri, headers) + + response = http.request(request) + data = JSON.parse(response.body) + data.each do |line| + next if line.empty? + # map node parameters + keys = {} + @cfg.map.each do |key, position| + keys[key.to_sym] = line[position] + end + keys[:model] = map_model keys[:model] if keys.key? :model + + # map node specific vars, empty value is considered as nil + vars = {} + @cfg.vars_map.each { |key, position| vars[key.to_sym] = line[position].to_s.empty? ? nil : line[position] } + keys[:vars] = vars unless vars.empty? + + nodes << keys + end + nodes + end + +end +end diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb index 7ed70ac..6bb2a22 100644 --- a/lib/oxidized/worker.rb +++ b/lib/oxidized/worker.rb @@ -37,8 +37,10 @@ module Oxidized msg = "update #{node.name}" msg += " from #{node.from}" if node.from msg += " with message '#{node.msg}'" if node.msg - node.output.new.store node.name, job.config, + if node.output.new.store node.name, job.config, :msg => msg, :user => node.user, :group => node.group + Log.info "Configuration updated for #{node.group}/#{node.name}" + end node.reset else msg = "#{node.name} status #{job.status}" diff --git a/oxidized.gemspec b/oxidized.gemspec index 49343e6..ae069cf 100644 --- a/oxidized.gemspec +++ b/oxidized.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'oxidized' - s.version = '0.5.0' + s.version = '0.7.2' s.licenses = %w( Apache-2.0 ) s.platform = Gem::Platform::RUBY s.authors = [ 'Saku Ytti', 'Samer Abdel-Hafez' ] @@ -17,5 +17,5 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'asetus', '~> 0.1' s.add_runtime_dependency 'slop', '~> 3.5' s.add_runtime_dependency 'net-ssh', '~> 2.8' - s.add_runtime_dependency 'rugged', '~> 0.21.4' + s.add_runtime_dependency 'rugged', '~> 0.21', '>= 0.21.4' end |