diff options
| -rw-r--r-- | Gemfile.lock | 1 | ||||
| -rw-r--r-- | README.md | 69 | ||||
| -rw-r--r-- | lib/oxidized/hook/slackdiff.rb | 6 | ||||
| -rw-r--r-- | lib/oxidized/model/firewareos.rb | 2 | ||||
| -rw-r--r-- | lib/oxidized/model/fortios.rb | 2 | ||||
| -rw-r--r-- | lib/oxidized/model/ios.rb | 15 | ||||
| -rw-r--r-- | lib/oxidized/model/mlnxos.rb | 2 | ||||
| -rw-r--r-- | lib/oxidized/node.rb | 36 | ||||
| -rw-r--r-- | lib/oxidized/output/gitcrypt.rb | 244 | ||||
| -rw-r--r-- | oxidized.gemspec | 1 |
10 files changed, 357 insertions, 21 deletions
diff --git a/Gemfile.lock b/Gemfile.lock index ab52715..2c887c1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,7 @@ PLATFORMS DEPENDENCIES bundler (~> 1.10) + git (~> 1) minitest (~> 5.8) mocha (~> 1.1) oxidized! @@ -32,6 +32,7 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacemen * [Source: Mysql](#source-mysql) * [Source: HTTP](#source-http) * [Output: GIT](#output-git) + * [Output: GIT-Crypt](#output-git-crypt) * [Output: HTTP](#output-http) * [Output: File](#output-file) * [Output types](#output-types) @@ -236,7 +237,7 @@ Oxidized supports ```CSV```, ```SQLite``` and ```HTTP``` as source backends. The ## Outputs -Possible outputs are either ```file``` or ```git```. The file backend takes a destination directory as argument and will keep a file per device, with most recent running version of a device. The GIT backend (recommended) will initialize an empty GIT repository in the specified path and create a new commit on every configuration change. Take a look at the [Cookbook](#cookbook) for more details. +Possible outputs are either ```file```, ```git``` or ```git-crypt```. The file backend takes a destination directory as argument and will keep a file per device, with most recent running version of a device. The GIT backend (recommended) will initialize an empty GIT repository in the specified path and create a new commit on every configuration change. The GIT-Crypt backend will also initialize a GIT repository but every configuration push to it will be encrypted on the fly by using ```git-crypt``` tool. Take a look at the [Cookbook](#cookbook) for more details. Maps define how to map a model's fields to model [model fields](https://github.com/ytti/oxidized/tree/master/lib/oxidized/model). Most of the settings should be self explanatory, log is ignored if `use_syslog`(requires Ruby >= 2.0) is set to `true`. @@ -604,6 +605,72 @@ output: ``` +### Output: Git-Crypt + +This uses the gem git and system git-crypt interfaces. Have a look at [GIT-Crypt](https://www.agwa.name/projects/git-crypt/) documentation to know how to install it. +Additionally to user and email informations, you have to provide the users ID that can be a key ID, a full fingerprint, an email address, or anything else that uniquely identifies a public key to GPG (see "HOW TO SPECIFY A USER ID" in the gpg man page). + + +For a single repositories for all devices: + +``` yaml +output: + default: gitcrypt + gitcrypt: + user: Oxidized + email: o@example.com + repo: "/var/lib/oxidized/devices" + users: + - "0x0123456789ABCDEF" + - "<user@example.com>" +``` + +And for groups repositories: + +``` yaml +output: + default: gitcrypt + gitcrypt: + user: Oxidized + email: o@example.com + repo: "/var/lib/oxidized/git-repos/default" + users: + - "0xABCDEF0123456789" + - "0x0123456789ABCDEF" +``` + +Oxidized will create a repository for each group in the same directory as the `default`. For +example: + +``` csv +host1:ios:first +host2:nxos:second +``` + +This will generate the following repositories: + +``` bash +$ ls /var/lib/oxidized/git-repos + +default.git first.git second.git +``` + +If you would like to use groups and a single repository, you can force this with the `single_repo` config. + +``` yaml +output: + default: gitcrypt + gitcrypt: + single_repo: true + repo: "/var/lib/oxidized/devices" + users: + - "0xABCDEF0123456789" + - "0x0123456789ABCDEF" + +``` + +Please note that user list is only updated once at creation. + ### Output: Http POST a config to the specified URL diff --git a/lib/oxidized/hook/slackdiff.rb b/lib/oxidized/hook/slackdiff.rb index b37c9c4..61f1743 100644 --- a/lib/oxidized/hook/slackdiff.rb +++ b/lib/oxidized/hook/slackdiff.rb @@ -10,7 +10,11 @@ class SlackDiff < Oxidized::Hook if ctx.node if ctx.event.to_s == "post_store" log "Connecting to slack" - client = Slack::Client.new token: cfg.token + Slack.configure do |config| + config.token = cfg.token + config.proxy = cfg.proxy if cfg.has_key?('proxy') + end + client = Slack::Client.new client.auth_test log "Connected" gitoutput = ctx.node.output.new diff --git a/lib/oxidized/model/firewareos.rb b/lib/oxidized/model/firewareos.rb index 3b24f89..5d51afc 100644 --- a/lib/oxidized/model/firewareos.rb +++ b/lib/oxidized/model/firewareos.rb @@ -1,6 +1,6 @@ class FirewareOS < Oxidized::Model - prompt /^\[?\w*\]?\w*<?\w*>?#\s*$/ + prompt /^\[?\w*\]?\w*<?\w*>?(#|>)\s*$/ comment '-- ' cmd :all do |cfg| diff --git a/lib/oxidized/model/fortios.rb b/lib/oxidized/model/fortios.rb index 2415537..cdaa282 100644 --- a/lib/oxidized/model/fortios.rb +++ b/lib/oxidized/model/fortios.rb @@ -15,7 +15,7 @@ class FortiOS < Oxidized::Model end cmd :secret do |cfg| - cfg.gsub! /(set (?:passwd|password|psksecret)).*/, '\\1 <configuration removed>' + cfg.gsub! /(set (?:passwd|password|psksecret|secret|key ENC)).*/, '\\1 <configuration removed>' cfg.gsub! /(set private-key).*-+END ENCRYPTED PRIVATE KEY-*"$/m , '\\1 <configuration removed>' cfg end diff --git a/lib/oxidized/model/ios.rb b/lib/oxidized/model/ios.rb index 0596f90..c3d5543 100644 --- a/lib/oxidized/model/ios.rb +++ b/lib/oxidized/model/ios.rb @@ -26,14 +26,15 @@ class IOS < Oxidized::Model cmd :secret do |cfg| cfg.gsub! /^(snmp-server community).*/, '\\1 <configuration removed>' - cfg.gsub! /username (\S+) privilege (\d+) (\S+).*/, '<secret hidden>' - cfg.gsub! /^username \S+ password \d \S+/, '<secret hidden>' - cfg.gsub! /^username \S+ secret \d \S+/, '<secret hidden>' - cfg.gsub! /^enable (password|secret) \d \S+/, '<secret hidden>' + cfg.gsub! /^(username \S+ privilege \d+) (\S+).*/, '\\1 <secret hidden>' + cfg.gsub! /^(username \S+ password \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(username \S+ secret \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(enable (password|secret) \d) (\S+)/, '\\1 <secret hidden>' cfg.gsub! /^(\s+(?:password|secret)) (?:\d )?\S+/, '\\1 <secret hidden>' - cfg.gsub! /wpa-psk ascii \d \S+/, '<secret hidden>' - cfg.gsub! /key 7 \d.+/, '<secret hidden>' - cfg.gsub! /^tacacs-server key \d \S+/, '<secret hidden>' + cfg.gsub! /^(.*wpa-psk ascii \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(.*key 7) (\d.+)/, '\\1 <secret hidden>' + cfg.gsub! /^(tacacs-server key \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(crypto isakmp key) (\S+) (.*)/, '\\1 <secret hidden> \\3' cfg end diff --git a/lib/oxidized/model/mlnxos.rb b/lib/oxidized/model/mlnxos.rb index 97e7098..49f3369 100644 --- a/lib/oxidized/model/mlnxos.rb +++ b/lib/oxidized/model/mlnxos.rb @@ -42,6 +42,6 @@ class MLNXOS < Oxidized::Model cfg :ssh do password /^Password:\s*/ - pre_logout 'exit' + pre_logout "\nexit" end end diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb index 6f89b56..cf71e48 100644 --- a/lib/oxidized/node.rb +++ b/lib/oxidized/node.rb @@ -166,18 +166,32 @@ module Oxidized end def resolve_repo opt - return unless is_git? opt - - remote_repo = Oxidized.config.output.git.repo - - if remote_repo.is_a?(::String) - if Oxidized.config.output.git.single_repo? || @group.nil? - remote_repo + if is_git? opt + remote_repo = Oxidized.config.output.git.repo + + if remote_repo.is_a?(::String) + if Oxidized.config.output.git.single_repo? || @group.nil? + remote_repo + else + File.join(File.dirname(remote_repo), @group + '.git') + end + else + remote_repo[@group] + end + elsif is_gitcrypt? opt + remote_repo = Oxidized.config.output.gitcrypt.repo + + if remote_repo.is_a?(::String) + if Oxidized.config.output.gitcrypt.single_repo? || @group.nil? + remote_repo + else + File.join(File.dirname(remote_repo), @group + '.git') + end else - File.join(File.dirname(remote_repo), @group + '.git') + remote_repo[@group] end else - remote_repo[@group] + return end end @@ -212,5 +226,9 @@ module Oxidized (opt[:output] || Oxidized.config.output.default) == 'git' end + def is_gitcrypt? opt + (opt[:output] || Oxidized.config.output.default) == 'gitcrypt' + end + end end diff --git a/lib/oxidized/output/gitcrypt.rb b/lib/oxidized/output/gitcrypt.rb new file mode 100644 index 0000000..b0d80f2 --- /dev/null +++ b/lib/oxidized/output/gitcrypt.rb @@ -0,0 +1,244 @@ +module Oxidized + class GitCrypt < Output + class GitCryptError < OxidizedError; end + begin + require 'git' + rescue LoadError + raise OxidizedError, 'git not found: sudo gem install ruby-git' + end + + attr_reader :commitref + + def initialize + @cfg = Oxidized.config.output.gitcrypt + @gitcrypt_cmd = "/usr/bin/git-crypt" + @gitcrypt_init = @gitcrypt_cmd + " init" + @gitcrypt_unlock = @gitcrypt_cmd + " unlock" + @gitcrypt_lock = @gitcrypt_cmd + " lock" + @gitcrypt_adduser = @gitcrypt_cmd + " add-gpg-user --trusted " + end + + def setup + if @cfg.empty? + Oxidized.asetus.user.output.gitcrypt.user = 'Oxidized' + Oxidized.asetus.user.output.gitcrypt.email = 'o@example.com' + Oxidized.asetus.user.output.gitcrypt.repo = File.join(Config::Root, 'oxidized.git') + Oxidized.asetus.save :user + raise NoConfig, 'no output git config, edit ~/.config/oxidized/config' + end + + if @cfg.repo.respond_to?(:each) + @cfg.repo.each do |group, repo| + @cfg.repo["#{group}="] = File.expand_path repo + end + else + @cfg.repo = File.expand_path @cfg.repo + end + end + + def crypt_init repo + repo.chdir do + system(@gitcrypt_init) + @cfg.users.each do |user| + system("#{@gitcrypt_adduser} #{user}") + end + File.write(".gitattributes", "* filter=git-crypt diff=git-crypt\n.gitattributes !filter !diff") + repo.add(".gitattributes") + repo.commit("Initial commit: crypt all config files") + end + end + + def lock repo + repo.chdir do + system(@gitcrypt_lock) + end + end + + def unlock repo + repo.chdir do + system(@gitcrypt_unlock) + end + end + + def store file, outputs, opt={} + @msg = opt[:msg] + @user = (opt[:user] or @cfg.user) + @email = (opt[:email] or @cfg.email) + @opt = opt + @commitref = nil + repo = @cfg.repo + + outputs.types.each do |type| + type_cfg = '' + type_repo = File.join(File.dirname(repo), type + '.git') + outputs.type(type).each do |output| + (type_cfg << output; next) if not output.name + type_file = file + '--' + output.name + if @cfg.type_as_directory? + type_file = type + '/' + type_file + type_repo = repo + end + update type_repo, type_file, output + end + update type_repo, file, type_cfg + end + + update repo, file, outputs.to_cfg + end + + + def fetch node, group + begin + repo, path = yield_repo_and_path(node, group) + repo = Git.open repo + unlock repo + index = repo.index + # Empty repo ? + empty = File.exists? index.path + if empty + raise 'Empty git repo' + else + File.read path + end + lock repo + rescue + 'node not found' + end + end + + # give a hash of all oid revision for the given node, and the date of the commit + def version node, group + begin + repo, path = yield_repo_and_path(node, group) + + repo = Git.open repo + unlock repo + walker = repo.log.path(path) + i = -1 + tab = [] + walker.each do |commit| + hash = {} + hash[:date] = commit.date.to_s + hash[:oid] = commit.objectish + hash[:author] = commit.author + hash[:message] = commit.message + tab[i += 1] = hash + end + walker.reset + tab + rescue + 'node not found' + end + end + + #give the blob of a specific revision + def get_version node, group, oid + begin + repo, path = yield_repo_and_path(node, group) + repo = Git.open repo + unlock repo + repo.gtree(oid).files[path].contents + rescue + 'version not found' + ensure + lock repo + end + end + + #give a hash with the patch of a diff between 2 revision and the stats (added and deleted lines) + def get_diff node, group, oid1, oid2 + begin + diff_commits = nil + repo, path = yield_repo_and_path(node, group) + repo = Git.open repo + unlock repo + commit = repo.gcommit(oid1) + + if oid2 + commit_old = repo.gcommit(oid2) + diff = repo.diff(commit_old, commit) + stats = [diff.stats[:files][node.name][:insertions], diff.stats[:files][node.name][:deletions]] + diff.each do |patch| + if /#{node.name}\s+/.match(patch.patch.to_s.lines.first) + diff_commits = {:patch => patch.patch.to_s, :stat => stats} + break + end + end + else + stat = commit.parents[0].diff(commit).stats + stat = [stat[:files][node.name][:insertions],stat[:files][node.name][:deletions]] + patch = commit.parents[0].diff(commit).patch + diff_commits = {:patch => patch, :stat => stat} + end + lock repo + diff_commits + rescue + 'no diffs' + ensure + lock repo + end + end + + private + + def yield_repo_and_path(node, group) + repo, path = node.repo, node.name + + if group and @cfg.single_repo? + path = "#{group}/#{node.name}" + end + + [repo, path] + end + + def update repo, file, data + return if data.empty? + + if @opt[:group] + if @cfg.single_repo? + file = File.join @opt[:group], file + else + repo = if repo.is_a?(::String) + File.join File.dirname(repo), @opt[:group] + '.git' + else + repo[@opt[:group]] + end + end + end + + begin + update_repo repo, file, data, @msg, @user, @email + rescue Git::GitExecuteError, ArgumentError => open_error + Oxidized.logger.debug "open_error #{open_error} #{file}" + begin + grepo = Git.init repo + crypt_init grepo + rescue => create_error + raise GitCryptError, "first '#{open_error.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo" + end + retry + end + end + + def update_repo repo, file, data, msg, user, email + grepo = Git.open repo + grepo.config('user.name', user) + grepo.config('user.email', email) + grepo.chdir do + unlock grepo + File.write(file, data) + grepo.add(file) + if grepo.status[file].nil? + grepo.commit(msg) + @commitref = grepo.log(1).first.objectish + true + elsif !grepo.status[file].type.nil? + grepo.commit(msg) + @commitref = grepo.log(1).first.objectish + true + end + lock grepo + end + end + end +end diff --git a/oxidized.gemspec b/oxidized.gemspec index 78324d4..ea088e7 100644 --- a/oxidized.gemspec +++ b/oxidized.gemspec @@ -33,4 +33,5 @@ Gem::Specification.new do |s| s.add_development_dependency 'rake', '~> 10.0' s.add_development_dependency 'minitest', '~> 5.8' s.add_development_dependency 'mocha', '~> 1.1' + s.add_development_dependency 'git', '~> 1' end |