diff options
41 files changed, 956 insertions, 69 deletions
@@ -3,9 +3,23 @@ MAINTAINER Samer Abdel-Hafez <sam@arahant.net> RUN add-apt-repository ppa:brightbox/ruby-ng && \ apt-get update && \ - apt-get install -y ruby2.3 ruby2.3-dev libsqlite3-dev libssl-dev pkg-config make cmake libssh2-1-dev + apt-get install -y ruby2.3 ruby2.3-dev libsqlite3-dev libssl-dev pkg-config make cmake libssh2-1-dev git g++ -RUN gem install oxidized oxidized-web --no-ri --no-rdoc +RUN mkdir -p /tmp/oxidized +COPY . /tmp/oxidized/ +WORKDIR /tmp/oxidized + +RUN gem build oxidized.gemspec +RUN gem install oxidized-*.gem + +# web interface +RUN gem install oxidized-web --no-ri --no-rdoc + +# dependencies for hooks +RUN gem install aws-sdk +RUN gem install slack-api + +RUN rm -rf /tmp/oxidized RUN apt-get remove -y ruby-dev pkg-config make cmake @@ -13,6 +27,7 @@ RUN apt-get -y autoremove ADD extra/oxidized.runit /etc/service/oxidized/run ADD extra/auto-reload-config.runit /etc/service/auto-reload-config/run +ADD extra/update-ca-certificates.runit /etc/service/update-ca-certificates/run VOLUME ["/root/.config/oxidized"] EXPOSE 8888/tcp diff --git a/Gemfile.lock b/Gemfile.lock index ab52715..2c887c1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,7 @@ PLATFORMS DEPENDENCIES bundler (~> 1.10) + git (~> 1) minitest (~> 5.8) mocha (~> 1.1) oxidized! @@ -19,6 +19,7 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacemen 2. [Installation](#installation) * [Debian](#debian) * [CentOS, Oracle Linux, Red Hat Linux](#centos-oracle-linux-red-hat-linux) + * [BSD](#freebsd) 3. [Initial Configuration](#configuration) 4. [Installing Ruby 2.1.2 using RVM](#installing-ruby-2.1.2-using-rvm) 5. [Running with Docker](#running-with-docker) @@ -32,6 +33,7 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacemen * [Source: Mysql](#source-mysql) * [Source: HTTP](#source-http) * [Output: GIT](#output-git) + * [Output: GIT-Crypt](#output-git-crypt) * [Output: HTTP](#output-http) * [Output: File](#output-file) * [Output types](#output-types) @@ -75,13 +77,14 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacemen * Check Point * [GaiaOS](lib/oxidized/model/gaiaos.rb) * Ciena - * [SOAS](lib/oxidized/model/saos.rb) + * [SAOS](lib/oxidized/model/saos.rb) * Cisco * [AireOS](lib/oxidized/model/aireos.rb) * [ASA](lib/oxidized/model/asa.rb) * [CatOS](lib/oxidized/model/catos.rb) * [IOS](lib/oxidized/model/ios.rb) * [IOSXR](lib/oxidized/model/iosxr.rb) + * [NGA](lib/oxidized/model/cisconga.rb) * [NXOS](lib/oxidized/model/nxos.rb) * [SMB (Nikola series)](lib/oxidized/model/ciscosmb.rb) * Citrix @@ -125,6 +128,7 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacemen * [ScreenOS (Netscreen)](lib/oxidized/model/screenos.rb) * Mellanox * [MLNX-OS](lib/oxidized/model/mlnxos.rb) + * [Voltaire](lib/oxidized/model/voltaire.rb) * Mikrotik * [RouterOS](lib/oxidized/model/routeros.rb) * Motorola @@ -138,6 +142,8 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacemen * [WISP Switch (As Netonix)](lib/oxidized/model/netonix.rb) * Nokia (formerly TiMetra, Alcatel, Alcatel-Lucent) * [SR OS (TiMOS)](lib/oxidized/model/timos.rb) + * OneAccess + * [OneOS](lib/oxidized/model/oneos.rb) * Opengear * [Opengear](lib/oxidized/model/opengear.rb) * Palo Alto @@ -146,10 +152,14 @@ Oxidized is a network device configuration backup tool. It's a RANCID replacemen * [pfSense](lib/oxidized/model/pfsense.rb) * Quanta * [Quanta / VxWorks 6.6 (1.1.0.8)](lib/oxidized/model/quantaos.rb) + * Siklu + * [EtherHaul](lib/oxidized/model/siklu.rb) * Supermicro * [Supermicro](lib/oxidized/model/supermicro.rb) * Trango Systems * [Trango](lib/oxidized/model/trango.rb) + * TPLink + * [TPLink](lib/oxidized/model/tplink.rb) * Ubiquiti * [AirOS](lib/oxidized/model/airos.rb) * [Edgeos](lib/oxidized/model/edgeos.rb) @@ -190,6 +200,19 @@ gem install oxidized gem install oxidized-script oxidized-web ``` +## FreeBSD +Use RVM to install Ruby v2.1.2 + +Install all required packages and gems. + +```shell +pkg install cmake pkgconf +gem install oxidized +gem install oxidized-script oxidized-web +``` + + + ## Build from Git ```shell git clone https://github.com/ytti/oxidized.git @@ -233,7 +256,7 @@ Oxidized supports ```CSV```, ```SQLite``` and ```HTTP``` as source backends. The ## Outputs -Possible outputs are either ```file``` or ```git```. The file backend takes a destination directory as argument and will keep a file per device, with most recent running version of a device. The GIT backend (recommended) will initialize an empty GIT repository in the specified path and create a new commit on every configuration change. Take a look at the [Cookbook](#cookbook) for more details. +Possible outputs are either ```file```, ```git``` or ```git-crypt```. The file backend takes a destination directory as argument and will keep a file per device, with most recent running version of a device. The GIT backend (recommended) will initialize an empty GIT repository in the specified path and create a new commit on every configuration change. The GIT-Crypt backend will also initialize a GIT repository but every configuration push to it will be encrypted on the fly by using ```git-crypt``` tool. Take a look at the [Cookbook](#cookbook) for more details. Maps define how to map a model's fields to model [model fields](https://github.com/ytti/oxidized/tree/master/lib/oxidized/model). Most of the settings should be self explanatory, log is ignored if `use_syslog`(requires Ruby >= 2.0) is set to `true`. @@ -359,6 +382,12 @@ If you want to have the config automatically reloaded (e.g. when using a http so docker run -v /etc/oxidized:/root/.config/oxidized -p 8888:8888/tcp -e CONFIG_RELOAD_INTERVAL=3600 -t oxidized/oxidized:latest ``` +If you need to use an internal CA (e.g. to connect to an private github instance) + +``` +docker run -v /etc/oxidized:/root/.config/oxidized -v /path/to/MY-CA.crt:/usr/local/share/ca-certificates/MY-CA.crt -p 8888:8888/tcp -e UPDATE_CA_CERTIFICATES=true -t oxidized/oxidized:latest +``` + ## Cookbook ### Debugging In case a model plugin doesn't work correctly (ios, procurve, etc.), you can enable live debugging of SSH/Telnet sessions. Just add a ```debug``` option containing the value true to the ```input``` section. The log files will be created depending on the parent directory of the logfile option. @@ -601,6 +630,72 @@ output: ``` +### Output: Git-Crypt + +This uses the gem git and system git-crypt interfaces. Have a look at [GIT-Crypt](https://www.agwa.name/projects/git-crypt/) documentation to know how to install it. +Additionally to user and email informations, you have to provide the users ID that can be a key ID, a full fingerprint, an email address, or anything else that uniquely identifies a public key to GPG (see "HOW TO SPECIFY A USER ID" in the gpg man page). + + +For a single repositories for all devices: + +``` yaml +output: + default: gitcrypt + gitcrypt: + user: Oxidized + email: o@example.com + repo: "/var/lib/oxidized/devices" + users: + - "0x0123456789ABCDEF" + - "<user@example.com>" +``` + +And for groups repositories: + +``` yaml +output: + default: gitcrypt + gitcrypt: + user: Oxidized + email: o@example.com + repo: "/var/lib/oxidized/git-repos/default" + users: + - "0xABCDEF0123456789" + - "0x0123456789ABCDEF" +``` + +Oxidized will create a repository for each group in the same directory as the `default`. For +example: + +``` csv +host1:ios:first +host2:nxos:second +``` + +This will generate the following repositories: + +``` bash +$ ls /var/lib/oxidized/git-repos + +default.git first.git second.git +``` + +If you would like to use groups and a single repository, you can force this with the `single_repo` config. + +``` yaml +output: + default: gitcrypt + gitcrypt: + single_repo: true + repo: "/var/lib/oxidized/devices" + users: + - "0xABCDEF0123456789" + - "0x0123456789ABCDEF" + +``` + +Please note that user list is only updated once at creation. + ### Output: Http POST a config to the specified URL @@ -745,6 +840,16 @@ map: group: 2 ``` +### Triggered backups + +A node can be moved to head-of-queue via the REST API `GET/POST /node/next/[NODE]`. + +In the default configuration this node will be processed when the next job worker becomes available, it could take some time if existing backups are in progress. To execute moved jobs immediately a new job can be added: + +``` +next_adds_job: true +``` + # Hooks You can define arbitrary number of hooks that subscribe different events. The hook system is modular and different kind of hook types can be enabled. @@ -864,6 +969,57 @@ AWS SNS hook requires the following configuration keys: Your AWS credentials should be stored in `~/.aws/credentials`. +## Hook type: slackdiff + +The `slackdiff` hook posts colorized config diffs to a [Slack](http://www.slack.com) channel of your choice. It only triggers for `post_store` events. + +You will need to manually install the `slack-api` gem on your system: + +``` +gem install slack-api +``` + +Configuration example: + +``` yaml +hooks: + slack: + type: slackdiff + events: [post_store] + token: SLACK_BOT_TOKEN + channel: "#network-changes" +``` + +# Extra + +## Ubuntu SystemV init setup + +The init script assumes that you have a used named 'oxidized' and that oxidized is in one of the following paths: + +``` +/sbin +/bin +/usr/sbin +/usr/bin +/usr/local/bin +``` + +1.)Copy init script from extra/ folder to /etc/init.d/oxidized +2.)Setup /var/run/ + +``` +mkdir /var/run/oxidized +chown oxidized:oxidized /var/run/oxidized +``` + +3.)Make oxidized start on boot + +``` +update-rc.d oxidized deafults +``` + +Note the channel name must be in quotes. + # Ruby API The following objects exist in Oxidized. diff --git a/extra/auto-reload-config.runit b/extra/auto-reload-config.runit index 5eaecc8..cf6e7c2 100755 --- a/extra/auto-reload-config.runit +++ b/extra/auto-reload-config.runit @@ -2,7 +2,7 @@ if [ -z "$CONFIG_RELOAD_INTERVAL" ]; then # Just stop and do nothing - read + sleep infinity fi while true; do diff --git a/extra/oxidized.init b/extra/oxidized.init index 197c5b1..7a9698d 100755 --- a/extra/oxidized.init +++ b/extra/oxidized.init @@ -14,12 +14,12 @@ set -e -PATH=/sbin:/bin:/usr/sbin:/usr/bin -DAEMON=/home/sts/oxidized/bin/oxidized +PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin +DAEMON=$(which oxidized) NAME="oxidized" DESC="Oxidized - Network Device Configuration Backup Tool" ARGS="" -USER="sts" +USER="oxidized" test -x $DAEMON || exit 0 diff --git a/extra/update-ca-certificates.runit b/extra/update-ca-certificates.runit new file mode 100755 index 0000000..53efdb9 --- /dev/null +++ b/extra/update-ca-certificates.runit @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ "$UPDATE_CA_CERTIFICATES" == "true" ]; then + update-ca-certificates +fi + +sleep infinity diff --git a/lib/oxidized/config.rb b/lib/oxidized/config.rb index b6b5c40..aba8b63 100644 --- a/lib/oxidized/config.rb +++ b/lib/oxidized/config.rb @@ -28,8 +28,9 @@ module Oxidized asetus.default.retries = 3 asetus.default.prompt = /^([\w.@-]+[#>]\s?)$/ asetus.default.rest = '127.0.0.1:8888' # or false to disable - asetus.default.vars = {} # could be 'enable'=>'enablePW' - asetus.default.groups = {} # group level configuration + asetus.default.next_adds_job = false # if true, /next adds job, so device is fetched immmeiately + asetus.default.vars = {} # could be 'enable'=>'enablePW' + asetus.default.groups = {} # group level configuration asetus.default.pid = File.join(Oxidized::Config::Root, 'pid') asetus.default.input.default = 'ssh, telnet' diff --git a/lib/oxidized/hook/slackdiff.rb b/lib/oxidized/hook/slackdiff.rb new file mode 100644 index 0000000..61f1743 --- /dev/null +++ b/lib/oxidized/hook/slackdiff.rb @@ -0,0 +1,34 @@ +require 'slack' + +class SlackDiff < Oxidized::Hook + def validate_cfg! + raise KeyError, 'hook.token is required' unless cfg.has_key?('token') + raise KeyError, 'hook.channel is required' unless cfg.has_key?('channel') + end + + def run_hook(ctx) + if ctx.node + if ctx.event.to_s == "post_store" + log "Connecting to slack" + Slack.configure do |config| + config.token = cfg.token + config.proxy = cfg.proxy if cfg.has_key?('proxy') + end + client = Slack::Client.new + client.auth_test + log "Connected" + gitoutput = ctx.node.output.new + diff = gitoutput.get_diff ctx.node, ctx.node.group, ctx.commitref, nil + title = "#{ctx.node.name.to_s} #{ctx.node.group.to_s} #{ctx.node.model.class.name.to_s.downcase}" + log "Posting diff as snippet to #{cfg.channel}" + client.files_upload(channels: cfg.channel, as_user: true, + content: diff[:patch].lines.to_a[4..-1].join, + filetype: "diff", + title: title, + filename: "change" + ) + log "Finished" + end + end + end +end diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb index 9a5c508..858d5cd 100644 --- a/lib/oxidized/input/ssh.rb +++ b/lib/oxidized/input/ssh.rb @@ -25,7 +25,10 @@ module Oxidized @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-ssh", 'w') if Oxidized.config.input.debug? port = vars(:ssh_port) || 22 if proxy_host = vars(:ssh_proxy) - proxy = Net::SSH::Proxy::Command.new("ssh #{proxy_host} -W %h:%p") + proxy_command = "ssh " + proxy_command += "-o StrictHostKeyChecking=no " unless secure + proxy_command += "#{proxy_host} -W %h:%p" + proxy = Net::SSH::Proxy::Command.new(proxy_command) end ssh_opts = { :port => port.to_i, diff --git a/lib/oxidized/model/aireos.rb b/lib/oxidized/model/aireos.rb index 7056e3f..ba13120 100644 --- a/lib/oxidized/model/aireos.rb +++ b/lib/oxidized/model/aireos.rb @@ -44,7 +44,7 @@ class Aireos < Oxidized::Model out = [] cfg.each_line do |line| next if line.match /^\s*$/ - next if line.match /rogue (adhoc|client) alert [\da-f]{2}:/ + next if line.match /rogue (adhoc|client) (alert|Unknown) [\da-f]{2}:/ line = line[1..-1] if line[0] == "\r" out << line.strip end diff --git a/lib/oxidized/model/airos.rb b/lib/oxidized/model/airos.rb index 775005f..7d82956 100644 --- a/lib/oxidized/model/airos.rb +++ b/lib/oxidized/model/airos.rb @@ -1,14 +1,19 @@ class Airos < Oxidized::Model # Ubiquiti AirOS circa 5.x - + prompt /^[^#]+# / - + comment '# ' + cmd 'cat /etc/board.info' do |cfg| cfg.split("\n").map { |line| "# #{line}" }.join("\n") + "\n" end - + + cmd 'cat /etc/version' do |cfg| + comment "airos version: #{cfg}" + end + cmd 'sort /tmp/system.cfg' - + cmd :secret do |cfg| cfg.gsub! /^(users\.\d+\.password|snmp\.community)=.+/, "# \\1=<hidden>" cfg diff --git a/lib/oxidized/model/alvarion.rb b/lib/oxidized/model/alvarion.rb index 3c762de..7a4dcc7 100644 --- a/lib/oxidized/model/alvarion.rb +++ b/lib/oxidized/model/alvarion.rb @@ -8,6 +8,8 @@ class Alvarion < Oxidized::Model end - cfg :tftp {} + cfg :tftp do + + end end diff --git a/lib/oxidized/model/aosw.rb b/lib/oxidized/model/aosw.rb index adf481b..a85ead7 100644 --- a/lib/oxidized/model/aosw.rb +++ b/lib/oxidized/model/aosw.rb @@ -1,47 +1,61 @@ class AOSW < Oxidized::Model - # AOSW Aruba Wireless + # AOSW Aruba Wireless, IAP, Instant Controller and Mobility Access Switches # Used in Alcatel OAW-4750 WLAN controller # Also Dell controllers + + # HPE Aruba Switches should use a different model as the software is based on the HP Procurve line. + + # Support for IAP & Instant Controller tested with 115, 205, 215 & 325 running 6.4.4.8-4.2.4.5_57965 + # Support for Mobility Access Switches tested with S2500-48P & S2500-24P running 7.4.1.4_54199 and S2500-24P running 7.4.1.7_57823 + # All IAPs connected to a Instant Controller will have the same config output. Only the controller needs to be monitored. comment '# ' - prompt /^\([^)]+\) [#>]/ + prompt /^\(?.+\)?\s?[#>]/ cmd :all do |cfg| cfg.each_line.to_a[1..-2].join end cmd :secret do |cfg| + cfg.gsub!(/secret (\S+)$/, 'secret <secret removed>') + cfg.gsub!(/enable secret (\S+)$/, 'enable secret <secret removed>') cfg.gsub!(/PRE-SHARE (\S+)$/, 'PRE-SHARE <secret removed>') cfg.gsub!(/ipsec (\S+)$/, 'ipsec <secret removed>') cfg.gsub!(/community (\S+)$/, 'community <secret removed>') cfg.gsub!(/ sha (\S+)/, ' sha <secret removed>') cfg.gsub!(/ des (\S+)/, ' des <secret removed>') cfg.gsub!(/mobility-manager (\S+) user (\S+) (\S+)/, 'mobility-manager \1 user \2 <secret removed>') - cfg.gsub!(/mgmt-user (\S+) (\S+) (\S+)$/, 'mgmt-user \1 \2 <secret removed>') + cfg.gsub!(/mgmt-user (\S+) (root|guest\-provisioning|network\-operations|read\-only|location\-api\-mgmt) (\S+)$/, 'mgmt-user \1 \2 <secret removed>') #MAS & Wireless Controler + cfg.gsub!(/mgmt-user (\S+) (\S+)( (read\-only|guest\-mgmt))?$/, 'mgmt-user \1 <secret removed> \3') #IAP +#MAS format: mgmt-user <username> <accesslevel> <password hash> +#IAP format (root user): mgmt-user <username> <password hash> +#IAP format: mgmt-user <username> <password hash> <access level> cfg.gsub!(/key (\S+)$/, 'key <secret removed>') - cfg.gsub!(/secret (\S+)$/, 'secret <secret removed>') cfg.gsub!(/wpa-passphrase (\S+)$/, 'wpa-passphrase <secret removed>') cfg.gsub!(/bkup-passwords (\S+)$/, 'bkup-passwords <secret removed>') + cfg.gsub!(/user (\S+) (\S+) (\S+)$/, 'user \1 <secret removed> \3') + cfg.gsub!(/virtual-controller-key (\S+)$/, 'virtual-controller-key <secret removed>') cfg end cmd 'show version' do |cfg| - cfg = cfg.each_line.select { |line| not line.match /Switch uptime/i } + cfg = cfg.each_line.select { |line| not line.match /(Switch|AP) uptime/i } rstrip_cfg comment cfg.join end cmd 'show inventory' do |cfg| + cfg = "" if cfg.match /(Invalid input detected at '\^' marker|Parse error)/ #Don't show for unsupported devices (IAP and MAS) rstrip_cfg clean cfg end cmd 'show slots' do |cfg| - cfg = "" if cfg.match /Invalid input detected at '\^' marker/ #Don't show for unsupported devices + cfg = "" if cfg.match /(Invalid input detected at '\^' marker|Parse error)/ #Don't show for unsupported devices (IAP and MAS) rstrip_cfg comment cfg end cmd 'show license' do |cfg| - cfg = "" if cfg.match /Invalid input detected at '\^' marker/ #Don't show for unsupported devices + cfg = "" if cfg.match /(Invalid input detected at '\^' marker|Parse error)/ #Don't show for unsupported devices (IAP and MAS) rstrip_cfg comment cfg end diff --git a/lib/oxidized/model/asa.rb b/lib/oxidized/model/asa.rb index df30059..038dd6b 100644 --- a/lib/oxidized/model/asa.rb +++ b/lib/oxidized/model/asa.rb @@ -13,9 +13,10 @@ class ASA < Oxidized::Model cmd :secret do |cfg| cfg.gsub! /enable password (\S+) (.*)/, 'enable password <secret hidden> \2' cfg.gsub! /username (\S+) password (\S+) (.*)/, 'username \1 password <secret hidden> \3' - cfg.gsub! /ikev2 pre-shared-key (\S+)/, 'ikev2 pre-shared-key <secret hidden>' - cfg.gsub! /ikev2 (remote|local)-authentication pre-shared-key (\S+)/, 'ikev2 \1-authentication pre-shared-key <secret hidden>' + cfg.gsub! /(ikev[12] ((remote|local)-authentication )?pre-shared-key) (\S+)/, '\1 <secret hidden>' cfg.gsub! /^(aaa-server TACACS\+? \(\S+\) host.*\n\skey) \S+$/mi, '\1 <secret hidden>' + cfg.gsub! /ldap-login-password (\S+)/, 'ldap-login-password <secret hidden>' + cfg.gsub! /^snmp-server host (.*) community (\S+)/, 'snmp-server host \1 community <secret hidden>' cfg end diff --git a/lib/oxidized/model/cisconga.rb b/lib/oxidized/model/cisconga.rb new file mode 100644 index 0000000..73fb51c --- /dev/null +++ b/lib/oxidized/model/cisconga.rb @@ -0,0 +1,19 @@ +class CiscoNGA < Oxidized::Model + + comment '# ' + prompt /([\w.@-]+[#>]\s?)$/ + + cmd 'show version' do |cfg| + comment cfg + end + + cmd 'show configuration' do |cfg| + cfg + end + + cfg :ssh do + post_login 'terminal length 0' + pre_logout 'exit' + end + +end diff --git a/lib/oxidized/model/comware.rb b/lib/oxidized/model/comware.rb index 27b70ae..d926854 100644 --- a/lib/oxidized/model/comware.rb +++ b/lib/oxidized/model/comware.rb @@ -18,6 +18,12 @@ class Comware < Oxidized::Model cfg.each_line.to_a[1..-2].join end + cmd :secret do |cfg| + cfg.gsub! /^( snmp-agent community).*/, '\\1 <configuration removed>' + cfg.gsub! /^( password hash).*/, '\\1 <configuration removed>' + cfg + end + cfg :telnet do username /^Username:$/ password /^Password:$/ diff --git a/lib/oxidized/model/cumulus.rb b/lib/oxidized/model/cumulus.rb index dc6792a..20acb8a 100644 --- a/lib/oxidized/model/cumulus.rb +++ b/lib/oxidized/model/cumulus.rb @@ -32,7 +32,10 @@ class Cumulus < Oxidized::Model cfg += add_comment 'IP Routes' cfg += cmd 'netstat -rn' - + + cfg += add_comment 'SNMP settings' + cfg += cmd 'cat /etc/snmp/snmpd.conf' + cfg += add_comment 'QUAGGA DAEMONS' cfg += cmd 'cat /etc/quagga/daemons' @@ -48,21 +51,30 @@ class Cumulus < Oxidized::Model cfg += add_comment 'QUAGGA OSPF6' cfg += cmd 'cat /etc/quagga/ospf6d.conf' + cfg += add_comment 'QUAGGA CONF' + cfg += cmd 'cat /etc/quagga/Quagga.conf' + cfg += add_comment 'MOTD' cfg += cmd 'cat /etc/motd' cfg += add_comment 'PASSWD' cfg += cmd 'cat /etc/passwd' - cfg += add_comment ' SWITCHD' + cfg += add_comment 'SWITCHD' cfg += cmd 'cat /etc/cumulus/switchd.conf' + cfg += add_comment 'PORTS' + cfg += cmd 'cat /etc/cumulus/ports.conf' + + cfg += add_comment 'TRAFFIC' + cfg += cmd 'cat /etc/cumulus/datapath/traffic.conf' + cfg += add_comment 'ACL' cfg += cmd 'iptables -L -n' cfg += add_comment 'VERSION' cfg += cmd 'cat /etc/cumulus/etc.replace/os-release' - + cfg += add_comment 'License' cfg += cmd 'cl-license' diff --git a/lib/oxidized/model/firewareos.rb b/lib/oxidized/model/firewareos.rb index f2bef4c..f456c60 100644 --- a/lib/oxidized/model/firewareos.rb +++ b/lib/oxidized/model/firewareos.rb @@ -1,12 +1,18 @@ class FirewareOS < Oxidized::Model - prompt /^\[?\w*\]?\w*<?\w*>?#\s*$/ + prompt /^([\w.@-]+[#>]\s?)$/ comment '-- ' cmd :all do |cfg| cfg.each_line.to_a[1..-2].join end + # Handle Logon Disclaimer added in XTM 11.9.3 + expect /^I have read and accept the Logon Disclaimer message. \(yes or no\)\? $/ do |data, re| + send "yes\n" + data.sub re, '' + end + cmd 'show sysinfo' do |cfg| # avoid commits due to uptime cfg = cfg.each_line.select { |line| not line.match /(.*time.*)|(.*memory.*)|(.*cpu.*)/ } diff --git a/lib/oxidized/model/fortios.rb b/lib/oxidized/model/fortios.rb index cdb50d5..cdaa282 100644 --- a/lib/oxidized/model/fortios.rb +++ b/lib/oxidized/model/fortios.rb @@ -15,7 +15,8 @@ class FortiOS < Oxidized::Model end cmd :secret do |cfg| - cfg.gsub! /(set (?:passwd|password)).*/, '\\1 <configuration removed>' + cfg.gsub! /(set (?:passwd|password|psksecret|secret|key ENC)).*/, '\\1 <configuration removed>' + cfg.gsub! /(set private-key).*-+END ENCRYPTED PRIVATE KEY-*"$/m , '\\1 <configuration removed>' cfg end @@ -30,14 +31,18 @@ class FortiOS < Oxidized::Model cfg << cmd('config global') if @vdom_enabled cfg << cmd('get hardware status') do |cfg| - comment cfg + comment cfg end - cfg << cmd('diagnose autoupdate version') do |cfg| - comment cfg.each_line.reject { |line| line.match /Last Update|Result/ }.join + #default behaviour: include autoupdate output (backwards compatibility) + #do not include if variable "show_autoupdate" is set to false + if defined?(vars(:fortios_autoupdate)).nil? || vars(:fortios_autoupdate) + cfg << cmd('diagnose autoupdate version') do |cfg| + comment cfg.each_line.reject { |line| line.match /Last Update|Result/ }.join + end end - cfg << cmd('end') if @vdom_enabled +cfg << cmd('end') if @vdom_enabled cfg << cmd('show') cfg.join "\n" @@ -53,3 +58,4 @@ class FortiOS < Oxidized::Model end end + diff --git a/lib/oxidized/model/ios.rb b/lib/oxidized/model/ios.rb index c8b0ef3..c3d5543 100644 --- a/lib/oxidized/model/ios.rb +++ b/lib/oxidized/model/ios.rb @@ -19,23 +19,96 @@ class IOS < Oxidized::Model cmd :all do |cfg| #cfg.gsub! /\cH+\s{8}/, '' # example how to handle pager #cfg.gsub! /\cH+/, '' # example how to handle pager + # get rid of errors for commands that don't work on some devices + cfg.gsub! /^% Invalid input detected at '\^' marker\.$|^\s+\^$/, '' cfg.each_line.to_a[1..-2].join end cmd :secret do |cfg| cfg.gsub! /^(snmp-server community).*/, '\\1 <configuration removed>' - cfg.gsub! /username (\S+) privilege (\d+) (\S+).*/, '<secret hidden>' - cfg.gsub! /^username \S+ password \d \S+/, '<secret hidden>' - cfg.gsub! /^username \S+ secret \d \S+/, '<secret hidden>' - cfg.gsub! /^enable (password|secret) \d \S+/, '<secret hidden>' + cfg.gsub! /^(username \S+ privilege \d+) (\S+).*/, '\\1 <secret hidden>' + cfg.gsub! /^(username \S+ password \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(username \S+ secret \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(enable (password|secret) \d) (\S+)/, '\\1 <secret hidden>' cfg.gsub! /^(\s+(?:password|secret)) (?:\d )?\S+/, '\\1 <secret hidden>' - cfg.gsub! /wpa-psk ascii \d \S+/, '<secret hidden>' - cfg.gsub! /^tacacs-server key \d \S+/, '<secret hidden>' + cfg.gsub! /^(.*wpa-psk ascii \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(.*key 7) (\d.+)/, '\\1 <secret hidden>' + cfg.gsub! /^(tacacs-server key \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(crypto isakmp key) (\S+) (.*)/, '\\1 <secret hidden> \\3' cfg end cmd 'show version' do |cfg| - comment cfg.lines.first + comments = [] + comments << cfg.lines.first + lines = cfg.lines + lines.each_with_index do |line,i| + slave = '' + slaveslot = '' + + if line.match /^Slave in slot (\d+) is running/ + slave = " Slave:"; + slaveslot = ", slot #{$1}"; + end + + if line.match /^Compiled (.*)$/ + comments << "Image:#{slave} Compiled: #{$1}" + end + + if line.match /^(?:Cisco )?IOS .* Software,? \(([A-Za-z0-9_-]*)\), .*Version\s+(.*)$/ + comments << "Image:#{slave} Software: #{$1}, #{$2}" + end + + if line.match /^ROM: (IOS \S+ )?(System )?Bootstrap.*(Version.*)$/ + comments << "ROM Bootstrap: #{$3}" + end + + if line.match /^BOOTFLASH: .*(Version.*)$/ + comments << "BOOTFLASH: #{$1}" + end + + if line.match /^(\d+[kK]) bytes of (non-volatile|NVRAM)/ + comments << "Memory: nvram #{$1}" + end + + if line.match /^(\d+[kK]) bytes of (flash memory|flash internal|processor board System flash|ATA CompactFlash)/i + comments << "Memory: flash #{$1}" + end + + if line.match (/^(\d+[kK]) bytes of (Flash|ATA)?.*PCMCIA .*(slot|disk) ?(\d)/i) + comments << "Memory: pcmcia #{$2} #{$3}#{$4} #{$1}"; + end + + if line.match /(\S+(?:\sseries)?)\s+(?:\((\S+)\)\s+processor|\(revision[^)]+\)).*\s+with (\S+k) bytes/i + sproc = $1 + cpu = $2 + mem = $3 + cpuxtra = '' + comments << "Chassis type:#{slave} #{sproc}"; + comments << "Memory:#{slave} main #{mem}"; + # check the next two lines for more CPU info + if cfg.lines[i+1].match /processor board id (\S+)/i + comments << "Processor ID: #{$1}"; + end + if cfg.lines[i+2].match /(cpu at |processor: |#{cpu} processor,)/i + # change implementation to impl and prepend comma + cpuxtra = cfg.lines[i+2].gsub(/implementation/,'impl').gsub(/^/,', ').chomp; + end + comments << "CPU:#{slave} #{cpu}#{cpuxtra}#{slaveslot}"; + end + + if line.match /^System image file is "([^\"]*)"$/ + comments << "Image: #{$1}" + end + end + comments << "\n" + comment comments.join "\n" + end + + cmd 'show vtp status' do |cfg| + cfg.gsub! /^$\n/, '' + cfg.gsub! /^/, 'VTP: ' if (!cfg.empty?) + comment "#{cfg}\n" end cmd 'show inventory' do |cfg| diff --git a/lib/oxidized/model/ironware.rb b/lib/oxidized/model/ironware.rb index c7c2197..9715c65 100644 --- a/lib/oxidized/model/ironware.rb +++ b/lib/oxidized/model/ironware.rb @@ -67,7 +67,7 @@ class IronWare < Oxidized::Model # match expected prompts on both older and newer # versions of IronWare username /^(Please Enter Login Name|Username):/ - password /^(Please Enter )Password:/ + password /^(Please Enter Password |Password):/ end #handle pager with enable diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb index 058e3cf..2f59414 100644 --- a/lib/oxidized/model/junos.rb +++ b/lib/oxidized/model/junos.rb @@ -8,6 +8,7 @@ class JunOS < Oxidized::Model cmd :all do |cfg| cfg = cfg.lines.to_a[1..-2].join if screenscrape + cfg.gsub!(/ scale-subscriber (\s+)(\d+)/,' scale-subscriber <count>') cfg.lines.map { |line| line.rstrip }.join("\n") + "\n" end @@ -36,6 +37,8 @@ class JunOS < Oxidized::Model end cmd('show chassis hardware') { |cfg| comment cfg } + cmd('show system license') { |cfg| comment cfg } + cmd('show system license keys') { |cfg| comment cfg } cfg :telnet do username(/^login:/) diff --git a/lib/oxidized/model/mlnxos.rb b/lib/oxidized/model/mlnxos.rb index 9542c88..49f3369 100644 --- a/lib/oxidized/model/mlnxos.rb +++ b/lib/oxidized/model/mlnxos.rb @@ -13,6 +13,10 @@ class MLNXOS < Oxidized::Model cfg.gsub! /\[\?1h=\r/, '' # Pager Handling cfg.gsub! /\r\[K/,'' # Pager Handling cfg.gsub! /\s/, '' # Linebreak Handling + cfg.gsub! /^CPU\ load\ averages\:\s.+/, '' # Omit constantly changing CPU info + cfg.gsub! /^System\ memory\:\s.+/, '' # Omit constantly changing memory info + cfg.gsub! /^Uptime\:\s.+/, '' # Omit constantly changing uptime info + cfg.gsub! /.+Generated\ at\s\d+.+/, '' # Omit constantly changing generation time info cfg = cfg.lines.to_a[2..-3].join end @@ -38,6 +42,6 @@ class MLNXOS < Oxidized::Model cfg :ssh do password /^Password:\s*/ - pre_logout 'exit' + pre_logout "\nexit" end end diff --git a/lib/oxidized/model/nxos.rb b/lib/oxidized/model/nxos.rb index fbe772d..e743415 100644 --- a/lib/oxidized/model/nxos.rb +++ b/lib/oxidized/model/nxos.rb @@ -3,6 +3,14 @@ class NXOS < Oxidized::Model prompt /^(\r?[\w.@_()-]+[#]\s?)$/ comment '! ' + cmd :secret do |cfg| + cfg.gsub! /^(snmp-server community).*/, '\\1 <configuration removed>' + cfg.gsub! /^(snmp-server user (\S+) (\S+) auth (\S+)) (\S+) (priv) (\S+)/, '\\1 <configuration removed> ' + cfg.gsub! /^(username \S+ password \d) (\S+)/, '\\1 <secret hidden>' + cfg.gsub! /^(radius-server key).*/, '\\1 <secret hidden>' + cfg + end + cmd 'show version' do |cfg| cfg = cfg.each_line.take_while { |line| not line.match(/uptime/i) } comment cfg.join "" diff --git a/lib/oxidized/model/oneos.rb b/lib/oxidized/model/oneos.rb new file mode 100644 index 0000000..eeaa2ce --- /dev/null +++ b/lib/oxidized/model/oneos.rb @@ -0,0 +1,58 @@ +class OneOS < Oxidized::Model + + prompt /^([\w.@()-]+#\s?)$/ + comment '! ' + + # example how to handle pager + #expect /^\s--More--\s+.*$/ do |data, re| + # send ' ' + # data.sub re, '' + #end + + # non-preferred way to handle additional PW prompt + #expect /^[\w.]+>$/ do |data| + # send "enable\n" + # send vars(:enable) + "\n" + # data + #end + + cmd :all do |cfg| + #cfg.gsub! /\cH+\s{8}/, '' # example how to handle pager + #cfg.gsub! /\cH+/, '' # example how to handle pager + cfg.each_line.to_a[1..-2].join + end + + cmd :secret do |cfg| + cfg.gsub! /^(snmp set-read-community ").*+?(".*)$/, '\\1<secret hidden>\\2' + cfg + end + + cmd 'show version' do |cfg| + comment cfg + end + + cmd 'show running-config' do |cfg| + cfg = cfg.each_line.to_a[0..-1].join + cfg.gsub! /^Building configuration...\s*[^\n]*\n/, '' + cfg.gsub! /^Current configuration :\s*[^\n]*\n/, '' + cfg + end + + cfg :telnet do + username /^Username:/ + password /^Password:/ + end + + cfg :telnet, :ssh do + # preferred way to handle additional passwords + if vars :enable + post_login do + send "enable\n" + cmd vars(:enable) + end + end + post_login 'term len 0' + pre_logout 'exit' + end + +end diff --git a/lib/oxidized/model/opengear.rb b/lib/oxidized/model/opengear.rb index 7f801f8..b7c697f 100644 --- a/lib/oxidized/model/opengear.rb +++ b/lib/oxidized/model/opengear.rb @@ -1,6 +1,8 @@ class OpenGear < Oxidized::Model comment '# ' + + prompt /^(\$\s)?$/ cmd :secret do |cfg| cfg.gsub!(/password (\S+)/, 'password <secret removed>') diff --git a/lib/oxidized/model/pfsense.rb b/lib/oxidized/model/pfsense.rb index c02c0d0..782969e 100644 --- a/lib/oxidized/model/pfsense.rb +++ b/lib/oxidized/model/pfsense.rb @@ -7,7 +7,8 @@ class PfSense < Oxidized::Model end cmd 'cat /cf/conf/config.xml' do |cfg| - cfg.gsub! /\s<revision>\s*.*\s*<time>\d*<\/time>\s*.*\s*<\/revision>/, '' + cfg.gsub! /\s<revision>\s*<time>\d*<\/time>\s*.*\s*.*\s*<\/revision>/, '' + cfg.gsub! /\s<last_rule_upd_time>\d*<\/last_rule_upd_time>/, '' cfg end diff --git a/lib/oxidized/model/powerconnect.rb b/lib/oxidized/model/powerconnect.rb index ac36c26..618f05a 100644 --- a/lib/oxidized/model/powerconnect.rb +++ b/lib/oxidized/model/powerconnect.rb @@ -67,6 +67,7 @@ class PowerConnect < Oxidized::Model end out << line.strip end + out = out.select { |line| not line[/Up\sTime/] } out = comment out.join "\n" out << "\n" end diff --git a/lib/oxidized/model/procurve.rb b/lib/oxidized/model/procurve.rb index c117df3..7dcf1fd 100644 --- a/lib/oxidized/model/procurve.rb +++ b/lib/oxidized/model/procurve.rb @@ -1,11 +1,16 @@ class Procurve < Oxidized::Model - # some models start lines with \r + # some models start lines with \r # previous command is repeated followed by "\eE", which sometimes ends up on last line - prompt /^\r?([\w -]+\eE)?([\w.-]+# )$/ + prompt /^\r?([\w.-]+# )$/ comment '! ' + # replace next line control sequence with a new line + expect /(\e\[1M\e\[\??\d+(;\d+)*[A-Za-z]\e\[1L)|(\eE)/ do |data, re| + data.gsub re, "\n" + end + # replace all used vt100 control sequences expect /\e\[\??\d+(;\d+)*[A-Za-z]/ do |data, re| data.gsub re, '' @@ -17,7 +22,7 @@ class Procurve < Oxidized::Model end cmd :all do |cfg| - cfg = cfg.each_line.to_a[1..-3].join + cfg = cfg.each_line.to_a[1..-2].join cfg = cfg.gsub /^\r/, '' end diff --git a/lib/oxidized/model/routeros.rb b/lib/oxidized/model/routeros.rb index a92ad5e..5717100 100644 --- a/lib/oxidized/model/routeros.rb +++ b/lib/oxidized/model/routeros.rb @@ -6,6 +6,10 @@ class RouterOS < Oxidized::Model comment cfg end + cmd '/system package update print' do |cfg| + comment cfg + end + cmd '/export' do |cfg| cfg.gsub! /\x1B\[([0-9]{1,3}((;[0-9]{1,3})*)?)?[m|K]/, '' # strip ANSI colours cfg.gsub! /\\\r\n\s+/, '' # strip new line diff --git a/lib/oxidized/model/saos.rb b/lib/oxidized/model/saos.rb index 5d460cf..5a4e79e 100644 --- a/lib/oxidized/model/saos.rb +++ b/lib/oxidized/model/saos.rb @@ -2,7 +2,7 @@ class SAOS < Oxidized::Model # Ciena SAOS switch # used for 6.x devices - + comment '! ' cmd :all do |cfg| @@ -10,6 +10,8 @@ class SAOS < Oxidized::Model end cmd 'configuration show' do |cfg| + cfg.gsub! /^! Created: [^\n]*\n/, '' + cfg.gsub! /^! On terminal: [^\n]*\n/, '' cfg end diff --git a/lib/oxidized/model/siklu.rb b/lib/oxidized/model/siklu.rb new file mode 100644 index 0000000..2bdfbc3 --- /dev/null +++ b/lib/oxidized/model/siklu.rb @@ -0,0 +1,19 @@ +class Siklu < Oxidized::Model + + # Siklu EtherHaul # + + prompt /^[\w-]+>$/ + + cmd 'copy startup-configuration display' do |cfg| + cfg.each_line.to_a[2..2].join + end + + cmd 'copy running-configuration display' do |cfg| + cfg.each_line.to_a[3..-2].join + end + + cfg :ssh do + pre_logout 'exit' + end + +end diff --git a/lib/oxidized/model/timos.rb b/lib/oxidized/model/timos.rb index d40e845..c230a8f 100644 --- a/lib/oxidized/model/timos.rb +++ b/lib/oxidized/model/timos.rb @@ -18,6 +18,8 @@ class TiMOS < Oxidized::Model # Show the boot options file. # cmd 'show bof' do |cfg| + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' comment cfg end @@ -29,6 +31,8 @@ class TiMOS < Oxidized::Model # Strip uptime. # cfg.sub! /^System Up Time.*\n/, '' + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' comment cfg end @@ -36,6 +40,8 @@ class TiMOS < Oxidized::Model # Show the card state. # cmd 'show card state' do |cfg| + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' comment cfg end @@ -48,6 +54,8 @@ class TiMOS < Oxidized::Model # cfg.gsub! /\r/, '' cfg.gsub! /[\b][\b][\b]/, "\n" + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' comment cfg end @@ -55,6 +63,8 @@ class TiMOS < Oxidized::Model # Show the running debug configuration. # cmd 'show debug' do |cfg| + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' comment cfg end @@ -66,6 +76,8 @@ class TiMOS < Oxidized::Model # Strip carriage returns. # cfg.gsub! /\r/, '' + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' comment cfg end @@ -77,6 +89,8 @@ class TiMOS < Oxidized::Model # Strip carriage returns. # cfg.gsub! /\r/, '' + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' comment cfg end @@ -88,6 +102,8 @@ class TiMOS < Oxidized::Model # Strip carriage returns. # cfg.gsub! /\r/, '' + cfg.gsub! /# Finished .*/, '' + cfg.gsub! /# Generated .*/, '' end cfg :telnet do diff --git a/lib/oxidized/model/tplink.rb b/lib/oxidized/model/tplink.rb new file mode 100644 index 0000000..bf13803 --- /dev/null +++ b/lib/oxidized/model/tplink.rb @@ -0,0 +1,65 @@ +class TPLink < Oxidized::Model + + # tp-link prompt + prompt /^\r?([\w.@()-]+[#>]\s?)$/ + comment '! ' + + # handle paging + # workaround for sometimes missing whitespaces with "\s?" + expect /Press\s?any\s?key\s?to\s?continue\s?\(Q\s?to\s?quit\)/ do |data, re| + send ' ' + data.sub re, '' + end + + # send carriage return because \n with the command is not enough + # checks if line ends with prompt >,# or \r,\nm otherwise send \r + expect /[^>#\r\n]$/ do |data, re| + send "\r" + data.sub re, '' + end + + cmd :all do |cfg| + # normalize linefeeds + cfg.gsub! /(\r|\r\n|\n\r)/,"\n" + # remove empty lines + cfg.each_line.reject { |line| line.match /^[\r\n\s\u0000#]+$/ }.join + end + + cmd :secret do |cfg| + cfg.gsub! /^(snmp-server community).*/, '\\1 <configuration removed>' + cfg.gsub! /secret (\d+) (\S+).*/, '<secret hidden>' + cfg + end + + cmd 'show system-info' do |cfg| + comment cfg.each_line.to_a[3..-3].join + end + + cmd 'show running-config' do |cfg| + lines = cfg.each_line.to_a[1..-1] + # cut config after "end" + lines[0..lines.index("end\n")].join + end + + cfg :telnet, :ssh do + username /^User ?[nN]ame:/ + password /^\r?Password:/ + end + + cfg :telnet, :ssh do + if vars :enable + post_login do + send "enable\r" + cmd vars(:enable) + end + end + + pre_logout do + send "exit\r" + send "logout\r" + end + + end + +end + diff --git a/lib/oxidized/model/voltaire.rb b/lib/oxidized/model/voltaire.rb new file mode 100644 index 0000000..1e7fad2 --- /dev/null +++ b/lib/oxidized/model/voltaire.rb @@ -0,0 +1,56 @@ +class VOLTAIRE < Oxidized::Model + + prompt /([\w.@()-\[:\s\]]+[#>]\s|(One or more tests have failed.*))$/ + comment '## ' + + # Pager Handling + expect /.+lines\s\d+\-\d+([\s]|\/\d+\s\(END\)\s).+$/ do |data, re| + send ' ' + data.sub re, '' + end + + + cmd :all do |cfg| + cfg.gsub! /\[\?1h=\r/, '' # Pager Handling + cfg.gsub! /\r\[K/,'' # Pager Handling + cfg.gsub! /\s/, '' # Linebreak Handling + cfg.gsub! /^CPU\ load\ averages\:\s.+/, '' # Omit constantly changing CPU info + cfg.gsub! /^System\ memory\:\s.+/, '' # Omit constantly changing memory info + cfg.gsub! /^Uptime\:\s.+/, '' # Omit constantly changing uptime info + cfg.gsub! /.+Generated\ at\s\d+.+/, '' # Omit constantly changing generation time info + cfg = cfg.lines.to_a[2..-3].join + end + + cmd :secret do |cfg| + cfg.gsub! /(snmp-server community).*/, ' <snmp-server community configuration removed>' + cfg.gsub! /username (\S+) password (\d+) (\S+).*/, '<secret hidden>' + cfg + end + + + cmd 'version show' do |cfg| + comment cfg + end + + cmd 'firmware-version show' do |cfg| + comment cfg + end + + cmd 'remote show' do |cfg| + cfg + end + + cmd 'sm-info show' do |cfg| + cfg + end + + cmd ' show' do |cfg| + cfg + end + + cfg :ssh do + post_login "no\n" + password /^Password:\s*/ + pre_logout 'exit' + end +end diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb index 6f89b56..cf71e48 100644 --- a/lib/oxidized/node.rb +++ b/lib/oxidized/node.rb @@ -166,18 +166,32 @@ module Oxidized end def resolve_repo opt - return unless is_git? opt - - remote_repo = Oxidized.config.output.git.repo - - if remote_repo.is_a?(::String) - if Oxidized.config.output.git.single_repo? || @group.nil? - remote_repo + if is_git? opt + remote_repo = Oxidized.config.output.git.repo + + if remote_repo.is_a?(::String) + if Oxidized.config.output.git.single_repo? || @group.nil? + remote_repo + else + File.join(File.dirname(remote_repo), @group + '.git') + end + else + remote_repo[@group] + end + elsif is_gitcrypt? opt + remote_repo = Oxidized.config.output.gitcrypt.repo + + if remote_repo.is_a?(::String) + if Oxidized.config.output.gitcrypt.single_repo? || @group.nil? + remote_repo + else + File.join(File.dirname(remote_repo), @group + '.git') + end else - File.join(File.dirname(remote_repo), @group + '.git') + remote_repo[@group] end else - remote_repo[@group] + return end end @@ -212,5 +226,9 @@ module Oxidized (opt[:output] || Oxidized.config.output.default) == 'git' end + def is_gitcrypt? opt + (opt[:output] || Oxidized.config.output.default) == 'gitcrypt' + end + end end diff --git a/lib/oxidized/nodes.rb b/lib/oxidized/nodes.rb index f5a1ad0..6751c7a 100644 --- a/lib/oxidized/nodes.rb +++ b/lib/oxidized/nodes.rb @@ -4,7 +4,7 @@ module Oxidized class Oxidized::NotSupported < OxidizedError; end class Oxidized::NodeNotFound < OxidizedError; end class Nodes < Array - attr_accessor :source + attr_accessor :source, :jobs alias :put :unshift def load node_want=nil with_lock do @@ -73,6 +73,7 @@ module Oxidized # set last job to nil so that the node is picked for immediate update n.last = nil put n + jobs.want += 1 if Oxidized.config.next_adds_job? end end end diff --git a/lib/oxidized/output/gitcrypt.rb b/lib/oxidized/output/gitcrypt.rb new file mode 100644 index 0000000..b0d80f2 --- /dev/null +++ b/lib/oxidized/output/gitcrypt.rb @@ -0,0 +1,244 @@ +module Oxidized + class GitCrypt < Output + class GitCryptError < OxidizedError; end + begin + require 'git' + rescue LoadError + raise OxidizedError, 'git not found: sudo gem install ruby-git' + end + + attr_reader :commitref + + def initialize + @cfg = Oxidized.config.output.gitcrypt + @gitcrypt_cmd = "/usr/bin/git-crypt" + @gitcrypt_init = @gitcrypt_cmd + " init" + @gitcrypt_unlock = @gitcrypt_cmd + " unlock" + @gitcrypt_lock = @gitcrypt_cmd + " lock" + @gitcrypt_adduser = @gitcrypt_cmd + " add-gpg-user --trusted " + end + + def setup + if @cfg.empty? + Oxidized.asetus.user.output.gitcrypt.user = 'Oxidized' + Oxidized.asetus.user.output.gitcrypt.email = 'o@example.com' + Oxidized.asetus.user.output.gitcrypt.repo = File.join(Config::Root, 'oxidized.git') + Oxidized.asetus.save :user + raise NoConfig, 'no output git config, edit ~/.config/oxidized/config' + end + + if @cfg.repo.respond_to?(:each) + @cfg.repo.each do |group, repo| + @cfg.repo["#{group}="] = File.expand_path repo + end + else + @cfg.repo = File.expand_path @cfg.repo + end + end + + def crypt_init repo + repo.chdir do + system(@gitcrypt_init) + @cfg.users.each do |user| + system("#{@gitcrypt_adduser} #{user}") + end + File.write(".gitattributes", "* filter=git-crypt diff=git-crypt\n.gitattributes !filter !diff") + repo.add(".gitattributes") + repo.commit("Initial commit: crypt all config files") + end + end + + def lock repo + repo.chdir do + system(@gitcrypt_lock) + end + end + + def unlock repo + repo.chdir do + system(@gitcrypt_unlock) + end + end + + def store file, outputs, opt={} + @msg = opt[:msg] + @user = (opt[:user] or @cfg.user) + @email = (opt[:email] or @cfg.email) + @opt = opt + @commitref = nil + repo = @cfg.repo + + outputs.types.each do |type| + type_cfg = '' + type_repo = File.join(File.dirname(repo), type + '.git') + outputs.type(type).each do |output| + (type_cfg << output; next) if not output.name + type_file = file + '--' + output.name + if @cfg.type_as_directory? + type_file = type + '/' + type_file + type_repo = repo + end + update type_repo, type_file, output + end + update type_repo, file, type_cfg + end + + update repo, file, outputs.to_cfg + end + + + def fetch node, group + begin + repo, path = yield_repo_and_path(node, group) + repo = Git.open repo + unlock repo + index = repo.index + # Empty repo ? + empty = File.exists? index.path + if empty + raise 'Empty git repo' + else + File.read path + end + lock repo + rescue + 'node not found' + end + end + + # give a hash of all oid revision for the given node, and the date of the commit + def version node, group + begin + repo, path = yield_repo_and_path(node, group) + + repo = Git.open repo + unlock repo + walker = repo.log.path(path) + i = -1 + tab = [] + walker.each do |commit| + hash = {} + hash[:date] = commit.date.to_s + hash[:oid] = commit.objectish + hash[:author] = commit.author + hash[:message] = commit.message + tab[i += 1] = hash + end + walker.reset + tab + rescue + 'node not found' + end + end + + #give the blob of a specific revision + def get_version node, group, oid + begin + repo, path = yield_repo_and_path(node, group) + repo = Git.open repo + unlock repo + repo.gtree(oid).files[path].contents + rescue + 'version not found' + ensure + lock repo + end + end + + #give a hash with the patch of a diff between 2 revision and the stats (added and deleted lines) + def get_diff node, group, oid1, oid2 + begin + diff_commits = nil + repo, path = yield_repo_and_path(node, group) + repo = Git.open repo + unlock repo + commit = repo.gcommit(oid1) + + if oid2 + commit_old = repo.gcommit(oid2) + diff = repo.diff(commit_old, commit) + stats = [diff.stats[:files][node.name][:insertions], diff.stats[:files][node.name][:deletions]] + diff.each do |patch| + if /#{node.name}\s+/.match(patch.patch.to_s.lines.first) + diff_commits = {:patch => patch.patch.to_s, :stat => stats} + break + end + end + else + stat = commit.parents[0].diff(commit).stats + stat = [stat[:files][node.name][:insertions],stat[:files][node.name][:deletions]] + patch = commit.parents[0].diff(commit).patch + diff_commits = {:patch => patch, :stat => stat} + end + lock repo + diff_commits + rescue + 'no diffs' + ensure + lock repo + end + end + + private + + def yield_repo_and_path(node, group) + repo, path = node.repo, node.name + + if group and @cfg.single_repo? + path = "#{group}/#{node.name}" + end + + [repo, path] + end + + def update repo, file, data + return if data.empty? + + if @opt[:group] + if @cfg.single_repo? + file = File.join @opt[:group], file + else + repo = if repo.is_a?(::String) + File.join File.dirname(repo), @opt[:group] + '.git' + else + repo[@opt[:group]] + end + end + end + + begin + update_repo repo, file, data, @msg, @user, @email + rescue Git::GitExecuteError, ArgumentError => open_error + Oxidized.logger.debug "open_error #{open_error} #{file}" + begin + grepo = Git.init repo + crypt_init grepo + rescue => create_error + raise GitCryptError, "first '#{open_error.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo" + end + retry + end + end + + def update_repo repo, file, data, msg, user, email + grepo = Git.open repo + grepo.config('user.name', user) + grepo.config('user.email', email) + grepo.chdir do + unlock grepo + File.write(file, data) + grepo.add(file) + if grepo.status[file].nil? + grepo.commit(msg) + @commitref = grepo.log(1).first.objectish + true + elsif !grepo.status[file].type.nil? + grepo.commit(msg) + @commitref = grepo.log(1).first.objectish + true + end + lock grepo + end + end + end +end diff --git a/lib/oxidized/source/http.rb b/lib/oxidized/source/http.rb index 4fd388b..6c12f29 100644 --- a/lib/oxidized/source/http.rb +++ b/lib/oxidized/source/http.rb @@ -29,25 +29,27 @@ class HTTP < Source end request = Net::HTTP::Get.new(uri.request_uri, headers) - if (@cfg.user && @cfg.pass) + if (@cfg.user? && @cfg.pass?) request.basic_auth(@cfg.user,@cfg.pass) end response = http.request(request) data = JSON.parse(response.body) - data.each do |line| - next if line.empty? + data.each do |node| + next if node.empty? # map node parameters keys = {} - @cfg.map.each do |key, position| - keys[key.to_sym] = node_var_interpolate line[position] + @cfg.map.each do |key, want_position| + want_positions = want_position.split('.') + keys[key.to_sym] = node_var_interpolate node.dig(*want_positions) end keys[:model] = map_model keys[:model] if keys.key? :model # map node specific vars vars = {} - @cfg.vars_map.each do |key, position| - vars[key.to_sym] = node_var_interpolate line[position] + @cfg.vars_map.each do |key, want_position| + want_positions = want_position.split('.') + vars[key.to_sym] = node_var_interpolate node.dig(*want_positions) end keys[:vars] = vars unless vars.empty? @@ -58,3 +60,18 @@ class HTTP < Source end end + +if RUBY_VERSION < '2.3' + class Hash + def dig(key, *rest) + value = self[key] + if value.nil? || rest.empty? + value + elsif value.respond_to?(:dig) + value.dig(*rest) + else # foo.bar.baz (bar exist but is not hash) + return nil + end + end + end +end diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb index 1952d01..80d65cb 100644 --- a/lib/oxidized/worker.rb +++ b/lib/oxidized/worker.rb @@ -3,8 +3,9 @@ module Oxidized require 'oxidized/jobs' class Worker def initialize nodes - @nodes = nodes - @jobs = Jobs.new(Oxidized.config.threads, Oxidized.config.interval, @nodes) + @nodes = nodes + @jobs = Jobs.new(Oxidized.config.threads, Oxidized.config.interval, @nodes) + @nodes.jobs = @jobs Thread.abort_on_exception = true end diff --git a/oxidized.gemspec b/oxidized.gemspec index c12dcb5..ea088e7 100644 --- a/oxidized.gemspec +++ b/oxidized.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'rugged', '~> 0.21', '>= 0.21.4' if defined?(RUBY_VERSION) && RUBY_VERSION > '2.3' - s.add_runtime_dependency 'net-telnet' + s.add_runtime_dependency 'net-telnet', '~> 0' end s.add_development_dependency 'pry', '~> 0' @@ -33,4 +33,5 @@ Gem::Specification.new do |s| s.add_development_dependency 'rake', '~> 10.0' s.add_development_dependency 'minitest', '~> 5.8' s.add_development_dependency 'mocha', '~> 1.1' + s.add_development_dependency 'git', '~> 1' end |