Initial commit

Silly for shit-and-giggles attempt at rancid

34 files changed, 1076 insertions, 0 deletions

diff --git a/Gemfile b/Gemfile
new file mode 100644
index 0000000..03c5da2
--- /dev/null
+++ b/Gemfile
@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+ruby '2.0.0'
+
+gem 'sequel'
+gem 'sqlite3'
+#gem 'sidekiq'
+gem 'net-ssh'
+gem 'grit'
+#gem 'rugged', '~> 0.17.0.b7'
+
+group :development do
+  gem 'pry'
+end
+
+group :test do
+  gem 'rspec'
+end
diff --git a/Gemfile.lock b/Gemfile.lock
new file mode 100644
index 0000000..e1cc091
--- /dev/null
+++ b/Gemfile.lock
@@ -0,0 +1,39 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.0.9)
+    diff-lcs (1.2.3)
+    grit (2.5.0)
+      diff-lcs (~> 1.1)
+      mime-types (~> 1.15)
+      posix-spawn (~> 0.3.6)
+    method_source (0.8.1)
+    mime-types (1.22)
+    net-ssh (2.6.7)
+    posix-spawn (0.3.6)
+    pry (0.9.12)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    sequel (3.46.0)
+    slop (3.4.4)
+    sqlite3 (1.3.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  grit
+  net-ssh
+  pry
+  rspec
+  sequel
+  sqlite3
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5e2b584
--- /dev/null
+++ b/README.md
@@ -0,0 +1,35 @@
+# Pitch
+ * automatically adds/removes threads to meet configured retrieval interval
+ * can move node immediately to head-of-queue (maybe trigger from snmp trap or syslog), to be serviced by next spawned thread
+
+# Install
+ early days, but try to run it and edit ~/.config/oxidized/config
+
+# API
+## Input
+ * gets config from nodes
+ * must implement 'connect', 'get'
+ * 'ssh' and 'telnet' implemented
+
+## Output
+ * stores config
+ * must implement 'update'
+ * 'git' and 'file' (store as flat ascii) implemented
+
+## Source
+ * gets list of nodes to poll
+ * must implement 'load'
+ * source can have 'name', 'model', 'group', 'username', 'password', 'input', 'output', 'prompt'
+   * name - name of the devices
+   * model - model to use ios/junos/xyz, model is loaded dynamically when needed (Also default in config file)
+   * input - method to acquire config, loaded dynamically as needed (Also default in config file)
+   * output - method to store config, loaded dynamically as needed (Also default in config file)
+   * prompt - prompt used for node (Also default in config file, can be specified in model too)
+ * 'sql' and 'csv' (supports any format with single entry per line, like router.db)
+
+## Model
+ * lists commands to gather from given device model
+ * can use 'cmd', 'prompt', 'comment', 'cfg'
+ * cfg is executed in input/output/source context
+ * cmd is executed in instance of model
+ * 'junos' and 'ios' implemented
diff --git a/Rakefile b/Rakefile
new file mode 100644
index 0000000..c415fb3
--- /dev/null
+++ b/Rakefile
@@ -0,0 +1,34 @@
+begin
+  require 'bundler'
+  require 'rspec/core/rake_task'
+  Bundler.setup
+rescue LoadError
+  warn 'missing dependencies'
+  exit 42
+end
+
+gemspec = eval(File.read(Dir['*.gemspec'].first))
+
+desc 'Validate the gemspec'
+task :gemspec do
+  gemspec.validate
+end
+
+RSpec::Core::RakeTask.new(:spec)
+
+desc "Build gem locally"
+task :build => [:spec, :gemspec] do
+  system "gem build #{gemspec.name}.gemspec"
+  FileUtils.mkdir_p "gems"
+  FileUtils.mv "#{gemspec.name}-#{gemspec.version}.gem", "gems"
+end
+
+desc "Install gem locally"
+task :install => :build do
+  system "sudo sh -c \'umask 022; gem install gems/#{gemspec.name}-#{gemspec.version}\'"
+end
+
+desc "Clean automatically generated files"
+task :clean do
+  FileUtils.rm_rf "gems"
+end
diff --git a/TODO.md b/TODO.md
new file mode 100644
index 0000000..e2ba06d
--- /dev/null
+++ b/TODO.md
@@ -0,0 +1,40 @@
+# expect call back:
+
+``` ruby
+expect /---more---/ do
+  @output.pop
+  cmd "\n"
+```
+I don't really need it myself, since I don't have platforms where it would be needed
+
+
+# thread number
+  * think about algo
+  * if job ended later than now-iteration have rand(node.size) == 0 to add thread
+  * if now is less than job_ended+iteration same chance to remove thread?
+  * should we try to avoid max threads from being hit? (like maybe non-success thread is pulling average?)
+
+
+# restful API (puma+sinatra):
+  * ask to reload node list
+  * move node to head of queue
+
+
+# config
+   * save keys as strings, load as symbols?
+
+# other 
+should it offer cli mass config-pusher? (I think not, I have ideas for such
+program and I'm not sure if synergies are high enough for shared code without
+making both bit awkward)
+
+use sidekiq? Any benefits?
+
+
+# docs, testing
+  * yard docs
+  * rspec tests
+
+
+# multiple input methods
+  * ssh, with telnet fallback
diff --git a/lib/oxidized.rb b/lib/oxidized.rb
new file mode 100644
index 0000000..ec1a901
--- /dev/null
+++ b/lib/oxidized.rb
@@ -0,0 +1,4 @@
+module Oxidized
+  Directory = File.expand_path File.join File.dirname(__FILE__), '../' 
+  require 'oxidized/core'
+end
diff --git a/lib/oxidized/config/bootstrap.rb b/lib/oxidized/config/bootstrap.rb
new file mode 100644
index 0000000..e44dde7
--- /dev/null
+++ b/lib/oxidized/config/bootstrap.rb
@@ -0,0 +1,27 @@
+module Oxidized
+  require 'fileutils'
+  FileUtils.mkdir_p Config::Root
+  CFG.username = 'username'
+  CFG.password = 'password'
+  CFG.model    = 'junos'
+  CFG.interval = 30
+  CFG.log      = File.join Config::Root, 'log'
+  CFG.debug    = false
+  CFG.threads  = 10
+  CFG.timeout  = 5
+  CFG.prompt   = /^([\w\.\-@]{3,30}[#>]\s?)$/ 
+  CFG.input    = {
+    :default => 'ssh',
+  }
+  CFG.output    = {
+    :default => 'git',
+  }
+  CFG.source   = {
+    :default => 'ascii',
+  }
+  CFG.model_map = {
+    'cisco'   => 'ios',
+    'juniper' => 'junos',
+  }
+  CFG.save
+end
diff --git a/lib/oxidized/config/core.rb b/lib/oxidized/config/core.rb
new file mode 100644
index 0000000..6c91920
--- /dev/null
+++ b/lib/oxidized/config/core.rb
@@ -0,0 +1,28 @@
+module Oxidized
+  require 'ostruct'
+  require 'yaml'
+  class Config < OpenStruct
+    require 'oxidized/config/defaults'
+    # @param file [string] configuration file location
+    def initialize file=File.join(Config::Root, 'config')
+      super()
+      @file = file.to_s
+    end
+    # load config from file or bootstrap with built-ins
+    def load
+      if File.exists? @file
+        marshal_load YAML.load_file @file 
+      else
+        require 'oxidized/config/bootstrap'
+      end
+    end
+    # save config to file
+    def save
+      File.write @file, YAML.dump(marshal_dump)
+    end
+  end
+  CFG = Config.new
+  CFG.load
+  Log.file = CFG.log if CFG.log
+  Log.level = Logger::INFO unless CFG.debug
+end
diff --git a/lib/oxidized/config/defaults.rb b/lib/oxidized/config/defaults.rb
new file mode 100644
index 0000000..a07e1a6
--- /dev/null
+++ b/lib/oxidized/config/defaults.rb
@@ -0,0 +1,12 @@
+module Oxidized
+  class Config
+    Root      = File.join ENV['HOME'], '.config', 'oxidized'
+    InputDir  = File.join Directory, %w(lib oxidized input)
+    OutputDir = File.join Directory, %w(lib oxidized output)
+    ModelDir  = File.join Directory, %w(lib oxidized model)
+    SourceDir = File.join Directory, %w(lib oxidized source) 
+  end
+  class << self
+    attr_accessor :mgr
+  end
+end
diff --git a/lib/oxidized/core.rb b/lib/oxidized/core.rb
new file mode 100644
index 0000000..76aa330
--- /dev/null
+++ b/lib/oxidized/core.rb
@@ -0,0 +1,21 @@
+module Oxidized
+  require 'oxidized/log'
+  require 'oxidized/config/core'
+  require 'oxidized/worker'
+  require 'oxidized/nodes'
+  require 'oxidized/manager'
+  class << self
+    def new *args
+      Core.new args
+    end
+  end
+
+  class Core
+    def initialize args
+      Oxidized.mgr = Manager.new
+      nodes        = Nodes.new
+      worker       = Worker.new nodes
+      loop { worker.work; sleep 1 }
+    end
+  end
+end
diff --git a/lib/oxidized/fix/grit.rb b/lib/oxidized/fix/grit.rb
new file mode 100644
index 0000000..49be76b
--- /dev/null
+++ b/lib/oxidized/fix/grit.rb
@@ -0,0 +1,18 @@
+Object.send :remove_const, :PACK_IDX_SIGNATURE
+PACK_IDX_SIGNATURE = "\377tOc".b
+
+class String
+  if self.method_defined?(:ord)
+    def getord(offset); self[offset].ord; end
+  else
+    alias :getord :[]
+  end
+
+  unless self.method_defined?(:b)
+    if self.method_defined?(:force_encoding)
+      def b; self.dup.force_encoding(Encoding::ASCII_8BIT); end
+    else
+      def b; self.dup; end
+    end
+  end
+end
diff --git a/lib/oxidized/input/cli.rb b/lib/oxidized/input/cli.rb
new file mode 100644
index 0000000..30a66f4
--- /dev/null
+++ b/lib/oxidized/input/cli.rb
@@ -0,0 +1,26 @@
+module Oxidized
+  class Input
+    module CLI
+  
+      def initialize
+        @post_login = []
+        @pre_logout = []
+      end
+
+      def get
+        @post_login.each { |command| cmd command }
+        d = @node.model.cmds
+        disconnect
+        d
+      end
+  
+      def post_login _post_login
+        @post_login << _post_login unless @exec
+      end
+  
+      def pre_logout _pre_logout
+        @pre_logout << _pre_logout unless @exec
+      end
+    end
+  end
+end
diff --git a/lib/oxidized/input/input.rb b/lib/oxidized/input/input.rb
new file mode 100644
index 0000000..e028ce4
--- /dev/null
+++ b/lib/oxidized/input/input.rb
@@ -0,0 +1,9 @@
+module Oxidized
+  class Input
+    class << self
+      def inherited klass
+        Oxidized.mgr.loader = { :class => klass }
+      end
+    end
+  end
+end
diff --git a/lib/oxidized/input/ssh.rb b/lib/oxidized/input/ssh.rb
new file mode 100644
index 0000000..45c89ba
--- /dev/null
+++ b/lib/oxidized/input/ssh.rb
@@ -0,0 +1,78 @@
+module Oxidized
+  require 'net/ssh'
+  require 'oxidized/input/cli'
+  class SSH < Input
+    include CLI
+    class NoShell < StandardError; end
+
+    def connect node
+      @node       = node
+      @output     = ''
+      @node.model.cfg['ssh'].each { |cb| instance_exec &cb }
+      begin
+        @ssh = Net::SSH.start @node.ip, @node.auth[:username],
+                              :password => @node.auth[:password], :timeout => CFG.timeout
+      rescue Timeout::Error, Net::SSH::Disconnect
+        return false
+      end
+      @ses = open_shell @ssh unless @exec
+      not @ssh.closed?
+    end
+
+    def cmd cmd, expect=@node.prompt
+      Log.debug "SSH: #{cmd} @ #{@node.name}"
+      if @exec
+        @ssh.exec! cmd
+      else
+        cmd_shell(cmd, expect).gsub(/\r\n/, "\n")
+      end
+    end
+
+    private
+
+    def disconnect
+      begin
+        @pre_logout.each { |command| cmd command }
+        @ssh.loop
+        @ssh.close if not @ssh.closed?
+      rescue Net::SSH::Disconnect
+      end
+    end
+
+    def open_shell ssh
+      ses = ssh.open_channel do |ch|
+        ch.on_data do |ch, data|
+          @output << data
+        end
+        ch.request_pty do |ch, success|
+          raise NoShell, "Can't get PTY" unless success
+          ch.send_channel_request 'shell' do |ch, success|
+            raise NoShell, "Can't get shell" unless success
+          end
+        end
+      end
+      expect @node.prompt
+      ses
+    end
+
+    def exec state=nil
+      state == nil ? @exec : (@exec=state)
+    end
+
+    def cmd_shell(cmd, expect_re)
+      @output = ''
+      @ses.send_data cmd + "\n"
+      @ses.process
+      expect expect_re if expect_re
+      @output
+    end
+  
+    def expect regexp
+      @ssh.loop(0.1) do
+        sleep 0.1
+        not @output.match regexp
+      end
+    end
+
+  end
+end
diff --git a/lib/oxidized/input/telnet.rb b/lib/oxidized/input/telnet.rb
new file mode 100644
index 0000000..6dae2d6
--- /dev/null
+++ b/lib/oxidized/input/telnet.rb
@@ -0,0 +1,51 @@
+module Oxidized
+  require 'net/telnet'
+  require 'oxidized/input/cli'
+  class Telnet < Input
+    include CLI 
+    attr_reader :telnet
+
+    def connect node
+      @node    = node
+      @timeout = CFG.timeout
+      @node.model.cfg['telnet'].each { |cb| instance_exec &cb }
+      begin
+        @telnet  = Net::Telnet.new 'Host' => @node.ip, 'Waittime' => @timeout
+        expect username
+        @telnet.puts @node.auth[:username]
+        expect password
+        @telnet.puts @node.auth[:password]
+        expect @node.prompt
+      rescue  Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout
+        return false
+      end
+    end
+
+    def cmd cmd, expect=@node.prompt
+      Log.debug "Telnet: #{cmd} @#{@node.name}"
+      args = { 'String' => cmd }
+      args.merge!({ 'Match' => expect, 'Timeout' => @timeout }) if expect
+      @telnet.cmd args
+    end
+
+    private
+
+    def expect re
+      @telnet.waitfor 'Match' => re, 'Timeout' => @timeout
+    end
+
+    def disconnect
+      @pre_logout.each { |command| cmd(command, nil) }
+      @telnet.close
+    end
+
+    def username re=/^(Username|login)/
+      @username or @username = re
+    end
+
+    def password re=/^Password/
+      @password or @password = re
+    end
+
+  end
+end
diff --git a/lib/oxidized/job.rb b/lib/oxidized/job.rb
new file mode 100644
index 0000000..6921c2b
--- /dev/null
+++ b/lib/oxidized/job.rb
@@ -0,0 +1,14 @@
+module Oxidized
+  class Job < Thread
+    attr_reader :start, :end, :status, :time, :node, :config
+    def initialize node
+      @node         = node
+      @start        = Time.now.utc
+      super do |node|
+        @status, @config = node.run 
+        @end             = Time.now.utc
+        @time            = @end - @start
+      end
+    end
+  end
+end
diff --git a/lib/oxidized/jobs.rb b/lib/oxidized/jobs.rb
new file mode 100644
index 0000000..6476744
--- /dev/null
+++ b/lib/oxidized/jobs.rb
@@ -0,0 +1,24 @@
+module Oxidized
+  class Jobs < Array
+    attr_accessor :interval, :duration, :max, :want
+    def initialize max, interval, nodes
+      @max       = max
+      #@interval  = interval * 60
+      @interval  = interval
+      @nodes     = nodes
+      @duration  = 4
+      new_count
+      super()
+    end
+    def duration last
+      @duration = (@duration + last) / 2
+      new_count
+    end
+    def new_count
+      @want = ((@nodes.size * @duration) / @interval).to_i
+      @want = 1 if @want < 1
+      @want = @nodes.size if @want > @nodes.size
+      @want = @max if @want > @max
+    end
+  end
+end
diff --git a/lib/oxidized/log.rb b/lib/oxidized/log.rb
new file mode 100644
index 0000000..db39462
--- /dev/null
+++ b/lib/oxidized/log.rb
@@ -0,0 +1,13 @@
+module Oxidized
+  require 'logger'
+  class Logger < Logger
+    def initialize target=STDOUT
+      super target
+      self.level = Logger::DEBUG
+    end
+    def file= target
+      @logdev = LogDevice.new target
+    end
+  end
+  Log = Logger.new
+end
diff --git a/lib/oxidized/manager.rb b/lib/oxidized/manager.rb
new file mode 100644
index 0000000..0edf9e7
--- /dev/null
+++ b/lib/oxidized/manager.rb
@@ -0,0 +1,45 @@
+module Oxidized
+  require 'oxidized/model/model'
+  require 'oxidized/input/input'
+  require 'oxidized/output/output'
+  require 'oxidized/source/source'
+  class Manager
+    class << self
+      def load dir, file
+        require File.join dir, file+'.rb'
+        obj, Oxidized.mgr.loader =  Oxidized.mgr.loader, nil
+        k = obj[:class].new
+        k.setup if k.respond_to? :setup
+        { file => obj[:class] }
+      end
+    end
+    attr_reader :input, :output, :model, :source
+    attr_accessor :loader
+    def initialize
+      @input  = {}
+      @output = {}
+      @model  = {}
+      @source = {}
+    end
+    def input= method
+      method = Manager.load Config::InputDir, method
+      return false if method.empty?
+      @input.merge! method
+    end
+    def output= method
+      method = Manager.load Config::OutputDir, method
+      return false if method.empty?
+      @output.merge! method
+    end
+    def model= _model
+      _model = Manager.load Config::ModelDir, _model
+      return false if _model.empty?
+      @model.merge! _model 
+    end
+    def source= _source
+      _source = Manager.load Config::SourceDir, _source
+      return false if _source.empty?
+      @source.merge! _source
+    end
+  end
+end
diff --git a/lib/oxidized/model/ios.rb b/lib/oxidized/model/ios.rb
new file mode 100644
index 0000000..a316ccf
--- /dev/null
+++ b/lib/oxidized/model/ios.rb
@@ -0,0 +1,26 @@
+class IOS < Oxidized::Model
+
+  comment  '! '
+
+  cmd 'show running-config' do |cfg|
+    cfg = cfg.each_line.to_a[3..-2].join
+    cfg.sub! /^(ntp clock-period).*/, '! \1'
+    cfg
+  end
+
+  cmd 'show inventory' do |cfg|
+    comment cfg.each_line.to_a[1..-2].join
+  end
+
+  cfg :telnet do
+    username /^Username:/
+    password /^Password:/
+  end
+
+  cfg :telnet, :ssh do
+    post_login 'terminal length 0'
+    post_login 'terminal width 0'
+    pre_logout 'exit'
+  end
+
+end
diff --git a/lib/oxidized/model/junos.rb b/lib/oxidized/model/junos.rb
new file mode 100644
index 0000000..caa6536
--- /dev/null
+++ b/lib/oxidized/model/junos.rb
@@ -0,0 +1,47 @@
+class JunOS < Oxidized::Model
+
+  comment  '# '
+
+  def telnet
+    @input.class.to_s.match /Telnet/
+  end
+
+  cmd 'show configuration' do |cfg|
+    # example how to handle different output from different methods. Other option would be to
+    # pass string to helper method, which checks if top/bottom has prompts and removes
+    cfg = cfg.lines[1..-2].join if telnet
+    cfg
+  end
+
+  cmd 'show version' do |cfg|
+    chassis = model $1 if cfg.match /^Model: (\S+)/
+    comment cfg << chassis.to_s
+  end
+
+  def model chassis
+    case chassis
+    when 'mx960'
+      cmd('show chassis fabric reachability')  { |cfg| comment cfg }
+    end
+  end
+
+  cmd 'show chassis hardware' do |cfg|
+    comment cfg
+  end
+
+  cfg :telnet do
+    username  /^login:/
+    password  /^Password:/
+  end
+
+  cfg :ssh do
+    exec true  # don't run shell, run each command in exec channel
+  end
+
+  cfg :telnet, :ssh do
+    post_login 'set cli screen-length 0'
+    post_login 'set cli screen-width 0'
+    pre_logout 'exit'
+  end
+
+end
diff --git a/lib/oxidized/model/model.rb b/lib/oxidized/model/model.rb
new file mode 100644
index 0000000..6f02659
--- /dev/null
+++ b/lib/oxidized/model/model.rb
@@ -0,0 +1,70 @@
+module Oxidized
+  class Model
+    class << self
+      def inherited klass
+        klass.instance_variable_set '@cmd', []
+        klass.instance_variable_set '@cfg', Hash.new { |h,k| h[k] = [] }
+        Oxidized.mgr.loader = { :class => klass }
+      end
+      def comment _comment='# '
+        return @comment if @comment
+        @comment = block_given? ? yield : _comment
+      end
+      def cfg *methods, &block
+        [methods].flatten.each do |method|
+          @cfg[method.to_s] << block
+        end
+      end
+      def prompt _prompt=nil
+        @prompt or @prompt = _prompt
+      end
+      def cfgs
+        @cfg
+      end
+      def cmd _cmd=nil, &block
+        @cmd << [_cmd, block]
+      end
+      def cmds
+        @cmd
+      end
+      def post_login &block
+        @post_login or @post_login = block
+      end
+    end
+
+    attr_accessor :input
+
+    def cmd string
+      out = @input.cmd string
+      out = yield out if block_given?
+      out
+    end
+
+    def  cfg
+      self.class.cfgs
+    end
+
+    def prompt
+      self.class.prompt
+    end
+
+    def cmds
+      data = ''
+      self.class.cmds.each do |cmd, cb|
+        out = @input.cmd cmd
+        out = instance_exec out, &cb if cb
+        data << out.to_s
+      end
+      data
+    end
+
+    def comment _comment
+      data = ''
+      _comment.each_line do |line|
+        data << self.class.comment << line
+      end
+      data
+    end
+
+  end
+end
diff --git a/lib/oxidized/node.rb b/lib/oxidized/node.rb
new file mode 100644
index 0000000..740834a
--- /dev/null
+++ b/lib/oxidized/node.rb
@@ -0,0 +1,69 @@
+module Oxidized
+  require 'resolv'
+  class MethodNotFound < StandardError; end
+  class ModelNotFound < StandardError; end
+  class Node
+    attr_reader :name, :ip, :model, :input, :output, :group, :auth, :prompt
+    attr_accessor :last, :running
+    alias :running? :running
+    def initialize opt
+      @name           = opt[:name]
+      @ip             = Resolv.getaddress @name
+      @group          = opt[:group]
+      @input, @output = resolve_io opt
+      @model          = resolve_model opt
+      @auth           = resolve_auth opt
+      @prompt         = resolve_prompt opt
+    end
+
+    def run
+      status, config = :fail, nil
+      @model.input = input = @input.new
+      if input.connect self
+        config = input.get
+        status = :success if config
+      else
+        status = :no_cconnection
+      end
+      [status, config]
+    end
+
+    private
+
+    def resolve_prompt opt
+      prompt =   opt[:prompt]
+      prompt ||= @model.prompt
+      prompt ||= CFG.prompt
+    end
+
+    def resolve_auth opt
+      auth = {}
+      auth[:username] = (opt[:username] or CFG.username)
+      auth[:password] = (opt[:passowrd] or CFG.password)
+      auth
+    end
+
+    def resolve_io opt
+      input  = (opt[:input]  or CFG.input[:default])
+      output = (opt[:output] or CFG.output[:default])
+      mgr = Oxidized.mgr
+      if not mgr.input[input]
+        mgr.input = input or raise MethodNotFound, "#{input} not found"
+      end
+      if not mgr.output[output]
+        mgr.output = output or raise MethodNotFound, "#{output} not found"
+      end
+      [ mgr.input[input], mgr.output[output] ]
+    end
+
+    def resolve_model opt
+      model = (opt[:model] or CFG.model)
+      mgr = Oxidized.mgr
+      if not mgr.model[model]
+        mgr.model = model or raise ModelNotFound, "#{model} not found"
+      end
+      mgr.model[model].new
+    end
+
+  end
+end
diff --git a/lib/oxidized/nodes.rb b/lib/oxidized/nodes.rb
new file mode 100644
index 0000000..467d3a0
--- /dev/null
+++ b/lib/oxidized/nodes.rb
@@ -0,0 +1,36 @@
+module Oxidized
+  require 'oxidized/node'
+  class Nodes < Array
+    attr_accessor :source
+    alias :del :delete
+    def initialize *args
+      super
+      load if args.empty?
+    end
+    def load
+      new = []
+      @source = CFG.source[:default]
+      Oxidized.mgr.source = @source
+      Oxidized.mgr.source[@source].new.load.each do |node|
+        new.push Node.new node
+      end
+      replace new
+    end
+    def list
+      self
+    end
+    # @param node [String] name of the node inserted into nodes array
+    def put node
+      unshift node
+    end
+    # @param node [String] name of the node moved into the head of array
+    def top node
+      n = del node
+      put n if n
+    end
+    # @return [String] node from the head of the array
+    def get
+      (self << shift).last
+    end
+  end
+end
diff --git a/lib/oxidized/output/file.rb b/lib/oxidized/output/file.rb
new file mode 100644
index 0000000..b988c1a
--- /dev/null
+++ b/lib/oxidized/output/file.rb
@@ -0,0 +1,29 @@
+module Oxidized
+class OxFile < Output
+  require 'fileutils'
+
+  def initialize
+    @cfg = CFG.output[:file]
+  end
+
+  def setup
+    if not @cfg
+      CFG.output[:file] = {
+        :directory => File.join(Config::Root, 'configs')
+      }
+      CFG.save
+    end
+  end
+
+  def update node, data, opt={}
+    file = @cfg[:directory]
+    if opt[:group]
+      file = File.join File.dirname(file), opt[:group]
+    end
+    FileUtils.mkdir_p file
+    file = File.join file, node
+    open(file, 'w') { |fh| fh.write data }
+  end
+
+end
+end
diff --git a/lib/oxidized/output/git.rb b/lib/oxidized/output/git.rb
new file mode 100644
index 0000000..77d18bb
--- /dev/null
+++ b/lib/oxidized/output/git.rb
@@ -0,0 +1,57 @@
+module Oxidized
+class Git < Output
+  require 'grit'
+  require 'oxidized/fix/grit' if RUBY_VERSION[0..1] == '2.'
+  include Grit
+
+  def initialize
+    @cfg = CFG.output[:git]
+  end
+
+  def setup
+    if not @cfg
+      CFG.output[:git] = {
+        :user  => 'Oxidized',
+        :email => 'o@example.com',
+        :repo  => File.join(Config::Root, 'oxidized.git')
+      }
+      CFG.save
+    end
+  end
+
+  def update file, data, opt={}
+    msg   = opt[:msg]
+    user  = (opt[:user]  or @cfg[:user])
+    email = (opt[:email] or @cfg[:email])
+    repo  = @cfg[:repo]
+    if opt[:group]
+      repo = File.join File.dirname(repo), opt[:group] + '.git'
+    end
+    begin
+
+      repo = Repo.new repo
+      actor = Actor.new user, email
+      update_repo repo, file, data, msg, actor
+    rescue Grit::NoSuchPathError
+      Repo.init_bare repo
+      retry
+    end
+  end
+
+  private
+
+  def update_repo repo, file, data, msg, actor
+    index  = repo.index
+    index.read_tree 'master'
+    old = index.write_tree index.tree, index.current_tree
+    index.add file, data
+    new = index.write_tree index.tree, index.current_tree
+    if old != new
+      parent = repo.commits(nil, 1).first
+      parent = [parent] if parent
+      Log.debug "GIT: comitting #{file}"
+      index.commit msg, parent, actor
+    end
+  end
+end
+end
diff --git a/lib/oxidized/output/output.rb b/lib/oxidized/output/output.rb
new file mode 100644
index 0000000..61cb2b5
--- /dev/null
+++ b/lib/oxidized/output/output.rb
@@ -0,0 +1,9 @@
+module Oxidized
+  class Output
+    class << self
+      def inherited klass
+        Oxidized.mgr.loader = { :class => klass }
+      end
+    end
+  end
+end
diff --git a/lib/oxidized/source/csv.rb b/lib/oxidized/source/csv.rb
new file mode 100644
index 0000000..6b08b18
--- /dev/null
+++ b/lib/oxidized/source/csv.rb
@@ -0,0 +1,37 @@
+module Oxidized
+class CSV < Source
+  def initialize
+    @cfg = CFG.source[:csv]
+    super
+  end
+
+  def setup
+    if not @cfg
+      CFG.source[:csv] = {
+        :file      => File.join(Config::Root, 'router.db'),
+        :delimiter => /:/,
+        :map       => {
+          :name  => 0,
+          :model => 1,
+        }
+      }
+    end
+    CFG.save
+  end
+
+  def load
+    nodes = []
+    open(@cfg[:file]).each_line do |line|
+      data  = line.chomp.split @cfg[:delimiter]
+      keys = {}
+      @cfg[:map].each do |key, position|
+        keys[key] = data[position]
+      end
+      keys[:model] = map_model keys[:model] if keys.key? :model
+      nodes << keys
+    end
+    nodes
+  end
+
+end
+end
diff --git a/lib/oxidized/source/source.rb b/lib/oxidized/source/source.rb
new file mode 100644
index 0000000..f5976a0
--- /dev/null
+++ b/lib/oxidized/source/source.rb
@@ -0,0 +1,15 @@
+module Oxidized
+  class Source
+    class << self
+      def inherited klass
+        Oxidized.mgr.loader = { :class => klass }
+      end
+    end
+    def initialize
+      @map = (CFG.model_map or {})
+    end
+    def map_model model
+      @map.key?(model) ? @map[model] : model
+    end
+  end
+end
diff --git a/lib/oxidized/source/sql.rb b/lib/oxidized/source/sql.rb
new file mode 100644
index 0000000..f7e6510
--- /dev/null
+++ b/lib/oxidized/source/sql.rb
@@ -0,0 +1,46 @@
+module Oxidized
+class SQL < Source
+  require 'sequel'
+
+  def initialize
+    super
+    @cfg = CFG.source[:sql]
+  end
+
+  def setup
+    if not @cfg
+      CFG.source[:sql] = {
+        :adapter => 'sqlite',
+        :file    => File.join(Config::Root, 'sqlite.db'),
+        :table   => 'devices',
+        :map     => {
+          :name      => 'name',
+          :model     => 'rancid',
+        }
+      }
+    end
+    CFG.save
+  end
+
+  def load
+    nodes = []
+    case @cfg[:adapter]
+    when 'sqlite'
+      require 'sqlite3'
+      Sequel.sqlite @cfg[:file]
+    end
+    klass = Class.new(Sequel::Model @cfg[:table].to_sym)
+    SQL.send :remove_const, :Nodes if SQL.const_defined? :Nodes
+    SQL.const_set :Nodes, klass
+    @cfg[:map].each { |new,old| Nodes.class_eval "alias #{new.to_sym} #{old.to_sym}" }
+    Nodes.each do |node|
+      keys = {}
+      @cfg[:map].each { |key, _| keys[key] = node.send(key.to_sym) }
+      keys[:model] = map_model keys[:model] if keys.key? :model
+      nodes << keys
+    end
+    nodes
+  end
+
+end
+end
diff --git a/lib/oxidized/worker.rb b/lib/oxidized/worker.rb
new file mode 100644
index 0000000..adbaa0e
--- /dev/null
+++ b/lib/oxidized/worker.rb
@@ -0,0 +1,34 @@
+module Oxidized
+  require 'oxidized/job'
+  require 'oxidized/jobs'
+  class Worker
+    def initialize nodes
+      @nodes   = nodes
+      @jobs    = Jobs.new CFG.threads, CFG.interval, @nodes
+      Thread.abort_on_exception = true
+    end
+    def work
+      ended = []
+      @jobs.delete_if { |job| ended << job if not job.alive? }
+      ended.each      { |job| process job }
+      while @jobs.size < @jobs.want
+        Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}"
+        node = @nodes.get
+        node.running? ? next : node.running = true
+        @jobs.push Job.new node
+      end
+    end
+    def process job
+      node = job.node
+      node.last = job
+      @jobs.duration job.time
+      if job.status == :success
+        node.output.new.update node.name, job.config, 
+                               :msg => "update #{node.name}", :group => node.group
+      else
+        Log.warn "#{node.name} status #{job.status}"
+      end
+      node.running = false
+    end
+  end
+end
diff --git a/lib/tst b/lib/tst
new file mode 100755
index 0000000..b529653
--- /dev/null
+++ b/lib/tst
@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby20
+
+$: << '.'
+require 'pry'
+#require 'pp'
+#require 'rubygems'
+require 'oxidized'
+
+k = Oxidized.new
+
diff --git a/oxidized.gemspec b/oxidized.gemspec
new file mode 100644
index 0000000..879475d
--- /dev/null
+++ b/oxidized.gemspec
@@ -0,0 +1,11 @@
+Gem::Specification.new do |s|
+  s.name              = 'oxidized'
+  s.version           = '0.0.1'
+  s.platform          = Gem::Platform::RUBY
+  s.authors           = [ 'Saku Ytti' ]
+  s.email             = %w( saku@ytti.fi )
+  s.summary           = 'feeble attempt at rancid'
+  s.rubyforge_project = s.name
+  s.files             = `git ls-files`.split("\n")
+  s.require_path      = 'lib'
+end
diff --git a/spec/nodes_spec.rb b/spec/nodes_spec.rb
new file mode 100644
index 0000000..ad51525
--- /dev/null
+++ b/spec/nodes_spec.rb
@@ -0,0 +1,45 @@
+require 'oxidized/nodes'
+
+describe Oxidized::Nodes do
+  before(:each) do
+    @nodes_org = %w(ltt-pe1.hel kes2-rr1.tku tor-peer1.oul
+                hal-p2.tre sav-gr1-sw1.kuo psl-sec-pe1.hel)
+    @nodes = Oxidized::Nodes.new @nodes_org.dup
+  end
+
+  describe '#put' do
+    it 'adds node to top of queue' do
+      node = 'kst-p1.sto'
+      @nodes.put node
+      expect(@nodes).to eq [node] + @nodes_org
+    end
+  end
+
+  describe '#get' do
+    it 'returns node from top of queue' do
+      expect(@nodes.get).to eq @nodes_org.first
+    end
+    it 'moves node from top to bottom' do
+      @nodes.get
+      expect(@nodes).to end_with [@nodes_org.first]
+    end
+    it 'does not change node count' do
+      before = @nodes.size
+      @nodes.get
+      expect(before).to eq @nodes.size
+    end
+  end
+
+  describe '#top' do
+    it 'moves node to top of queue' do
+      node = @nodes[3]
+      @nodes.top node
+      expect(@nodes).to start_with [node]
+    end
+    it 'does not change node count' do
+      before = @nodes.size
+      @nodes.top @nodes[3]
+      expect(before).to eq @nodes.size
+    end
+  end
+end