Relocated the binaries to bin/

7 files changed, 755 insertions, 0 deletions

diff --git a/bin/byteback-backup b/bin/byteback-backup
new file mode 100755
index 0000000..ac9165a
--- /dev/null
+++ b/bin/byteback-backup
@@ -0,0 +1,264 @@
+#!/usr/bin/ruby
+#
+# Back up this system to a byteback-enabled server (just some command line
+# tools and SSH setup).  We aim to make sure this backups are easy, complete
+# and safe for most types of hosting customer.
+#
+# See 'man byteback' for more information.
+
+require 'resolv'
+
+$LOAD_PATH.unshift('/usr/lib/byteback')
+
+require 'trollop'
+require 'byteback/util'
+require 'byteback/log'
+include Byteback::Util
+include Byteback::Log
+
+ME = $PROGRAM_NAME.split('/').last
+
+opts = Trollop.options do
+  banner "#{ME}: Back up this system to a byteback-enabled server\n "
+
+  opt :destination, 'Backup destination (i.e. user@host:/path)',
+      type: :string
+
+  opt :source, 'Source paths',
+      type: :strings,
+      default: ['/']
+
+  opt :exclude, 'Paths to exclude',
+      type: :strings,
+      short: 'x'
+
+  opt :verbose, 'Show debugging messages'
+
+  opt :retry_number, 'Number of retries on error',
+      type: :integer,
+      default: 3
+
+  opt :retry_delay, 'Number of seconds between retries after an error',
+      type: :integer,
+      default: 300
+
+  opt :ssh_key, 'SSH key filename',
+      type: :string,
+      default: '/etc/byteback/key',
+      short: 'k'
+
+  opt :help, 'Show this message',
+      short: 'h'
+
+  banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file.  See the rsync man page for information on how this works.\n"
+end
+
+lock_out_other_processes('byteback-backup')
+
+@ssh_key = opts[:ssh_key]
+@verbose = opts[:verbose] ? '--verbose' : nil
+@sources = opts[:source] if opts[:source]
+@excludes = opts[:exclude] if opts[:exclude]
+@destination = opts[:destination]
+@retry_number = opts[:retry_number]
+@retry_delay = opts[:retry_delay]
+
+# Read the default destination
+if File.exist?('/etc/byteback/destination')
+  @destination = File.read('/etc/byteback/destination').chomp
+end
+
+# Set the default SSH key
+if File.exist?('/etc/byteback/key')
+  @ssh_key = '/etc/byteback/key'
+end
+
+#
+# Check our destination
+#
+if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/
+  @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)]
+else
+  fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups')
+end
+
+#
+# Validate & normalise source directories
+#
+@sources = ['/'] if @sources.nil?
+
+fatal('No sources specified') if @sources.empty?
+
+@sources = @sources.map do |s|
+  s = s.gsub(/\/+/, '/')
+  fatal("Can't read directory #{s}") unless File.readable?(s)
+  s
+end
+
+# Automatically exclude anything mounted on a non-local filesystem, plus
+# various cache and temporary directories common on Bytemark & Debian
+# systems
+#
+if @excludes.nil?
+
+  PROBABLY_LOCAL = %w(
+    btrfs
+    ext2
+    ext3
+    ext4
+    reiserfs
+    xfs
+    nilfs
+    jfs
+    reiser4
+    zfs
+    rootfs
+  )
+
+  COMMON_JUNK = %w(
+    /swap.file
+    /tmp
+    /var/backups/localhost
+    /var/cache/apt/archives
+    /var/lib/php5
+    /var/tmp
+  )
+
+  MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno )
+                   .map(&:to_sym)
+
+  mounts = File.read('/proc/mounts').split("\n").map do |line|
+    Hash[MOUNT_HEADINGS.zip(line.split(' '))]
+  end
+
+  @excludes =
+
+    mounts
+    .select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) }
+    .map { |m| m[:file] } +
+
+    COMMON_JUNK.select { |f| File.exist?(f) }
+
+end
+
+@excludes = @excludes.map do |e|
+  e.gsub(/\/+/, '/')
+end
+
+fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination
+
+#
+# Test ssh connection is good before we start
+#
+fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key)
+
+def ssh(*ssh_args)
+  args = ['ssh',
+          '-o', 'BatchMode=yes',
+          '-o', 'ConnectionAttempts=5',
+          '-o', 'ConnectTimeout=30',
+          '-o', 'ServerAliveInterval=60',
+          '-o', 'TCPKeepAlive=yes',
+          '-x', '-a',
+          '-i', @ssh_key,
+          '-l', @destination_user,
+          @destination_host
+         ] +
+         ssh_args
+         .map { |a| a ? a : '' }
+
+  log_system(*args)
+end
+
+fatal("Could not connect to #{@destination}") unless
+  ssh('byteback-receive', '--ping', @verbose) == 0
+
+#
+# Call rsync to copy certain sources, returns exit status (see man rsync)
+#
+def rsync(*sources)
+  # Default options include --inplace because we only care about consistency
+  # at the end of the job, and rsync will do more work for big files without
+  # it.
+  #
+  # The timeout is set to 12 hours - rsync can spend a long time at the
+  # far end checking over its files at the far end without transfer, so we
+  # want to wait as long as possible without jeopardising the timing of the
+  # next backup run.  Obviously if rsync itself takes nearly 24 hours for a
+  # given filesystem, daily backups (with this tool) are out of the question.
+  #
+  args = %w( rsync --archive --numeric-ids --delete-delay --inplace --relative --timeout 43200 )
+
+  args += ['--rsh', "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"]
+  args << '--verbose' if @verbose
+  args += @excludes.map { |x| ['--exclude', x] }.flatten
+
+  #
+  # Add in the rsync excludes and sources files, if present.
+  #
+  if File.exist?('/etc/byteback/excludes')
+    args += ['--exclude-from', '/etc/byteback/excludes']
+  end
+
+  #
+  # Add in an rsync_filter if required.
+  #
+  if File.exist?('/etc/byteback/rsync_filter')
+    args += ['--filter', 'merge /etc/byteback/rsync_filter']
+  end
+
+  #
+  # To add extra rsync flags, a file can be used.  This can have flags all on one line, or one per line.
+  #
+  if File.exist?('/etc/byteback/rsync_flags')
+    args += File.readlines('/etc/byteback/rsync_flags').map(&:chomp)
+  end
+
+  args += ['--rsync-path', 'rsync --fake-super']
+
+  args += sources
+  args << @destination
+
+  log_system(*args)
+end
+
+#
+# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial
+# transfer due to vanished source files", which we treat as success otherwise
+# on some hosts the backup process never finishes.
+#
+RSYNC_EXIT_STATUSES_TO_ACCEPT   = [0, 24]
+RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30]
+
+# Run the file copy, retrying if necessary
+#
+loop do
+  status = rsync(*@sources)
+
+  if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status }
+    break
+  elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status }
+
+    warn "rsync exited with status #{status}"
+
+    if @retry_number > 0
+      warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s"
+      @retry_number -= 1
+      sleep @retry_delay
+      redo
+    else
+      fatal('Maximum number of rsync retries reached')
+    end
+  else
+    fatal("Fatal rsync error occurred (#{status})")
+  end
+end
+
+info('Backup completed, requesting snapshot')
+
+# Mark the backup as done on the other end
+#
+fatal('Backup could not be marked complete') unless
+  ssh('byteback-receive', '--complete', @verbose) == 0
+
+info('Finished')
diff --git a/bin/byteback-prune b/bin/byteback-prune
new file mode 100755
index 0000000..d005289
--- /dev/null
+++ b/bin/byteback-prune
@@ -0,0 +1,140 @@
+#!/usr/bin/ruby
+#
+# Program to free up space on the backup-storage volume, by removing
+# backups (whether by age, or importance).
+#
+
+$LOAD_PATH.unshift('/usr/lib/byteback')
+
+require 'trollop'
+require 'byteback'
+require 'sys/filesystem'
+include Byteback
+include Byteback::Log
+include Byteback::Util
+
+opts = Trollop.options do
+  banner "Prune old backup directories to ensure there's enough space"
+
+  opt :minpercent, 'Start prune when disk has less than this %age free',
+      type: :integer,
+      default: 5
+
+  opt :maxpercent, 'Stop prune when disk has more than this %age free',
+      type: :integer,
+      default: 10
+
+  opt :list, 'List backups in pruning order, no other action'
+
+  opt :prune, 'Prune the next backup if necessary'
+
+  opt :prune_force, 'Prune the next backup regardless'
+
+  opt :order, "Order backups by 'age' or 'importance'",
+      type: :string,
+      default: 'importance'
+
+  opt :verbose, 'Show debugging messages'
+end
+
+@order = opts[:order]
+@verbose = opts[:verbose]
+@do_list = opts[:list]
+@do_prune = opts[:prune]
+@do_prune_force = opts[:prune_force]
+@minpercent = opts[:minpercent]
+@maxpercent = opts[:maxpercent]
+
+@do_prune = true if @do_prune_force
+
+fatal('Must specify one of --prune or --list') unless
+  (@do_prune || @do_list) &&
+  !(@do_prune && @do_list)
+
+fatal("Must specify --order as 'age' or 'importance'") unless
+  @order == 'age' || @order == 'importance'
+
+if BackupDirectory.all.empty?
+  fatal('No backup directories found, need to run byteback-snapshot')
+end
+
+lock_out_other_processes('byteback-prune')
+
+@df_history = DiskFreeHistory.new(ENV['HOME'])
+begin
+  @df_history.new_reading!
+rescue Errno::ENOSPC
+  if @do_list
+    warn("Couldn't write disk history file due to lack of space, ignoring")
+  else
+    warn("Couldn't write disk history file due to lack of space, going to --prune-force")
+    @do_prune = @do_prune_force = true
+  end
+rescue => anything_else
+  error("Couldn't record disk history of #{@df_history.mountpoint} in #{@df_history.history_file}, installation problem?")
+  raise
+end
+
+gradient_30m = @df_history.gradient(1800)
+
+# Check whether we should still be pruning
+#
+@free = @df_history.list.last.percent_free
+PRUNING_FLAG = "#{ENV['HOME']}/.byteback.pruning"
+
+if @do_prune_force
+  info('Forcing prune')
+elsif @free <= @minpercent && !File.exist?(PRUNING_FLAG)
+  info("Starting prune #{@free}% -> #{@maxpercent} free")
+  File.write(PRUNING_FLAG, '')
+elsif @free >= @maxpercent && File.exist?(PRUNING_FLAG)
+  info("Stopping prune, reached #{@free}% free")
+  File.unlink(PRUNING_FLAG)
+elsif File.exist?(PRUNING_FLAG)
+  info("Continuing prune #{@free}% -> #{@maxpercent}, gradient = #{gradient_30m}")
+end
+
+debug("Disc free #{@free}%, 30m gradient = #{gradient_30m}")
+
+def snapshots_in_order
+  list = BackupDirectory.all_snapshots
+  if @order == 'importance'
+    Snapshot.sort_by_importance(list)
+  elsif @order == 'age'
+    list.sort.reverse
+  else
+    fail ArgumentError.new("Unknown snapshot sort method #{method}")
+  end
+end
+
+snapshots = snapshots_in_order
+
+if @do_list
+  print "Backups by #{@order}:\n"
+  snapshots.each_with_index do |snapshot, index|
+    print "#{sprintf('% 3d', index)}: #{snapshot.path}\n"
+  end
+end
+
+# Don't do anything if we've not got two hours of readings
+#
+unless @do_prune_force
+  if @df_history.list.last.time - @df_history.list.first.time < 1800
+    warn('Not enough disc space history to make a decision')
+    exit 0
+  end
+end
+
+exit 0 unless
+  (@do_prune && File.exist?(PRUNING_FLAG)) ||
+  @do_prune_force
+
+exit 0 unless @do_prune_force || gradient_30m == 0
+
+if snapshots.empty?
+  error('No snapshots to delete, is there enough disc space?')
+  exit 1
+end
+
+info("Deleting #{snapshots.last.path}")
+log_system("/sbin/btrfs subvolume delete #{snapshots.last.path}")
diff --git a/bin/byteback-receive b/bin/byteback-receive
new file mode 100755
index 0000000..4b949c2
--- /dev/null
+++ b/bin/byteback-receive
@@ -0,0 +1,62 @@
+#!/usr/bin/ruby
+#
+# Program to receive backups and run rsync in receive mode.  Must check that
+# user as authorised by SSH is allowed to access particular directory.
+
+$LOAD_PATH << '/usr/lib/byteback'
+
+require 'trollop'
+require 'byteback'
+include Byteback::Log
+
+if ENV['SSH_ORIGINAL_COMMAND']
+  ARGV.concat(ENV['SSH_ORIGINAL_COMMAND'].split(' '))
+end
+
+byteback_host = ENV['BYTEBACK_HOST']
+fatal('BYTEBACK_HOST environment not set') unless byteback_host
+
+byteback_root = ENV['HOME'] + '/' + ENV['BYTEBACK_HOST']
+fatal("#{byteback_root} does not exist") unless File.directory?(byteback_root)
+
+#
+#  Force restores to be limited to the hostname we're connecting form
+#
+if (ARGV[0] == 'restore')
+  ARGV[0] = 'rsync'
+  a = []
+  ARGV.each do |tmp|
+    if  tmp =~ /^\/(.*)/
+      tmp = "#{byteback_host}/#{Regexp.last_match(1).dup}"
+    end
+    a.push(tmp)
+  end
+  exec(*a)
+elsif ARGV[0] == 'rsync'
+  ARGV[-1] = "#{byteback_root}/current"
+  exec(*ARGV)
+elsif ARGV[0] == 'byteback-snapshot'
+  ARGV.concat(['--root', "#{byteback_root}"])
+  exec(*ARGV)
+end
+
+opts = Trollop.options do
+  opt :verbose, 'Print diagnostics'
+  opt :ping, 'Check connection parameters and exit'
+  opt :list, 'Show backed up files matching the given pattern', type: :string
+  opt :restore, 'Perform a restoration operation', type: :string
+  opt :complete, 'Mark current backup as complete'
+end
+
+error('Please only choose one mode') if opts[:ping] && opts[:complete]
+if opts[:complete]
+  system('byteback-snapshot', '--root', byteback_root)
+elsif opts[:list]
+  system("cd #{byteback_root} && find . -print | grep #{opts[:list]}")
+  exit(0)
+elsif opts[:ping]
+  exit 0
+else
+  STDERR.print "byteback-receive failed\n"
+  exit 9
+end
diff --git a/bin/byteback-restore b/bin/byteback-restore
new file mode 100755
index 0000000..48eea9a
--- /dev/null
+++ b/bin/byteback-restore
@@ -0,0 +1,125 @@
+#!/usr/bin/ruby
+#
+# Restore a file from the most recent backup, from the remote host.
+#
+
+$LOAD_PATH.unshift('/usr/lib/byteback')
+$LOAD_PATH.unshift('./lib/')
+
+require 'trollop'
+
+#
+#  Show an error message and abort.
+#
+def fatal(str)
+  STDERR.puts(str)
+  exit(1)
+end
+
+#
+#  Run a remote command.
+#
+def ssh(*ssh_args)
+  args = ['ssh',
+          '-o', 'BatchMode=yes',
+          '-o', 'ConnectionAttempts=5',
+          '-o', 'ConnectTimeout=30',
+          '-o', 'ServerAliveInterval=60',
+          '-o', 'TCPKeepAlive=yes',
+          '-x', '-a',
+          '-i', @ssh_key,
+          '-l', @destination_user,
+          @destination_host
+         ] +
+         ssh_args
+         .map { |a| a ? a : '' }
+
+  system(*args)
+end
+
+def list_files(pattern)
+  ssh('byteback-receive', '--list', pattern)
+end
+
+#
+#  We cannot use plain 'rsync' here because the receiver command will
+# see that, and rewrite our arguments.
+#
+#  To cater to this we have to wrap the rsync for the restore and we
+# do that by setting "rsync-path" to point to the receiver program.
+#
+#
+def restore_file(path, revision)
+  cmd = %w( rsync )
+  cmd += ['--rsh', 'ssh -o BatchMode=yes -x -a -i /etc/byteback/key -l byteback']
+  cmd += ['--rsync-path', 'restore --fake-super']
+  cmd += ['-aApzrX',  '--numeric-ids']
+  cmd += ["#{@destination_host}:/#{revision}/#{path}", '.']
+  system(*cmd)
+end
+
+#
+#  Parse our command-line arguments
+#
+opts = Trollop.options do
+  banner "byteback-restore: Restore a file\n "
+
+  opt :file, 'The file to restore',
+      type: :string
+
+  opt :revision, "The version of the file to restore - default is 'latest'",
+      type: :string
+
+  opt :destination, 'Backup destination (i.e. user@host:/path)',
+      type: :string
+
+  opt :ssh_key, 'SSH key filename',
+      type: :string,
+      default: '/etc/byteback/key',
+      short: 'k'
+end
+
+#
+#  Setup default destination and key.
+#
+@destination = File.read('/etc/byteback/destination').chomp if
+  File.exist?('/etc/byteback/destination')
+@ssh_key = '/etc/byteback/key' if File.exist?('/etc/byteback/key')
+
+#
+#  Allow the command-line to override them.
+#
+@ssh_key = opts[:ssh_key] unless  opts[:ssh_key].nil?
+@destination = opts[:destination] unless  opts[:destination].nil?
+
+#
+# Check our destination is well-formed
+#
+if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/
+  @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)]
+else
+  fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups')
+end
+
+#
+#  If the user didn't specify a file then we're not restoring anything,
+# and we should abort.
+#
+if  opts[:file].nil?
+  fatal('You must specify a file to restore')
+end
+
+#
+#  If the user specified a file, but not a revision, then we list
+# the available revisions.
+#
+if  opts[:revision].nil?
+  list_files(opts[:file])
+  exit(0)
+end
+
+#
+#  Restore a file
+#
+restore_file(opts[:file], opts[:revision])
+exit(0)
diff --git a/bin/byteback-setup-client b/bin/byteback-setup-client
new file mode 100755
index 0000000..d8ffbfa
--- /dev/null
+++ b/bin/byteback-setup-client
@@ -0,0 +1,70 @@
+#!/usr/bin/ruby
+#
+# Run on a client machine to set up backups for the first time
+#
+
+$LOAD_PATH.unshift('/usr/lib/byteback')
+
+require 'fileutils'
+require 'trollop'
+require 'byteback/util'
+require 'byteback/log'
+include Byteback::Util
+include Byteback::Log
+
+def error(message)
+  STDERR.print "*** #{message}\n"
+  exit 1
+end
+
+def verbose(message)
+  print "#{message}\n"
+end
+
+opts = Trollop.options do
+  opt :hostname, 'Set host name for backups',
+      type: :string
+
+  opt :destination, 'Backup destination (i.e. user@host:/path)',
+      type: :string
+end
+
+@destination = opts[:destination]
+@hostname = opts[:hostname]
+
+_dummy, @destination_user, @destination_host, colon, @destination_path =
+                                                     /^(.*)?(?:@)([^:]+)(:)(.*)?$/.match(@destination).to_a
+
+@destination_user ||= 'byteback'
+@destination_path ||= ''
+@destination_host ||= @destination
+
+unless @hostname
+  @hostname = `hostname -f`.chomp
+  warn "No hostname set, using #{@hostname}\n"
+end
+
+FileUtils.mkdir_p('/etc/byteback')
+
+if File.readable?('/etc/byteback/key')
+  warn "Skipping key generation, delete /etc/byteback/key if that's wrong"
+else
+
+  error "Couldn't generate SSH key" unless
+    system <<-KEYGEN
+      ssh-keygen -q -t rsa -C "byteback client key" \
+      -N "" -f /etc/byteback/key
+    KEYGEN
+
+end
+
+key_pub = File.read('/etc/byteback/key.pub').chomp
+
+error "Remote setup didn't work" unless
+  system("ssh -i /etc/byteback/key -l #{@destination_user} #{@destination_host} byteback-setup-client-receive #{@hostname} #{key_pub}")
+
+File.open('/etc/byteback/destination', 'w') do |f|
+  f.print "#{@destination_user}@#{@destination_host}:#{@destination_path}"
+end
+
+print "Setup worked!  To take your first backup run: byteback-backup --verbose\n"
diff --git a/bin/byteback-setup-client-receive b/bin/byteback-setup-client-receive
new file mode 100755
index 0000000..3673b6a
--- /dev/null
+++ b/bin/byteback-setup-client-receive
@@ -0,0 +1,54 @@
+#!/usr/bin/ruby
+#
+# Called by byteback-setup-client to set up a new byteback-setup-client
+#
+
+$LOAD_PATH.unshift('/usr/lib/byteback')
+
+require 'fileutils'
+require 'trollop'
+require 'byteback/util'
+require 'byteback/log'
+include Byteback::Util
+include Byteback::Log
+
+def error(message)
+  STDERR.print "*** #{message}\n"
+  exit 1
+end
+
+@hostname = ARGV.shift
+@pubkey = ARGV.join(' ')
+
+error('You must call this from byteback-setup-client on remote host') unless
+  @hostname &&
+  /^ssh/.match(@pubkey) &&
+  ENV['SSH_CONNECTION']
+
+@client_ip = ENV['SSH_CONNECTION'].split(' ').first
+
+Dir.chdir(ENV['HOME']) # don't know why we wouldn't be here
+
+FileUtils.mkdir_p(@hostname)
+
+error("Couldn't create btrfs subvolume") unless
+  system("/sbin/btrfs subvolume create #{@hostname}/current")
+
+FileUtils.mkdir_p('.ssh')
+
+if File.exist?('.ssh/authorized_keys') &&
+   File.read('.ssh/authorized_keys').match(@pubkey.split(/\s+/)[1])
+
+  warn('This key already exists in .ssh/authorized_keys on server, nothing to do!')
+
+else
+
+  File.open('.ssh/authorized_keys', 'a+') do |fh|
+    fh.print <<-LINE.gsub(/\n/, '') + "\n"
+command="byteback-receive",
+from="#{@client_ip}",
+environment="BYTEBACK_HOST=#{@hostname}"
+ #{@pubkey}
+    LINE
+  end
+end
diff --git a/bin/byteback-snapshot b/bin/byteback-snapshot
new file mode 100755
index 0000000..fc9aab3
--- /dev/null
+++ b/bin/byteback-snapshot
@@ -0,0 +1,40 @@
+#!/usr/bin/ruby
+#
+# Program to create a snapshot and/or rotate a directory of backup snapshots
+# using btrfs subvolume commands.
+#
+
+$LOAD_PATH.unshift('/usr/lib/byteback')
+
+require 'trollop'
+require 'byteback'
+include Byteback
+include Byteback::Log
+
+opts = Trollop.options do
+  opt :root, 'Backups directory (must be a btrfs subvolume)',
+      type: :string
+
+  opt :snapshot, '(ignored for compatibility)'
+
+  opt :verbose, 'Print diagnostics'
+end
+
+@root = opts[:root]
+@verbose = opts[:verbose]
+
+fatal('--root not readable') unless File.directory?("#{@root}")
+
+@backups = BackupDirectory.new(@root)
+snapshots = @backups.snapshots
+
+unless snapshots.empty?
+  last_snapshot_time = snapshots.last.time
+  fatal('Last snapshot was less than six hours ago') unless
+    !last_snapshot_time ||
+    Time.now - last_snapshot_time >= 6 * 60 * 60 # FIXME: make configurable
+end
+
+info 'Making new snapshot'
+@backups.new_snapshot!
+info 'Finished'