From a29380762b93737ae6949121010cd9bceb8196b2 Mon Sep 17 00:00:00 2001 From: Steve Kemp Date: Wed, 3 Jun 2015 15:55:28 +0100 Subject: Relocated the binaries to bin/ --- bin/byteback-backup | 264 ++++++++++++++++++++++++++++++++++++++ bin/byteback-prune | 140 ++++++++++++++++++++ bin/byteback-receive | 62 +++++++++ bin/byteback-restore | 125 ++++++++++++++++++ bin/byteback-setup-client | 70 ++++++++++ bin/byteback-setup-client-receive | 54 ++++++++ bin/byteback-snapshot | 40 ++++++ byteback-backup | 264 -------------------------------------- byteback-prune | 140 -------------------- byteback-receive | 62 --------- byteback-restore | 125 ------------------ byteback-setup-client | 70 ---------- byteback-setup-client-receive | 54 -------- byteback-snapshot | 40 ------ 14 files changed, 755 insertions(+), 755 deletions(-) create mode 100755 bin/byteback-backup create mode 100755 bin/byteback-prune create mode 100755 bin/byteback-receive create mode 100755 bin/byteback-restore create mode 100755 bin/byteback-setup-client create mode 100755 bin/byteback-setup-client-receive create mode 100755 bin/byteback-snapshot delete mode 100755 byteback-backup delete mode 100755 byteback-prune delete mode 100755 byteback-receive delete mode 100755 byteback-restore delete mode 100755 byteback-setup-client delete mode 100755 byteback-setup-client-receive delete mode 100755 byteback-snapshot diff --git a/bin/byteback-backup b/bin/byteback-backup new file mode 100755 index 0000000..ac9165a --- /dev/null +++ b/bin/byteback-backup @@ -0,0 +1,264 @@ +#!/usr/bin/ruby +# +# Back up this system to a byteback-enabled server (just some command line +# tools and SSH setup). We aim to make sure this backups are easy, complete +# and safe for most types of hosting customer. +# +# See 'man byteback' for more information. + +require 'resolv' + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'trollop' +require 'byteback/util' +require 'byteback/log' +include Byteback::Util +include Byteback::Log + +ME = $PROGRAM_NAME.split('/').last + +opts = Trollop.options do + banner "#{ME}: Back up this system to a byteback-enabled server\n " + + opt :destination, 'Backup destination (i.e. user@host:/path)', + type: :string + + opt :source, 'Source paths', + type: :strings, + default: ['/'] + + opt :exclude, 'Paths to exclude', + type: :strings, + short: 'x' + + opt :verbose, 'Show debugging messages' + + opt :retry_number, 'Number of retries on error', + type: :integer, + default: 3 + + opt :retry_delay, 'Number of seconds between retries after an error', + type: :integer, + default: 300 + + opt :ssh_key, 'SSH key filename', + type: :string, + default: '/etc/byteback/key', + short: 'k' + + opt :help, 'Show this message', + short: 'h' + + banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file. See the rsync man page for information on how this works.\n" +end + +lock_out_other_processes('byteback-backup') + +@ssh_key = opts[:ssh_key] +@verbose = opts[:verbose] ? '--verbose' : nil +@sources = opts[:source] if opts[:source] +@excludes = opts[:exclude] if opts[:exclude] +@destination = opts[:destination] +@retry_number = opts[:retry_number] +@retry_delay = opts[:retry_delay] + +# Read the default destination +if File.exist?('/etc/byteback/destination') + @destination = File.read('/etc/byteback/destination').chomp +end + +# Set the default SSH key +if File.exist?('/etc/byteback/key') + @ssh_key = '/etc/byteback/key' +end + +# +# Check our destination +# +if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ + @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] +else + fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') +end + +# +# Validate & normalise source directories +# +@sources = ['/'] if @sources.nil? + +fatal('No sources specified') if @sources.empty? + +@sources = @sources.map do |s| + s = s.gsub(/\/+/, '/') + fatal("Can't read directory #{s}") unless File.readable?(s) + s +end + +# Automatically exclude anything mounted on a non-local filesystem, plus +# various cache and temporary directories common on Bytemark & Debian +# systems +# +if @excludes.nil? + + PROBABLY_LOCAL = %w( + btrfs + ext2 + ext3 + ext4 + reiserfs + xfs + nilfs + jfs + reiser4 + zfs + rootfs + ) + + COMMON_JUNK = %w( + /swap.file + /tmp + /var/backups/localhost + /var/cache/apt/archives + /var/lib/php5 + /var/tmp + ) + + MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ) + .map(&:to_sym) + + mounts = File.read('/proc/mounts').split("\n").map do |line| + Hash[MOUNT_HEADINGS.zip(line.split(' '))] + end + + @excludes = + + mounts + .select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) } + .map { |m| m[:file] } + + + COMMON_JUNK.select { |f| File.exist?(f) } + +end + +@excludes = @excludes.map do |e| + e.gsub(/\/+/, '/') +end + +fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination + +# +# Test ssh connection is good before we start +# +fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) + +def ssh(*ssh_args) + args = ['ssh', + '-o', 'BatchMode=yes', + '-o', 'ConnectionAttempts=5', + '-o', 'ConnectTimeout=30', + '-o', 'ServerAliveInterval=60', + '-o', 'TCPKeepAlive=yes', + '-x', '-a', + '-i', @ssh_key, + '-l', @destination_user, + @destination_host + ] + + ssh_args + .map { |a| a ? a : '' } + + log_system(*args) +end + +fatal("Could not connect to #{@destination}") unless + ssh('byteback-receive', '--ping', @verbose) == 0 + +# +# Call rsync to copy certain sources, returns exit status (see man rsync) +# +def rsync(*sources) + # Default options include --inplace because we only care about consistency + # at the end of the job, and rsync will do more work for big files without + # it. + # + # The timeout is set to 12 hours - rsync can spend a long time at the + # far end checking over its files at the far end without transfer, so we + # want to wait as long as possible without jeopardising the timing of the + # next backup run. Obviously if rsync itself takes nearly 24 hours for a + # given filesystem, daily backups (with this tool) are out of the question. + # + args = %w( rsync --archive --numeric-ids --delete-delay --inplace --relative --timeout 43200 ) + + args += ['--rsh', "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"] + args << '--verbose' if @verbose + args += @excludes.map { |x| ['--exclude', x] }.flatten + + # + # Add in the rsync excludes and sources files, if present. + # + if File.exist?('/etc/byteback/excludes') + args += ['--exclude-from', '/etc/byteback/excludes'] + end + + # + # Add in an rsync_filter if required. + # + if File.exist?('/etc/byteback/rsync_filter') + args += ['--filter', 'merge /etc/byteback/rsync_filter'] + end + + # + # To add extra rsync flags, a file can be used. This can have flags all on one line, or one per line. + # + if File.exist?('/etc/byteback/rsync_flags') + args += File.readlines('/etc/byteback/rsync_flags').map(&:chomp) + end + + args += ['--rsync-path', 'rsync --fake-super'] + + args += sources + args << @destination + + log_system(*args) +end + +# +# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial +# transfer due to vanished source files", which we treat as success otherwise +# on some hosts the backup process never finishes. +# +RSYNC_EXIT_STATUSES_TO_ACCEPT = [0, 24] +RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] + +# Run the file copy, retrying if necessary +# +loop do + status = rsync(*@sources) + + if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } + break + elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } + + warn "rsync exited with status #{status}" + + if @retry_number > 0 + warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" + @retry_number -= 1 + sleep @retry_delay + redo + else + fatal('Maximum number of rsync retries reached') + end + else + fatal("Fatal rsync error occurred (#{status})") + end +end + +info('Backup completed, requesting snapshot') + +# Mark the backup as done on the other end +# +fatal('Backup could not be marked complete') unless + ssh('byteback-receive', '--complete', @verbose) == 0 + +info('Finished') diff --git a/bin/byteback-prune b/bin/byteback-prune new file mode 100755 index 0000000..d005289 --- /dev/null +++ b/bin/byteback-prune @@ -0,0 +1,140 @@ +#!/usr/bin/ruby +# +# Program to free up space on the backup-storage volume, by removing +# backups (whether by age, or importance). +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'trollop' +require 'byteback' +require 'sys/filesystem' +include Byteback +include Byteback::Log +include Byteback::Util + +opts = Trollop.options do + banner "Prune old backup directories to ensure there's enough space" + + opt :minpercent, 'Start prune when disk has less than this %age free', + type: :integer, + default: 5 + + opt :maxpercent, 'Stop prune when disk has more than this %age free', + type: :integer, + default: 10 + + opt :list, 'List backups in pruning order, no other action' + + opt :prune, 'Prune the next backup if necessary' + + opt :prune_force, 'Prune the next backup regardless' + + opt :order, "Order backups by 'age' or 'importance'", + type: :string, + default: 'importance' + + opt :verbose, 'Show debugging messages' +end + +@order = opts[:order] +@verbose = opts[:verbose] +@do_list = opts[:list] +@do_prune = opts[:prune] +@do_prune_force = opts[:prune_force] +@minpercent = opts[:minpercent] +@maxpercent = opts[:maxpercent] + +@do_prune = true if @do_prune_force + +fatal('Must specify one of --prune or --list') unless + (@do_prune || @do_list) && + !(@do_prune && @do_list) + +fatal("Must specify --order as 'age' or 'importance'") unless + @order == 'age' || @order == 'importance' + +if BackupDirectory.all.empty? + fatal('No backup directories found, need to run byteback-snapshot') +end + +lock_out_other_processes('byteback-prune') + +@df_history = DiskFreeHistory.new(ENV['HOME']) +begin + @df_history.new_reading! +rescue Errno::ENOSPC + if @do_list + warn("Couldn't write disk history file due to lack of space, ignoring") + else + warn("Couldn't write disk history file due to lack of space, going to --prune-force") + @do_prune = @do_prune_force = true + end +rescue => anything_else + error("Couldn't record disk history of #{@df_history.mountpoint} in #{@df_history.history_file}, installation problem?") + raise +end + +gradient_30m = @df_history.gradient(1800) + +# Check whether we should still be pruning +# +@free = @df_history.list.last.percent_free +PRUNING_FLAG = "#{ENV['HOME']}/.byteback.pruning" + +if @do_prune_force + info('Forcing prune') +elsif @free <= @minpercent && !File.exist?(PRUNING_FLAG) + info("Starting prune #{@free}% -> #{@maxpercent} free") + File.write(PRUNING_FLAG, '') +elsif @free >= @maxpercent && File.exist?(PRUNING_FLAG) + info("Stopping prune, reached #{@free}% free") + File.unlink(PRUNING_FLAG) +elsif File.exist?(PRUNING_FLAG) + info("Continuing prune #{@free}% -> #{@maxpercent}, gradient = #{gradient_30m}") +end + +debug("Disc free #{@free}%, 30m gradient = #{gradient_30m}") + +def snapshots_in_order + list = BackupDirectory.all_snapshots + if @order == 'importance' + Snapshot.sort_by_importance(list) + elsif @order == 'age' + list.sort.reverse + else + fail ArgumentError.new("Unknown snapshot sort method #{method}") + end +end + +snapshots = snapshots_in_order + +if @do_list + print "Backups by #{@order}:\n" + snapshots.each_with_index do |snapshot, index| + print "#{sprintf('% 3d', index)}: #{snapshot.path}\n" + end +end + +# Don't do anything if we've not got two hours of readings +# +unless @do_prune_force + if @df_history.list.last.time - @df_history.list.first.time < 1800 + warn('Not enough disc space history to make a decision') + exit 0 + end +end + +exit 0 unless + (@do_prune && File.exist?(PRUNING_FLAG)) || + @do_prune_force + +exit 0 unless @do_prune_force || gradient_30m == 0 + +if snapshots.empty? + error('No snapshots to delete, is there enough disc space?') + exit 1 +end + +info("Deleting #{snapshots.last.path}") +log_system("/sbin/btrfs subvolume delete #{snapshots.last.path}") diff --git a/bin/byteback-receive b/bin/byteback-receive new file mode 100755 index 0000000..4b949c2 --- /dev/null +++ b/bin/byteback-receive @@ -0,0 +1,62 @@ +#!/usr/bin/ruby +# +# Program to receive backups and run rsync in receive mode. Must check that +# user as authorised by SSH is allowed to access particular directory. + +$LOAD_PATH << '/usr/lib/byteback' + +require 'trollop' +require 'byteback' +include Byteback::Log + +if ENV['SSH_ORIGINAL_COMMAND'] + ARGV.concat(ENV['SSH_ORIGINAL_COMMAND'].split(' ')) +end + +byteback_host = ENV['BYTEBACK_HOST'] +fatal('BYTEBACK_HOST environment not set') unless byteback_host + +byteback_root = ENV['HOME'] + '/' + ENV['BYTEBACK_HOST'] +fatal("#{byteback_root} does not exist") unless File.directory?(byteback_root) + +# +# Force restores to be limited to the hostname we're connecting form +# +if (ARGV[0] == 'restore') + ARGV[0] = 'rsync' + a = [] + ARGV.each do |tmp| + if tmp =~ /^\/(.*)/ + tmp = "#{byteback_host}/#{Regexp.last_match(1).dup}" + end + a.push(tmp) + end + exec(*a) +elsif ARGV[0] == 'rsync' + ARGV[-1] = "#{byteback_root}/current" + exec(*ARGV) +elsif ARGV[0] == 'byteback-snapshot' + ARGV.concat(['--root', "#{byteback_root}"]) + exec(*ARGV) +end + +opts = Trollop.options do + opt :verbose, 'Print diagnostics' + opt :ping, 'Check connection parameters and exit' + opt :list, 'Show backed up files matching the given pattern', type: :string + opt :restore, 'Perform a restoration operation', type: :string + opt :complete, 'Mark current backup as complete' +end + +error('Please only choose one mode') if opts[:ping] && opts[:complete] +if opts[:complete] + system('byteback-snapshot', '--root', byteback_root) +elsif opts[:list] + system("cd #{byteback_root} && find . -print | grep #{opts[:list]}") + exit(0) +elsif opts[:ping] + exit 0 +else + STDERR.print "byteback-receive failed\n" + exit 9 +end diff --git a/bin/byteback-restore b/bin/byteback-restore new file mode 100755 index 0000000..48eea9a --- /dev/null +++ b/bin/byteback-restore @@ -0,0 +1,125 @@ +#!/usr/bin/ruby +# +# Restore a file from the most recent backup, from the remote host. +# + +$LOAD_PATH.unshift('/usr/lib/byteback') +$LOAD_PATH.unshift('./lib/') + +require 'trollop' + +# +# Show an error message and abort. +# +def fatal(str) + STDERR.puts(str) + exit(1) +end + +# +# Run a remote command. +# +def ssh(*ssh_args) + args = ['ssh', + '-o', 'BatchMode=yes', + '-o', 'ConnectionAttempts=5', + '-o', 'ConnectTimeout=30', + '-o', 'ServerAliveInterval=60', + '-o', 'TCPKeepAlive=yes', + '-x', '-a', + '-i', @ssh_key, + '-l', @destination_user, + @destination_host + ] + + ssh_args + .map { |a| a ? a : '' } + + system(*args) +end + +def list_files(pattern) + ssh('byteback-receive', '--list', pattern) +end + +# +# We cannot use plain 'rsync' here because the receiver command will +# see that, and rewrite our arguments. +# +# To cater to this we have to wrap the rsync for the restore and we +# do that by setting "rsync-path" to point to the receiver program. +# +# +def restore_file(path, revision) + cmd = %w( rsync ) + cmd += ['--rsh', 'ssh -o BatchMode=yes -x -a -i /etc/byteback/key -l byteback'] + cmd += ['--rsync-path', 'restore --fake-super'] + cmd += ['-aApzrX', '--numeric-ids'] + cmd += ["#{@destination_host}:/#{revision}/#{path}", '.'] + system(*cmd) +end + +# +# Parse our command-line arguments +# +opts = Trollop.options do + banner "byteback-restore: Restore a file\n " + + opt :file, 'The file to restore', + type: :string + + opt :revision, "The version of the file to restore - default is 'latest'", + type: :string + + opt :destination, 'Backup destination (i.e. user@host:/path)', + type: :string + + opt :ssh_key, 'SSH key filename', + type: :string, + default: '/etc/byteback/key', + short: 'k' +end + +# +# Setup default destination and key. +# +@destination = File.read('/etc/byteback/destination').chomp if + File.exist?('/etc/byteback/destination') +@ssh_key = '/etc/byteback/key' if File.exist?('/etc/byteback/key') + +# +# Allow the command-line to override them. +# +@ssh_key = opts[:ssh_key] unless opts[:ssh_key].nil? +@destination = opts[:destination] unless opts[:destination].nil? + +# +# Check our destination is well-formed +# +if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ + @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] +else + fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') +end + +# +# If the user didn't specify a file then we're not restoring anything, +# and we should abort. +# +if opts[:file].nil? + fatal('You must specify a file to restore') +end + +# +# If the user specified a file, but not a revision, then we list +# the available revisions. +# +if opts[:revision].nil? + list_files(opts[:file]) + exit(0) +end + +# +# Restore a file +# +restore_file(opts[:file], opts[:revision]) +exit(0) diff --git a/bin/byteback-setup-client b/bin/byteback-setup-client new file mode 100755 index 0000000..d8ffbfa --- /dev/null +++ b/bin/byteback-setup-client @@ -0,0 +1,70 @@ +#!/usr/bin/ruby +# +# Run on a client machine to set up backups for the first time +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'fileutils' +require 'trollop' +require 'byteback/util' +require 'byteback/log' +include Byteback::Util +include Byteback::Log + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +def verbose(message) + print "#{message}\n" +end + +opts = Trollop.options do + opt :hostname, 'Set host name for backups', + type: :string + + opt :destination, 'Backup destination (i.e. user@host:/path)', + type: :string +end + +@destination = opts[:destination] +@hostname = opts[:hostname] + +_dummy, @destination_user, @destination_host, colon, @destination_path = + /^(.*)?(?:@)([^:]+)(:)(.*)?$/.match(@destination).to_a + +@destination_user ||= 'byteback' +@destination_path ||= '' +@destination_host ||= @destination + +unless @hostname + @hostname = `hostname -f`.chomp + warn "No hostname set, using #{@hostname}\n" +end + +FileUtils.mkdir_p('/etc/byteback') + +if File.readable?('/etc/byteback/key') + warn "Skipping key generation, delete /etc/byteback/key if that's wrong" +else + + error "Couldn't generate SSH key" unless + system <<-KEYGEN + ssh-keygen -q -t rsa -C "byteback client key" \ + -N "" -f /etc/byteback/key + KEYGEN + +end + +key_pub = File.read('/etc/byteback/key.pub').chomp + +error "Remote setup didn't work" unless + system("ssh -i /etc/byteback/key -l #{@destination_user} #{@destination_host} byteback-setup-client-receive #{@hostname} #{key_pub}") + +File.open('/etc/byteback/destination', 'w') do |f| + f.print "#{@destination_user}@#{@destination_host}:#{@destination_path}" +end + +print "Setup worked! To take your first backup run: byteback-backup --verbose\n" diff --git a/bin/byteback-setup-client-receive b/bin/byteback-setup-client-receive new file mode 100755 index 0000000..3673b6a --- /dev/null +++ b/bin/byteback-setup-client-receive @@ -0,0 +1,54 @@ +#!/usr/bin/ruby +# +# Called by byteback-setup-client to set up a new byteback-setup-client +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'fileutils' +require 'trollop' +require 'byteback/util' +require 'byteback/log' +include Byteback::Util +include Byteback::Log + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +@hostname = ARGV.shift +@pubkey = ARGV.join(' ') + +error('You must call this from byteback-setup-client on remote host') unless + @hostname && + /^ssh/.match(@pubkey) && + ENV['SSH_CONNECTION'] + +@client_ip = ENV['SSH_CONNECTION'].split(' ').first + +Dir.chdir(ENV['HOME']) # don't know why we wouldn't be here + +FileUtils.mkdir_p(@hostname) + +error("Couldn't create btrfs subvolume") unless + system("/sbin/btrfs subvolume create #{@hostname}/current") + +FileUtils.mkdir_p('.ssh') + +if File.exist?('.ssh/authorized_keys') && + File.read('.ssh/authorized_keys').match(@pubkey.split(/\s+/)[1]) + + warn('This key already exists in .ssh/authorized_keys on server, nothing to do!') + +else + + File.open('.ssh/authorized_keys', 'a+') do |fh| + fh.print <<-LINE.gsub(/\n/, '') + "\n" +command="byteback-receive", +from="#{@client_ip}", +environment="BYTEBACK_HOST=#{@hostname}" + #{@pubkey} + LINE + end +end diff --git a/bin/byteback-snapshot b/bin/byteback-snapshot new file mode 100755 index 0000000..fc9aab3 --- /dev/null +++ b/bin/byteback-snapshot @@ -0,0 +1,40 @@ +#!/usr/bin/ruby +# +# Program to create a snapshot and/or rotate a directory of backup snapshots +# using btrfs subvolume commands. +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'trollop' +require 'byteback' +include Byteback +include Byteback::Log + +opts = Trollop.options do + opt :root, 'Backups directory (must be a btrfs subvolume)', + type: :string + + opt :snapshot, '(ignored for compatibility)' + + opt :verbose, 'Print diagnostics' +end + +@root = opts[:root] +@verbose = opts[:verbose] + +fatal('--root not readable') unless File.directory?("#{@root}") + +@backups = BackupDirectory.new(@root) +snapshots = @backups.snapshots + +unless snapshots.empty? + last_snapshot_time = snapshots.last.time + fatal('Last snapshot was less than six hours ago') unless + !last_snapshot_time || + Time.now - last_snapshot_time >= 6 * 60 * 60 # FIXME: make configurable +end + +info 'Making new snapshot' +@backups.new_snapshot! +info 'Finished' diff --git a/byteback-backup b/byteback-backup deleted file mode 100755 index ac9165a..0000000 --- a/byteback-backup +++ /dev/null @@ -1,264 +0,0 @@ -#!/usr/bin/ruby -# -# Back up this system to a byteback-enabled server (just some command line -# tools and SSH setup). We aim to make sure this backups are easy, complete -# and safe for most types of hosting customer. -# -# See 'man byteback' for more information. - -require 'resolv' - -$LOAD_PATH.unshift('/usr/lib/byteback') - -require 'trollop' -require 'byteback/util' -require 'byteback/log' -include Byteback::Util -include Byteback::Log - -ME = $PROGRAM_NAME.split('/').last - -opts = Trollop.options do - banner "#{ME}: Back up this system to a byteback-enabled server\n " - - opt :destination, 'Backup destination (i.e. user@host:/path)', - type: :string - - opt :source, 'Source paths', - type: :strings, - default: ['/'] - - opt :exclude, 'Paths to exclude', - type: :strings, - short: 'x' - - opt :verbose, 'Show debugging messages' - - opt :retry_number, 'Number of retries on error', - type: :integer, - default: 3 - - opt :retry_delay, 'Number of seconds between retries after an error', - type: :integer, - default: 300 - - opt :ssh_key, 'SSH key filename', - type: :string, - default: '/etc/byteback/key', - short: 'k' - - opt :help, 'Show this message', - short: 'h' - - banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file. See the rsync man page for information on how this works.\n" -end - -lock_out_other_processes('byteback-backup') - -@ssh_key = opts[:ssh_key] -@verbose = opts[:verbose] ? '--verbose' : nil -@sources = opts[:source] if opts[:source] -@excludes = opts[:exclude] if opts[:exclude] -@destination = opts[:destination] -@retry_number = opts[:retry_number] -@retry_delay = opts[:retry_delay] - -# Read the default destination -if File.exist?('/etc/byteback/destination') - @destination = File.read('/etc/byteback/destination').chomp -end - -# Set the default SSH key -if File.exist?('/etc/byteback/key') - @ssh_key = '/etc/byteback/key' -end - -# -# Check our destination -# -if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ - @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] -else - fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') -end - -# -# Validate & normalise source directories -# -@sources = ['/'] if @sources.nil? - -fatal('No sources specified') if @sources.empty? - -@sources = @sources.map do |s| - s = s.gsub(/\/+/, '/') - fatal("Can't read directory #{s}") unless File.readable?(s) - s -end - -# Automatically exclude anything mounted on a non-local filesystem, plus -# various cache and temporary directories common on Bytemark & Debian -# systems -# -if @excludes.nil? - - PROBABLY_LOCAL = %w( - btrfs - ext2 - ext3 - ext4 - reiserfs - xfs - nilfs - jfs - reiser4 - zfs - rootfs - ) - - COMMON_JUNK = %w( - /swap.file - /tmp - /var/backups/localhost - /var/cache/apt/archives - /var/lib/php5 - /var/tmp - ) - - MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ) - .map(&:to_sym) - - mounts = File.read('/proc/mounts').split("\n").map do |line| - Hash[MOUNT_HEADINGS.zip(line.split(' '))] - end - - @excludes = - - mounts - .select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) } - .map { |m| m[:file] } + - - COMMON_JUNK.select { |f| File.exist?(f) } - -end - -@excludes = @excludes.map do |e| - e.gsub(/\/+/, '/') -end - -fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination - -# -# Test ssh connection is good before we start -# -fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) - -def ssh(*ssh_args) - args = ['ssh', - '-o', 'BatchMode=yes', - '-o', 'ConnectionAttempts=5', - '-o', 'ConnectTimeout=30', - '-o', 'ServerAliveInterval=60', - '-o', 'TCPKeepAlive=yes', - '-x', '-a', - '-i', @ssh_key, - '-l', @destination_user, - @destination_host - ] + - ssh_args - .map { |a| a ? a : '' } - - log_system(*args) -end - -fatal("Could not connect to #{@destination}") unless - ssh('byteback-receive', '--ping', @verbose) == 0 - -# -# Call rsync to copy certain sources, returns exit status (see man rsync) -# -def rsync(*sources) - # Default options include --inplace because we only care about consistency - # at the end of the job, and rsync will do more work for big files without - # it. - # - # The timeout is set to 12 hours - rsync can spend a long time at the - # far end checking over its files at the far end without transfer, so we - # want to wait as long as possible without jeopardising the timing of the - # next backup run. Obviously if rsync itself takes nearly 24 hours for a - # given filesystem, daily backups (with this tool) are out of the question. - # - args = %w( rsync --archive --numeric-ids --delete-delay --inplace --relative --timeout 43200 ) - - args += ['--rsh', "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"] - args << '--verbose' if @verbose - args += @excludes.map { |x| ['--exclude', x] }.flatten - - # - # Add in the rsync excludes and sources files, if present. - # - if File.exist?('/etc/byteback/excludes') - args += ['--exclude-from', '/etc/byteback/excludes'] - end - - # - # Add in an rsync_filter if required. - # - if File.exist?('/etc/byteback/rsync_filter') - args += ['--filter', 'merge /etc/byteback/rsync_filter'] - end - - # - # To add extra rsync flags, a file can be used. This can have flags all on one line, or one per line. - # - if File.exist?('/etc/byteback/rsync_flags') - args += File.readlines('/etc/byteback/rsync_flags').map(&:chomp) - end - - args += ['--rsync-path', 'rsync --fake-super'] - - args += sources - args << @destination - - log_system(*args) -end - -# -# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial -# transfer due to vanished source files", which we treat as success otherwise -# on some hosts the backup process never finishes. -# -RSYNC_EXIT_STATUSES_TO_ACCEPT = [0, 24] -RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] - -# Run the file copy, retrying if necessary -# -loop do - status = rsync(*@sources) - - if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } - break - elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } - - warn "rsync exited with status #{status}" - - if @retry_number > 0 - warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" - @retry_number -= 1 - sleep @retry_delay - redo - else - fatal('Maximum number of rsync retries reached') - end - else - fatal("Fatal rsync error occurred (#{status})") - end -end - -info('Backup completed, requesting snapshot') - -# Mark the backup as done on the other end -# -fatal('Backup could not be marked complete') unless - ssh('byteback-receive', '--complete', @verbose) == 0 - -info('Finished') diff --git a/byteback-prune b/byteback-prune deleted file mode 100755 index d005289..0000000 --- a/byteback-prune +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/ruby -# -# Program to free up space on the backup-storage volume, by removing -# backups (whether by age, or importance). -# - -$LOAD_PATH.unshift('/usr/lib/byteback') - -require 'trollop' -require 'byteback' -require 'sys/filesystem' -include Byteback -include Byteback::Log -include Byteback::Util - -opts = Trollop.options do - banner "Prune old backup directories to ensure there's enough space" - - opt :minpercent, 'Start prune when disk has less than this %age free', - type: :integer, - default: 5 - - opt :maxpercent, 'Stop prune when disk has more than this %age free', - type: :integer, - default: 10 - - opt :list, 'List backups in pruning order, no other action' - - opt :prune, 'Prune the next backup if necessary' - - opt :prune_force, 'Prune the next backup regardless' - - opt :order, "Order backups by 'age' or 'importance'", - type: :string, - default: 'importance' - - opt :verbose, 'Show debugging messages' -end - -@order = opts[:order] -@verbose = opts[:verbose] -@do_list = opts[:list] -@do_prune = opts[:prune] -@do_prune_force = opts[:prune_force] -@minpercent = opts[:minpercent] -@maxpercent = opts[:maxpercent] - -@do_prune = true if @do_prune_force - -fatal('Must specify one of --prune or --list') unless - (@do_prune || @do_list) && - !(@do_prune && @do_list) - -fatal("Must specify --order as 'age' or 'importance'") unless - @order == 'age' || @order == 'importance' - -if BackupDirectory.all.empty? - fatal('No backup directories found, need to run byteback-snapshot') -end - -lock_out_other_processes('byteback-prune') - -@df_history = DiskFreeHistory.new(ENV['HOME']) -begin - @df_history.new_reading! -rescue Errno::ENOSPC - if @do_list - warn("Couldn't write disk history file due to lack of space, ignoring") - else - warn("Couldn't write disk history file due to lack of space, going to --prune-force") - @do_prune = @do_prune_force = true - end -rescue => anything_else - error("Couldn't record disk history of #{@df_history.mountpoint} in #{@df_history.history_file}, installation problem?") - raise -end - -gradient_30m = @df_history.gradient(1800) - -# Check whether we should still be pruning -# -@free = @df_history.list.last.percent_free -PRUNING_FLAG = "#{ENV['HOME']}/.byteback.pruning" - -if @do_prune_force - info('Forcing prune') -elsif @free <= @minpercent && !File.exist?(PRUNING_FLAG) - info("Starting prune #{@free}% -> #{@maxpercent} free") - File.write(PRUNING_FLAG, '') -elsif @free >= @maxpercent && File.exist?(PRUNING_FLAG) - info("Stopping prune, reached #{@free}% free") - File.unlink(PRUNING_FLAG) -elsif File.exist?(PRUNING_FLAG) - info("Continuing prune #{@free}% -> #{@maxpercent}, gradient = #{gradient_30m}") -end - -debug("Disc free #{@free}%, 30m gradient = #{gradient_30m}") - -def snapshots_in_order - list = BackupDirectory.all_snapshots - if @order == 'importance' - Snapshot.sort_by_importance(list) - elsif @order == 'age' - list.sort.reverse - else - fail ArgumentError.new("Unknown snapshot sort method #{method}") - end -end - -snapshots = snapshots_in_order - -if @do_list - print "Backups by #{@order}:\n" - snapshots.each_with_index do |snapshot, index| - print "#{sprintf('% 3d', index)}: #{snapshot.path}\n" - end -end - -# Don't do anything if we've not got two hours of readings -# -unless @do_prune_force - if @df_history.list.last.time - @df_history.list.first.time < 1800 - warn('Not enough disc space history to make a decision') - exit 0 - end -end - -exit 0 unless - (@do_prune && File.exist?(PRUNING_FLAG)) || - @do_prune_force - -exit 0 unless @do_prune_force || gradient_30m == 0 - -if snapshots.empty? - error('No snapshots to delete, is there enough disc space?') - exit 1 -end - -info("Deleting #{snapshots.last.path}") -log_system("/sbin/btrfs subvolume delete #{snapshots.last.path}") diff --git a/byteback-receive b/byteback-receive deleted file mode 100755 index 4b949c2..0000000 --- a/byteback-receive +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/ruby -# -# Program to receive backups and run rsync in receive mode. Must check that -# user as authorised by SSH is allowed to access particular directory. - -$LOAD_PATH << '/usr/lib/byteback' - -require 'trollop' -require 'byteback' -include Byteback::Log - -if ENV['SSH_ORIGINAL_COMMAND'] - ARGV.concat(ENV['SSH_ORIGINAL_COMMAND'].split(' ')) -end - -byteback_host = ENV['BYTEBACK_HOST'] -fatal('BYTEBACK_HOST environment not set') unless byteback_host - -byteback_root = ENV['HOME'] + '/' + ENV['BYTEBACK_HOST'] -fatal("#{byteback_root} does not exist") unless File.directory?(byteback_root) - -# -# Force restores to be limited to the hostname we're connecting form -# -if (ARGV[0] == 'restore') - ARGV[0] = 'rsync' - a = [] - ARGV.each do |tmp| - if tmp =~ /^\/(.*)/ - tmp = "#{byteback_host}/#{Regexp.last_match(1).dup}" - end - a.push(tmp) - end - exec(*a) -elsif ARGV[0] == 'rsync' - ARGV[-1] = "#{byteback_root}/current" - exec(*ARGV) -elsif ARGV[0] == 'byteback-snapshot' - ARGV.concat(['--root', "#{byteback_root}"]) - exec(*ARGV) -end - -opts = Trollop.options do - opt :verbose, 'Print diagnostics' - opt :ping, 'Check connection parameters and exit' - opt :list, 'Show backed up files matching the given pattern', type: :string - opt :restore, 'Perform a restoration operation', type: :string - opt :complete, 'Mark current backup as complete' -end - -error('Please only choose one mode') if opts[:ping] && opts[:complete] -if opts[:complete] - system('byteback-snapshot', '--root', byteback_root) -elsif opts[:list] - system("cd #{byteback_root} && find . -print | grep #{opts[:list]}") - exit(0) -elsif opts[:ping] - exit 0 -else - STDERR.print "byteback-receive failed\n" - exit 9 -end diff --git a/byteback-restore b/byteback-restore deleted file mode 100755 index 48eea9a..0000000 --- a/byteback-restore +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/ruby -# -# Restore a file from the most recent backup, from the remote host. -# - -$LOAD_PATH.unshift('/usr/lib/byteback') -$LOAD_PATH.unshift('./lib/') - -require 'trollop' - -# -# Show an error message and abort. -# -def fatal(str) - STDERR.puts(str) - exit(1) -end - -# -# Run a remote command. -# -def ssh(*ssh_args) - args = ['ssh', - '-o', 'BatchMode=yes', - '-o', 'ConnectionAttempts=5', - '-o', 'ConnectTimeout=30', - '-o', 'ServerAliveInterval=60', - '-o', 'TCPKeepAlive=yes', - '-x', '-a', - '-i', @ssh_key, - '-l', @destination_user, - @destination_host - ] + - ssh_args - .map { |a| a ? a : '' } - - system(*args) -end - -def list_files(pattern) - ssh('byteback-receive', '--list', pattern) -end - -# -# We cannot use plain 'rsync' here because the receiver command will -# see that, and rewrite our arguments. -# -# To cater to this we have to wrap the rsync for the restore and we -# do that by setting "rsync-path" to point to the receiver program. -# -# -def restore_file(path, revision) - cmd = %w( rsync ) - cmd += ['--rsh', 'ssh -o BatchMode=yes -x -a -i /etc/byteback/key -l byteback'] - cmd += ['--rsync-path', 'restore --fake-super'] - cmd += ['-aApzrX', '--numeric-ids'] - cmd += ["#{@destination_host}:/#{revision}/#{path}", '.'] - system(*cmd) -end - -# -# Parse our command-line arguments -# -opts = Trollop.options do - banner "byteback-restore: Restore a file\n " - - opt :file, 'The file to restore', - type: :string - - opt :revision, "The version of the file to restore - default is 'latest'", - type: :string - - opt :destination, 'Backup destination (i.e. user@host:/path)', - type: :string - - opt :ssh_key, 'SSH key filename', - type: :string, - default: '/etc/byteback/key', - short: 'k' -end - -# -# Setup default destination and key. -# -@destination = File.read('/etc/byteback/destination').chomp if - File.exist?('/etc/byteback/destination') -@ssh_key = '/etc/byteback/key' if File.exist?('/etc/byteback/key') - -# -# Allow the command-line to override them. -# -@ssh_key = opts[:ssh_key] unless opts[:ssh_key].nil? -@destination = opts[:destination] unless opts[:destination].nil? - -# -# Check our destination is well-formed -# -if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ - @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] -else - fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') -end - -# -# If the user didn't specify a file then we're not restoring anything, -# and we should abort. -# -if opts[:file].nil? - fatal('You must specify a file to restore') -end - -# -# If the user specified a file, but not a revision, then we list -# the available revisions. -# -if opts[:revision].nil? - list_files(opts[:file]) - exit(0) -end - -# -# Restore a file -# -restore_file(opts[:file], opts[:revision]) -exit(0) diff --git a/byteback-setup-client b/byteback-setup-client deleted file mode 100755 index d8ffbfa..0000000 --- a/byteback-setup-client +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/ruby -# -# Run on a client machine to set up backups for the first time -# - -$LOAD_PATH.unshift('/usr/lib/byteback') - -require 'fileutils' -require 'trollop' -require 'byteback/util' -require 'byteback/log' -include Byteback::Util -include Byteback::Log - -def error(message) - STDERR.print "*** #{message}\n" - exit 1 -end - -def verbose(message) - print "#{message}\n" -end - -opts = Trollop.options do - opt :hostname, 'Set host name for backups', - type: :string - - opt :destination, 'Backup destination (i.e. user@host:/path)', - type: :string -end - -@destination = opts[:destination] -@hostname = opts[:hostname] - -_dummy, @destination_user, @destination_host, colon, @destination_path = - /^(.*)?(?:@)([^:]+)(:)(.*)?$/.match(@destination).to_a - -@destination_user ||= 'byteback' -@destination_path ||= '' -@destination_host ||= @destination - -unless @hostname - @hostname = `hostname -f`.chomp - warn "No hostname set, using #{@hostname}\n" -end - -FileUtils.mkdir_p('/etc/byteback') - -if File.readable?('/etc/byteback/key') - warn "Skipping key generation, delete /etc/byteback/key if that's wrong" -else - - error "Couldn't generate SSH key" unless - system <<-KEYGEN - ssh-keygen -q -t rsa -C "byteback client key" \ - -N "" -f /etc/byteback/key - KEYGEN - -end - -key_pub = File.read('/etc/byteback/key.pub').chomp - -error "Remote setup didn't work" unless - system("ssh -i /etc/byteback/key -l #{@destination_user} #{@destination_host} byteback-setup-client-receive #{@hostname} #{key_pub}") - -File.open('/etc/byteback/destination', 'w') do |f| - f.print "#{@destination_user}@#{@destination_host}:#{@destination_path}" -end - -print "Setup worked! To take your first backup run: byteback-backup --verbose\n" diff --git a/byteback-setup-client-receive b/byteback-setup-client-receive deleted file mode 100755 index 3673b6a..0000000 --- a/byteback-setup-client-receive +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/ruby -# -# Called by byteback-setup-client to set up a new byteback-setup-client -# - -$LOAD_PATH.unshift('/usr/lib/byteback') - -require 'fileutils' -require 'trollop' -require 'byteback/util' -require 'byteback/log' -include Byteback::Util -include Byteback::Log - -def error(message) - STDERR.print "*** #{message}\n" - exit 1 -end - -@hostname = ARGV.shift -@pubkey = ARGV.join(' ') - -error('You must call this from byteback-setup-client on remote host') unless - @hostname && - /^ssh/.match(@pubkey) && - ENV['SSH_CONNECTION'] - -@client_ip = ENV['SSH_CONNECTION'].split(' ').first - -Dir.chdir(ENV['HOME']) # don't know why we wouldn't be here - -FileUtils.mkdir_p(@hostname) - -error("Couldn't create btrfs subvolume") unless - system("/sbin/btrfs subvolume create #{@hostname}/current") - -FileUtils.mkdir_p('.ssh') - -if File.exist?('.ssh/authorized_keys') && - File.read('.ssh/authorized_keys').match(@pubkey.split(/\s+/)[1]) - - warn('This key already exists in .ssh/authorized_keys on server, nothing to do!') - -else - - File.open('.ssh/authorized_keys', 'a+') do |fh| - fh.print <<-LINE.gsub(/\n/, '') + "\n" -command="byteback-receive", -from="#{@client_ip}", -environment="BYTEBACK_HOST=#{@hostname}" - #{@pubkey} - LINE - end -end diff --git a/byteback-snapshot b/byteback-snapshot deleted file mode 100755 index fc9aab3..0000000 --- a/byteback-snapshot +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/ruby -# -# Program to create a snapshot and/or rotate a directory of backup snapshots -# using btrfs subvolume commands. -# - -$LOAD_PATH.unshift('/usr/lib/byteback') - -require 'trollop' -require 'byteback' -include Byteback -include Byteback::Log - -opts = Trollop.options do - opt :root, 'Backups directory (must be a btrfs subvolume)', - type: :string - - opt :snapshot, '(ignored for compatibility)' - - opt :verbose, 'Print diagnostics' -end - -@root = opts[:root] -@verbose = opts[:verbose] - -fatal('--root not readable') unless File.directory?("#{@root}") - -@backups = BackupDirectory.new(@root) -snapshots = @backups.snapshots - -unless snapshots.empty? - last_snapshot_time = snapshots.last.time - fatal('Last snapshot was less than six hours ago') unless - !last_snapshot_time || - Time.now - last_snapshot_time >= 6 * 60 * 60 # FIXME: make configurable -end - -info 'Making new snapshot' -@backups.new_snapshot! -info 'Finished' -- cgit v1.2.1