diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/byteback-backup | 264 | ||||
| -rwxr-xr-x | bin/byteback-prune | 140 | ||||
| -rwxr-xr-x | bin/byteback-receive | 62 | ||||
| -rwxr-xr-x | bin/byteback-restore | 125 | ||||
| -rwxr-xr-x | bin/byteback-setup-client | 70 | ||||
| -rwxr-xr-x | bin/byteback-setup-client-receive | 54 | ||||
| -rwxr-xr-x | bin/byteback-snapshot | 40 |
7 files changed, 755 insertions, 0 deletions
diff --git a/bin/byteback-backup b/bin/byteback-backup new file mode 100755 index 0000000..ac9165a --- /dev/null +++ b/bin/byteback-backup @@ -0,0 +1,264 @@ +#!/usr/bin/ruby +# +# Back up this system to a byteback-enabled server (just some command line +# tools and SSH setup). We aim to make sure this backups are easy, complete +# and safe for most types of hosting customer. +# +# See 'man byteback' for more information. + +require 'resolv' + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'trollop' +require 'byteback/util' +require 'byteback/log' +include Byteback::Util +include Byteback::Log + +ME = $PROGRAM_NAME.split('/').last + +opts = Trollop.options do + banner "#{ME}: Back up this system to a byteback-enabled server\n " + + opt :destination, 'Backup destination (i.e. user@host:/path)', + type: :string + + opt :source, 'Source paths', + type: :strings, + default: ['/'] + + opt :exclude, 'Paths to exclude', + type: :strings, + short: 'x' + + opt :verbose, 'Show debugging messages' + + opt :retry_number, 'Number of retries on error', + type: :integer, + default: 3 + + opt :retry_delay, 'Number of seconds between retries after an error', + type: :integer, + default: 300 + + opt :ssh_key, 'SSH key filename', + type: :string, + default: '/etc/byteback/key', + short: 'k' + + opt :help, 'Show this message', + short: 'h' + + banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file. See the rsync man page for information on how this works.\n" +end + +lock_out_other_processes('byteback-backup') + +@ssh_key = opts[:ssh_key] +@verbose = opts[:verbose] ? '--verbose' : nil +@sources = opts[:source] if opts[:source] +@excludes = opts[:exclude] if opts[:exclude] +@destination = opts[:destination] +@retry_number = opts[:retry_number] +@retry_delay = opts[:retry_delay] + +# Read the default destination +if File.exist?('/etc/byteback/destination') + @destination = File.read('/etc/byteback/destination').chomp +end + +# Set the default SSH key +if File.exist?('/etc/byteback/key') + @ssh_key = '/etc/byteback/key' +end + +# +# Check our destination +# +if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ + @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] +else + fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') +end + +# +# Validate & normalise source directories +# +@sources = ['/'] if @sources.nil? + +fatal('No sources specified') if @sources.empty? + +@sources = @sources.map do |s| + s = s.gsub(/\/+/, '/') + fatal("Can't read directory #{s}") unless File.readable?(s) + s +end + +# Automatically exclude anything mounted on a non-local filesystem, plus +# various cache and temporary directories common on Bytemark & Debian +# systems +# +if @excludes.nil? + + PROBABLY_LOCAL = %w( + btrfs + ext2 + ext3 + ext4 + reiserfs + xfs + nilfs + jfs + reiser4 + zfs + rootfs + ) + + COMMON_JUNK = %w( + /swap.file + /tmp + /var/backups/localhost + /var/cache/apt/archives + /var/lib/php5 + /var/tmp + ) + + MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ) + .map(&:to_sym) + + mounts = File.read('/proc/mounts').split("\n").map do |line| + Hash[MOUNT_HEADINGS.zip(line.split(' '))] + end + + @excludes = + + mounts + .select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) } + .map { |m| m[:file] } + + + COMMON_JUNK.select { |f| File.exist?(f) } + +end + +@excludes = @excludes.map do |e| + e.gsub(/\/+/, '/') +end + +fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination + +# +# Test ssh connection is good before we start +# +fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) + +def ssh(*ssh_args) + args = ['ssh', + '-o', 'BatchMode=yes', + '-o', 'ConnectionAttempts=5', + '-o', 'ConnectTimeout=30', + '-o', 'ServerAliveInterval=60', + '-o', 'TCPKeepAlive=yes', + '-x', '-a', + '-i', @ssh_key, + '-l', @destination_user, + @destination_host + ] + + ssh_args + .map { |a| a ? a : '' } + + log_system(*args) +end + +fatal("Could not connect to #{@destination}") unless + ssh('byteback-receive', '--ping', @verbose) == 0 + +# +# Call rsync to copy certain sources, returns exit status (see man rsync) +# +def rsync(*sources) + # Default options include --inplace because we only care about consistency + # at the end of the job, and rsync will do more work for big files without + # it. + # + # The timeout is set to 12 hours - rsync can spend a long time at the + # far end checking over its files at the far end without transfer, so we + # want to wait as long as possible without jeopardising the timing of the + # next backup run. Obviously if rsync itself takes nearly 24 hours for a + # given filesystem, daily backups (with this tool) are out of the question. + # + args = %w( rsync --archive --numeric-ids --delete-delay --inplace --relative --timeout 43200 ) + + args += ['--rsh', "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"] + args << '--verbose' if @verbose + args += @excludes.map { |x| ['--exclude', x] }.flatten + + # + # Add in the rsync excludes and sources files, if present. + # + if File.exist?('/etc/byteback/excludes') + args += ['--exclude-from', '/etc/byteback/excludes'] + end + + # + # Add in an rsync_filter if required. + # + if File.exist?('/etc/byteback/rsync_filter') + args += ['--filter', 'merge /etc/byteback/rsync_filter'] + end + + # + # To add extra rsync flags, a file can be used. This can have flags all on one line, or one per line. + # + if File.exist?('/etc/byteback/rsync_flags') + args += File.readlines('/etc/byteback/rsync_flags').map(&:chomp) + end + + args += ['--rsync-path', 'rsync --fake-super'] + + args += sources + args << @destination + + log_system(*args) +end + +# +# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial +# transfer due to vanished source files", which we treat as success otherwise +# on some hosts the backup process never finishes. +# +RSYNC_EXIT_STATUSES_TO_ACCEPT = [0, 24] +RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] + +# Run the file copy, retrying if necessary +# +loop do + status = rsync(*@sources) + + if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } + break + elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } + + warn "rsync exited with status #{status}" + + if @retry_number > 0 + warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" + @retry_number -= 1 + sleep @retry_delay + redo + else + fatal('Maximum number of rsync retries reached') + end + else + fatal("Fatal rsync error occurred (#{status})") + end +end + +info('Backup completed, requesting snapshot') + +# Mark the backup as done on the other end +# +fatal('Backup could not be marked complete') unless + ssh('byteback-receive', '--complete', @verbose) == 0 + +info('Finished') diff --git a/bin/byteback-prune b/bin/byteback-prune new file mode 100755 index 0000000..d005289 --- /dev/null +++ b/bin/byteback-prune @@ -0,0 +1,140 @@ +#!/usr/bin/ruby +# +# Program to free up space on the backup-storage volume, by removing +# backups (whether by age, or importance). +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'trollop' +require 'byteback' +require 'sys/filesystem' +include Byteback +include Byteback::Log +include Byteback::Util + +opts = Trollop.options do + banner "Prune old backup directories to ensure there's enough space" + + opt :minpercent, 'Start prune when disk has less than this %age free', + type: :integer, + default: 5 + + opt :maxpercent, 'Stop prune when disk has more than this %age free', + type: :integer, + default: 10 + + opt :list, 'List backups in pruning order, no other action' + + opt :prune, 'Prune the next backup if necessary' + + opt :prune_force, 'Prune the next backup regardless' + + opt :order, "Order backups by 'age' or 'importance'", + type: :string, + default: 'importance' + + opt :verbose, 'Show debugging messages' +end + +@order = opts[:order] +@verbose = opts[:verbose] +@do_list = opts[:list] +@do_prune = opts[:prune] +@do_prune_force = opts[:prune_force] +@minpercent = opts[:minpercent] +@maxpercent = opts[:maxpercent] + +@do_prune = true if @do_prune_force + +fatal('Must specify one of --prune or --list') unless + (@do_prune || @do_list) && + !(@do_prune && @do_list) + +fatal("Must specify --order as 'age' or 'importance'") unless + @order == 'age' || @order == 'importance' + +if BackupDirectory.all.empty? + fatal('No backup directories found, need to run byteback-snapshot') +end + +lock_out_other_processes('byteback-prune') + +@df_history = DiskFreeHistory.new(ENV['HOME']) +begin + @df_history.new_reading! +rescue Errno::ENOSPC + if @do_list + warn("Couldn't write disk history file due to lack of space, ignoring") + else + warn("Couldn't write disk history file due to lack of space, going to --prune-force") + @do_prune = @do_prune_force = true + end +rescue => anything_else + error("Couldn't record disk history of #{@df_history.mountpoint} in #{@df_history.history_file}, installation problem?") + raise +end + +gradient_30m = @df_history.gradient(1800) + +# Check whether we should still be pruning +# +@free = @df_history.list.last.percent_free +PRUNING_FLAG = "#{ENV['HOME']}/.byteback.pruning" + +if @do_prune_force + info('Forcing prune') +elsif @free <= @minpercent && !File.exist?(PRUNING_FLAG) + info("Starting prune #{@free}% -> #{@maxpercent} free") + File.write(PRUNING_FLAG, '') +elsif @free >= @maxpercent && File.exist?(PRUNING_FLAG) + info("Stopping prune, reached #{@free}% free") + File.unlink(PRUNING_FLAG) +elsif File.exist?(PRUNING_FLAG) + info("Continuing prune #{@free}% -> #{@maxpercent}, gradient = #{gradient_30m}") +end + +debug("Disc free #{@free}%, 30m gradient = #{gradient_30m}") + +def snapshots_in_order + list = BackupDirectory.all_snapshots + if @order == 'importance' + Snapshot.sort_by_importance(list) + elsif @order == 'age' + list.sort.reverse + else + fail ArgumentError.new("Unknown snapshot sort method #{method}") + end +end + +snapshots = snapshots_in_order + +if @do_list + print "Backups by #{@order}:\n" + snapshots.each_with_index do |snapshot, index| + print "#{sprintf('% 3d', index)}: #{snapshot.path}\n" + end +end + +# Don't do anything if we've not got two hours of readings +# +unless @do_prune_force + if @df_history.list.last.time - @df_history.list.first.time < 1800 + warn('Not enough disc space history to make a decision') + exit 0 + end +end + +exit 0 unless + (@do_prune && File.exist?(PRUNING_FLAG)) || + @do_prune_force + +exit 0 unless @do_prune_force || gradient_30m == 0 + +if snapshots.empty? + error('No snapshots to delete, is there enough disc space?') + exit 1 +end + +info("Deleting #{snapshots.last.path}") +log_system("/sbin/btrfs subvolume delete #{snapshots.last.path}") diff --git a/bin/byteback-receive b/bin/byteback-receive new file mode 100755 index 0000000..4b949c2 --- /dev/null +++ b/bin/byteback-receive @@ -0,0 +1,62 @@ +#!/usr/bin/ruby +# +# Program to receive backups and run rsync in receive mode. Must check that +# user as authorised by SSH is allowed to access particular directory. + +$LOAD_PATH << '/usr/lib/byteback' + +require 'trollop' +require 'byteback' +include Byteback::Log + +if ENV['SSH_ORIGINAL_COMMAND'] + ARGV.concat(ENV['SSH_ORIGINAL_COMMAND'].split(' ')) +end + +byteback_host = ENV['BYTEBACK_HOST'] +fatal('BYTEBACK_HOST environment not set') unless byteback_host + +byteback_root = ENV['HOME'] + '/' + ENV['BYTEBACK_HOST'] +fatal("#{byteback_root} does not exist") unless File.directory?(byteback_root) + +# +# Force restores to be limited to the hostname we're connecting form +# +if (ARGV[0] == 'restore') + ARGV[0] = 'rsync' + a = [] + ARGV.each do |tmp| + if tmp =~ /^\/(.*)/ + tmp = "#{byteback_host}/#{Regexp.last_match(1).dup}" + end + a.push(tmp) + end + exec(*a) +elsif ARGV[0] == 'rsync' + ARGV[-1] = "#{byteback_root}/current" + exec(*ARGV) +elsif ARGV[0] == 'byteback-snapshot' + ARGV.concat(['--root', "#{byteback_root}"]) + exec(*ARGV) +end + +opts = Trollop.options do + opt :verbose, 'Print diagnostics' + opt :ping, 'Check connection parameters and exit' + opt :list, 'Show backed up files matching the given pattern', type: :string + opt :restore, 'Perform a restoration operation', type: :string + opt :complete, 'Mark current backup as complete' +end + +error('Please only choose one mode') if opts[:ping] && opts[:complete] +if opts[:complete] + system('byteback-snapshot', '--root', byteback_root) +elsif opts[:list] + system("cd #{byteback_root} && find . -print | grep #{opts[:list]}") + exit(0) +elsif opts[:ping] + exit 0 +else + STDERR.print "byteback-receive failed\n" + exit 9 +end diff --git a/bin/byteback-restore b/bin/byteback-restore new file mode 100755 index 0000000..48eea9a --- /dev/null +++ b/bin/byteback-restore @@ -0,0 +1,125 @@ +#!/usr/bin/ruby +# +# Restore a file from the most recent backup, from the remote host. +# + +$LOAD_PATH.unshift('/usr/lib/byteback') +$LOAD_PATH.unshift('./lib/') + +require 'trollop' + +# +# Show an error message and abort. +# +def fatal(str) + STDERR.puts(str) + exit(1) +end + +# +# Run a remote command. +# +def ssh(*ssh_args) + args = ['ssh', + '-o', 'BatchMode=yes', + '-o', 'ConnectionAttempts=5', + '-o', 'ConnectTimeout=30', + '-o', 'ServerAliveInterval=60', + '-o', 'TCPKeepAlive=yes', + '-x', '-a', + '-i', @ssh_key, + '-l', @destination_user, + @destination_host + ] + + ssh_args + .map { |a| a ? a : '' } + + system(*args) +end + +def list_files(pattern) + ssh('byteback-receive', '--list', pattern) +end + +# +# We cannot use plain 'rsync' here because the receiver command will +# see that, and rewrite our arguments. +# +# To cater to this we have to wrap the rsync for the restore and we +# do that by setting "rsync-path" to point to the receiver program. +# +# +def restore_file(path, revision) + cmd = %w( rsync ) + cmd += ['--rsh', 'ssh -o BatchMode=yes -x -a -i /etc/byteback/key -l byteback'] + cmd += ['--rsync-path', 'restore --fake-super'] + cmd += ['-aApzrX', '--numeric-ids'] + cmd += ["#{@destination_host}:/#{revision}/#{path}", '.'] + system(*cmd) +end + +# +# Parse our command-line arguments +# +opts = Trollop.options do + banner "byteback-restore: Restore a file\n " + + opt :file, 'The file to restore', + type: :string + + opt :revision, "The version of the file to restore - default is 'latest'", + type: :string + + opt :destination, 'Backup destination (i.e. user@host:/path)', + type: :string + + opt :ssh_key, 'SSH key filename', + type: :string, + default: '/etc/byteback/key', + short: 'k' +end + +# +# Setup default destination and key. +# +@destination = File.read('/etc/byteback/destination').chomp if + File.exist?('/etc/byteback/destination') +@ssh_key = '/etc/byteback/key' if File.exist?('/etc/byteback/key') + +# +# Allow the command-line to override them. +# +@ssh_key = opts[:ssh_key] unless opts[:ssh_key].nil? +@destination = opts[:destination] unless opts[:destination].nil? + +# +# Check our destination is well-formed +# +if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ + @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] +else + fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') +end + +# +# If the user didn't specify a file then we're not restoring anything, +# and we should abort. +# +if opts[:file].nil? + fatal('You must specify a file to restore') +end + +# +# If the user specified a file, but not a revision, then we list +# the available revisions. +# +if opts[:revision].nil? + list_files(opts[:file]) + exit(0) +end + +# +# Restore a file +# +restore_file(opts[:file], opts[:revision]) +exit(0) diff --git a/bin/byteback-setup-client b/bin/byteback-setup-client new file mode 100755 index 0000000..d8ffbfa --- /dev/null +++ b/bin/byteback-setup-client @@ -0,0 +1,70 @@ +#!/usr/bin/ruby +# +# Run on a client machine to set up backups for the first time +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'fileutils' +require 'trollop' +require 'byteback/util' +require 'byteback/log' +include Byteback::Util +include Byteback::Log + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +def verbose(message) + print "#{message}\n" +end + +opts = Trollop.options do + opt :hostname, 'Set host name for backups', + type: :string + + opt :destination, 'Backup destination (i.e. user@host:/path)', + type: :string +end + +@destination = opts[:destination] +@hostname = opts[:hostname] + +_dummy, @destination_user, @destination_host, colon, @destination_path = + /^(.*)?(?:@)([^:]+)(:)(.*)?$/.match(@destination).to_a + +@destination_user ||= 'byteback' +@destination_path ||= '' +@destination_host ||= @destination + +unless @hostname + @hostname = `hostname -f`.chomp + warn "No hostname set, using #{@hostname}\n" +end + +FileUtils.mkdir_p('/etc/byteback') + +if File.readable?('/etc/byteback/key') + warn "Skipping key generation, delete /etc/byteback/key if that's wrong" +else + + error "Couldn't generate SSH key" unless + system <<-KEYGEN + ssh-keygen -q -t rsa -C "byteback client key" \ + -N "" -f /etc/byteback/key + KEYGEN + +end + +key_pub = File.read('/etc/byteback/key.pub').chomp + +error "Remote setup didn't work" unless + system("ssh -i /etc/byteback/key -l #{@destination_user} #{@destination_host} byteback-setup-client-receive #{@hostname} #{key_pub}") + +File.open('/etc/byteback/destination', 'w') do |f| + f.print "#{@destination_user}@#{@destination_host}:#{@destination_path}" +end + +print "Setup worked! To take your first backup run: byteback-backup --verbose\n" diff --git a/bin/byteback-setup-client-receive b/bin/byteback-setup-client-receive new file mode 100755 index 0000000..3673b6a --- /dev/null +++ b/bin/byteback-setup-client-receive @@ -0,0 +1,54 @@ +#!/usr/bin/ruby +# +# Called by byteback-setup-client to set up a new byteback-setup-client +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'fileutils' +require 'trollop' +require 'byteback/util' +require 'byteback/log' +include Byteback::Util +include Byteback::Log + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +@hostname = ARGV.shift +@pubkey = ARGV.join(' ') + +error('You must call this from byteback-setup-client on remote host') unless + @hostname && + /^ssh/.match(@pubkey) && + ENV['SSH_CONNECTION'] + +@client_ip = ENV['SSH_CONNECTION'].split(' ').first + +Dir.chdir(ENV['HOME']) # don't know why we wouldn't be here + +FileUtils.mkdir_p(@hostname) + +error("Couldn't create btrfs subvolume") unless + system("/sbin/btrfs subvolume create #{@hostname}/current") + +FileUtils.mkdir_p('.ssh') + +if File.exist?('.ssh/authorized_keys') && + File.read('.ssh/authorized_keys').match(@pubkey.split(/\s+/)[1]) + + warn('This key already exists in .ssh/authorized_keys on server, nothing to do!') + +else + + File.open('.ssh/authorized_keys', 'a+') do |fh| + fh.print <<-LINE.gsub(/\n/, '') + "\n" +command="byteback-receive", +from="#{@client_ip}", +environment="BYTEBACK_HOST=#{@hostname}" + #{@pubkey} + LINE + end +end diff --git a/bin/byteback-snapshot b/bin/byteback-snapshot new file mode 100755 index 0000000..fc9aab3 --- /dev/null +++ b/bin/byteback-snapshot @@ -0,0 +1,40 @@ +#!/usr/bin/ruby +# +# Program to create a snapshot and/or rotate a directory of backup snapshots +# using btrfs subvolume commands. +# + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'trollop' +require 'byteback' +include Byteback +include Byteback::Log + +opts = Trollop.options do + opt :root, 'Backups directory (must be a btrfs subvolume)', + type: :string + + opt :snapshot, '(ignored for compatibility)' + + opt :verbose, 'Print diagnostics' +end + +@root = opts[:root] +@verbose = opts[:verbose] + +fatal('--root not readable') unless File.directory?("#{@root}") + +@backups = BackupDirectory.new(@root) +snapshots = @backups.snapshots + +unless snapshots.empty? + last_snapshot_time = snapshots.last.time + fatal('Last snapshot was less than six hours ago') unless + !last_snapshot_time || + Time.now - last_snapshot_time >= 6 * 60 * 60 # FIXME: make configurable +end + +info 'Making new snapshot' +@backups.new_snapshot! +info 'Finished' |