From a29380762b93737ae6949121010cd9bceb8196b2 Mon Sep 17 00:00:00 2001 From: Steve Kemp Date: Wed, 3 Jun 2015 15:55:28 +0100 Subject: Relocated the binaries to bin/ --- bin/byteback-backup | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100755 bin/byteback-backup (limited to 'bin/byteback-backup') diff --git a/bin/byteback-backup b/bin/byteback-backup new file mode 100755 index 0000000..ac9165a --- /dev/null +++ b/bin/byteback-backup @@ -0,0 +1,264 @@ +#!/usr/bin/ruby +# +# Back up this system to a byteback-enabled server (just some command line +# tools and SSH setup). We aim to make sure this backups are easy, complete +# and safe for most types of hosting customer. +# +# See 'man byteback' for more information. + +require 'resolv' + +$LOAD_PATH.unshift('/usr/lib/byteback') + +require 'trollop' +require 'byteback/util' +require 'byteback/log' +include Byteback::Util +include Byteback::Log + +ME = $PROGRAM_NAME.split('/').last + +opts = Trollop.options do + banner "#{ME}: Back up this system to a byteback-enabled server\n " + + opt :destination, 'Backup destination (i.e. user@host:/path)', + type: :string + + opt :source, 'Source paths', + type: :strings, + default: ['/'] + + opt :exclude, 'Paths to exclude', + type: :strings, + short: 'x' + + opt :verbose, 'Show debugging messages' + + opt :retry_number, 'Number of retries on error', + type: :integer, + default: 3 + + opt :retry_delay, 'Number of seconds between retries after an error', + type: :integer, + default: 300 + + opt :ssh_key, 'SSH key filename', + type: :string, + default: '/etc/byteback/key', + short: 'k' + + opt :help, 'Show this message', + short: 'h' + + banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file. See the rsync man page for information on how this works.\n" +end + +lock_out_other_processes('byteback-backup') + +@ssh_key = opts[:ssh_key] +@verbose = opts[:verbose] ? '--verbose' : nil +@sources = opts[:source] if opts[:source] +@excludes = opts[:exclude] if opts[:exclude] +@destination = opts[:destination] +@retry_number = opts[:retry_number] +@retry_delay = opts[:retry_delay] + +# Read the default destination +if File.exist?('/etc/byteback/destination') + @destination = File.read('/etc/byteback/destination').chomp +end + +# Set the default SSH key +if File.exist?('/etc/byteback/key') + @ssh_key = '/etc/byteback/key' +end + +# +# Check our destination +# +if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ + @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] +else + fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') +end + +# +# Validate & normalise source directories +# +@sources = ['/'] if @sources.nil? + +fatal('No sources specified') if @sources.empty? + +@sources = @sources.map do |s| + s = s.gsub(/\/+/, '/') + fatal("Can't read directory #{s}") unless File.readable?(s) + s +end + +# Automatically exclude anything mounted on a non-local filesystem, plus +# various cache and temporary directories common on Bytemark & Debian +# systems +# +if @excludes.nil? + + PROBABLY_LOCAL = %w( + btrfs + ext2 + ext3 + ext4 + reiserfs + xfs + nilfs + jfs + reiser4 + zfs + rootfs + ) + + COMMON_JUNK = %w( + /swap.file + /tmp + /var/backups/localhost + /var/cache/apt/archives + /var/lib/php5 + /var/tmp + ) + + MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ) + .map(&:to_sym) + + mounts = File.read('/proc/mounts').split("\n").map do |line| + Hash[MOUNT_HEADINGS.zip(line.split(' '))] + end + + @excludes = + + mounts + .select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) } + .map { |m| m[:file] } + + + COMMON_JUNK.select { |f| File.exist?(f) } + +end + +@excludes = @excludes.map do |e| + e.gsub(/\/+/, '/') +end + +fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination + +# +# Test ssh connection is good before we start +# +fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) + +def ssh(*ssh_args) + args = ['ssh', + '-o', 'BatchMode=yes', + '-o', 'ConnectionAttempts=5', + '-o', 'ConnectTimeout=30', + '-o', 'ServerAliveInterval=60', + '-o', 'TCPKeepAlive=yes', + '-x', '-a', + '-i', @ssh_key, + '-l', @destination_user, + @destination_host + ] + + ssh_args + .map { |a| a ? a : '' } + + log_system(*args) +end + +fatal("Could not connect to #{@destination}") unless + ssh('byteback-receive', '--ping', @verbose) == 0 + +# +# Call rsync to copy certain sources, returns exit status (see man rsync) +# +def rsync(*sources) + # Default options include --inplace because we only care about consistency + # at the end of the job, and rsync will do more work for big files without + # it. + # + # The timeout is set to 12 hours - rsync can spend a long time at the + # far end checking over its files at the far end without transfer, so we + # want to wait as long as possible without jeopardising the timing of the + # next backup run. Obviously if rsync itself takes nearly 24 hours for a + # given filesystem, daily backups (with this tool) are out of the question. + # + args = %w( rsync --archive --numeric-ids --delete-delay --inplace --relative --timeout 43200 ) + + args += ['--rsh', "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"] + args << '--verbose' if @verbose + args += @excludes.map { |x| ['--exclude', x] }.flatten + + # + # Add in the rsync excludes and sources files, if present. + # + if File.exist?('/etc/byteback/excludes') + args += ['--exclude-from', '/etc/byteback/excludes'] + end + + # + # Add in an rsync_filter if required. + # + if File.exist?('/etc/byteback/rsync_filter') + args += ['--filter', 'merge /etc/byteback/rsync_filter'] + end + + # + # To add extra rsync flags, a file can be used. This can have flags all on one line, or one per line. + # + if File.exist?('/etc/byteback/rsync_flags') + args += File.readlines('/etc/byteback/rsync_flags').map(&:chomp) + end + + args += ['--rsync-path', 'rsync --fake-super'] + + args += sources + args << @destination + + log_system(*args) +end + +# +# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial +# transfer due to vanished source files", which we treat as success otherwise +# on some hosts the backup process never finishes. +# +RSYNC_EXIT_STATUSES_TO_ACCEPT = [0, 24] +RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] + +# Run the file copy, retrying if necessary +# +loop do + status = rsync(*@sources) + + if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } + break + elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } + + warn "rsync exited with status #{status}" + + if @retry_number > 0 + warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" + @retry_number -= 1 + sleep @retry_delay + redo + else + fatal('Maximum number of rsync retries reached') + end + else + fatal("Fatal rsync error occurred (#{status})") + end +end + +info('Backup completed, requesting snapshot') + +# Mark the backup as done on the other end +# +fatal('Backup could not be marked complete') unless + ssh('byteback-receive', '--complete', @verbose) == 0 + +info('Finished') -- cgit v1.2.1