diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/byteback-backup | 374 |
1 files changed, 203 insertions, 171 deletions
diff --git a/bin/byteback-backup b/bin/byteback-backup index bb9b721..872469d 100755 --- a/bin/byteback-backup +++ b/bin/byteback-backup @@ -16,149 +16,11 @@ require 'byteback/log' include Byteback::Util include Byteback::Log -ME = $PROGRAM_NAME.split('/').last -opts = Trollop.options do - banner "#{ME}: Back up this system to a byteback-enabled server\n " - - opt :destination, 'Backup destination (i.e. user@host:/path)', - :type => :string - - opt :source, 'Source paths', - :type => :strings, - :default => ['/'] - - opt :exclude, 'Paths to exclude', - :type => :strings, - :short => 'x' - - opt :verbose, 'Show debugging messages' - - opt :retry_number, 'Number of retries on error', - :type => :integer, - :default => 3 - - opt :io_timeout, 'Number of seconds to allow I/O timeout for', - :type => :integer, - :default => 10800 - - opt :retry_delay, 'Number of seconds between retries after an error', - :type => :integer, - :default => 300 - - opt :ssh_key, 'SSH key filename', - :type => :string, - :default => '/etc/byteback/key', - :short => 'k' - - opt :help, 'Show this message', - :short => 'h' - - banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file. See the rsync man page for information on how this works.\n" -end - -lock_out_other_processes('byteback-backup') - -@ssh_key = opts[:ssh_key] -@verbose = opts[:verbose] ? '--verbose' : nil -@sources = opts[:source] if opts[:source] -@excludes = opts[:exclude] if opts[:exclude] -@destination = opts[:destination] -@retry_number = opts[:retry_number] -@retry_delay = opts[:retry_delay] -@io_timeout = opts[:io_timeout] if opts[:io_timeout] - - -# Read the default destination -if File.exist?('/etc/byteback/destination') - @destination = File.read('/etc/byteback/destination').chomp -end - -# Set the default SSH key -if File.exist?('/etc/byteback/key') - @ssh_key = '/etc/byteback/key' -end - -# If we have a local timeout-file then use that -if File.exist?('/etc/byteback/io_timeout') - @io_timeout = File.foreach('/etc/byteback/io_timeout').first.to_i -end - - -# -# Check our destination -# -if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ - @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] -else - fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') -end - -# -# Validate & normalise source directories -# -@sources = ['/'] if @sources.nil? - -fatal('No sources specified') if @sources.empty? - -@sources = @sources.map do |s| - s = s.gsub(/\/+/, '/') - fatal("Can't read directory #{s}") unless File.readable?(s) - s -end - -# Automatically exclude anything mounted on a non-local filesystem, plus -# various cache and temporary directories common on Bytemark & Debian -# systems -# -if @excludes.nil? - - PROBABLY_LOCAL = %w( - btrfs - ext2 - ext3 - ext4 - reiserfs - xfs - nilfs - jfs - reiser4 - zfs - rootfs - ) - - COMMON_JUNK = %w( - /swap.file - /tmp - /var/backups/localhost - /var/cache/apt/archives - /var/lib/php5 - /var/tmp - ) - - MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ).map(&:to_sym) - - mounts = File.read('/proc/mounts').split("\n").map do |line| - Hash[MOUNT_HEADINGS.zip(line.split(' '))] - end - - @excludes = - - mounts.select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) }.map { |m| m[:file] } + COMMON_JUNK.select { |f| File.exist?(f) } - -end - -@excludes = @excludes.map do |e| - e.gsub(/\/+/, '/') -end - -fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination # -# Test ssh connection is good before we start +# Run an ssh-command. # -fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) - def ssh(*ssh_args) args = ['ssh', '-o', 'BatchMode=yes', @@ -178,8 +40,7 @@ def ssh(*ssh_args) log_system(*args) end -fatal("Could not connect to #{@destination}") unless - ssh('byteback-receive', '--ping', @verbose) == 0 + # # Call rsync to copy certain sources, returns exit status (see man rsync) @@ -235,43 +96,214 @@ def rsync(*sources) log_system(*args) end -# -# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial -# transfer due to vanished source files", which we treat as success otherwise -# on some hosts the backup process never finishes. -# -RSYNC_EXIT_STATUSES_TO_ACCEPT = [0, 24] -RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] -# Run the file copy, retrying if necessary -# -loop do - status = rsync(*@sources) - if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } - break - elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } - warn "rsync exited with status #{status}" - if @retry_number > 0 - warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" - @retry_number -= 1 - sleep @retry_delay - redo +## +## Entry-point to our code. +## +if __FILE__ == $PROGRAM_NAME + + + ME = $PROGRAM_NAME.split('/').last + + opts = Trollop.options do + banner "#{ME}: Back up this system to a byteback-enabled server\n " + + opt :destination, 'Backup destination (i.e. user@host:/path)', + :type => :string + + opt :source, 'Source paths', + :type => :strings, + :default => ['/'] + + opt :exclude, 'Paths to exclude', + :type => :strings, + :short => 'x' + + opt :verbose, 'Show debugging messages' + + opt :retry_number, 'Number of retries on error', + :type => :integer, + :default => 3 + + opt :io_timeout, 'Number of seconds to allow I/O timeout for', + :type => :integer, + :default => 10800 + + opt :retry_delay, 'Number of seconds between retries after an error', + :type => :integer, + :default => 300 + + opt :ssh_key, 'SSH key filename', + :type => :string, + :default => '/etc/byteback/key', + :short => 'k' + + opt :help, 'Show this message', + :short => 'h' + + banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file. See the rsync man page for information on how this works.\n" + end + + + # + # Abort if we're already running. + # + lock_out_other_processes('byteback-backup') + + @ssh_key = opts[:ssh_key] + @verbose = opts[:verbose] ? '--verbose' : nil + @sources = opts[:source] if opts[:source] + @excludes = opts[:exclude] if opts[:exclude] + @destination = opts[:destination] + @retry_number = opts[:retry_number] + @retry_delay = opts[:retry_delay] + @io_timeout = opts[:io_timeout] if opts[:io_timeout] + + + # Read the default destination + if File.exist?('/etc/byteback/destination') + @destination = File.read('/etc/byteback/destination').chomp + end + + # Set the default SSH key + if File.exist?('/etc/byteback/key') + @ssh_key = '/etc/byteback/key' + end + + # If we have a local timeout-file then use that + if File.exist?('/etc/byteback/io_timeout') + @io_timeout = File.foreach('/etc/byteback/io_timeout').first.to_i + end + + + # + # Check our destination + # + if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ + @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] + else + fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') + end + + # + # Validate & normalise source directories + # + @sources = ['/'] if @sources.nil? + + fatal('No sources specified') if @sources.empty? + + @sources = @sources.map do |s| + s = s.gsub(/\/+/, '/') + fatal("Can't read directory #{s}") unless File.readable?(s) + s + end + + # Automatically exclude anything mounted on a non-local filesystem, plus + # various cache and temporary directories common on Bytemark & Debian + # systems + # + if @excludes.nil? + + PROBABLY_LOCAL = %w( + btrfs + ext2 + ext3 + ext4 + reiserfs + xfs + nilfs + jfs + reiser4 + zfs + rootfs + ) + + COMMON_JUNK = %w( + /swap.file + /tmp + /var/backups/localhost + /var/cache/apt/archives + /var/lib/php5 + /var/tmp + ) + + MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ).map(&:to_sym) + + mounts = File.read('/proc/mounts').split("\n").map do |line| + Hash[MOUNT_HEADINGS.zip(line.split(' '))] + end + + @excludes = + + mounts.select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) }.map { |m| m[:file] } + COMMON_JUNK.select { |f| File.exist?(f) } + + end + + @excludes = @excludes.map do |e| + e.gsub(/\/+/, '/') + end + + fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination + + # + # Test that we have an SSH-key which we can read. + # + fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) + + # + # Test ssh connection is good before we start + # + fatal("Could not connect to #{@destination}") unless + ssh('byteback-receive', '--ping', @verbose) == 0 + + + # + # We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial + # transfer due to vanished source files", which we treat as success otherwise + # on some hosts the backup process never finishes. + # + RSYNC_EXIT_STATUSES_TO_ACCEPT = [0, 24] + RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] + + # Run the file copy, retrying if necessary + # + loop do + status = rsync(*@sources) + + if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } + break + elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } + + warn "rsync exited with status #{status}" + + if @retry_number > 0 + warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" + @retry_number -= 1 + sleep @retry_delay + redo + else + fatal('Maximum number of rsync retries reached') + end else - fatal('Maximum number of rsync retries reached') + fatal("Fatal rsync error occurred (#{status})") end - else - fatal("Fatal rsync error occurred (#{status})") end -end -info('Backup completed, requesting snapshot') + info('Backup completed, requesting snapshot') -# Mark the backup as done on the other end -# -fatal('Backup could not be marked complete') unless - ssh('byteback-receive', '--complete', @verbose) == 0 + # Mark the backup as done on the other end + # + fatal('Backup could not be marked complete') unless + ssh('byteback-receive', '--complete', @verbose) == 0 -info('Finished') + + # + # Run our completion-actions, if any. + # + run_parts( "/etc/byteback/post-backup.d/" ) + + info('Finished') +end |