diff options
| author | Steve Kemp <steve@steve.org.uk> | 2015-08-13 12:42:46 +0300 | 
|---|---|---|
| committer | Steve Kemp <steve@steve.org.uk> | 2015-08-13 12:42:46 +0300 | 
| commit | 5d76f4b26a4f538db75f9594a8ab516dae79680a (patch) | |
| tree | 7c66193a66d24db78041209636d9d5dc4232e1fe /bin/byteback-backup | |
| parent | d6154cf7f5530b5e29c4a3b6fb4cdd664c763ac3 (diff) | |
Reorder source-code.
Moved the functions to the top of the script, wrap the body
of the program in a test to ensure we're being invoked.
Diffstat (limited to 'bin/byteback-backup')
| -rwxr-xr-x | bin/byteback-backup | 374 | 
1 files changed, 203 insertions, 171 deletions
| diff --git a/bin/byteback-backup b/bin/byteback-backup index bb9b721..872469d 100755 --- a/bin/byteback-backup +++ b/bin/byteback-backup @@ -16,149 +16,11 @@ require 'byteback/log'  include Byteback::Util  include Byteback::Log -ME = $PROGRAM_NAME.split('/').last -opts = Trollop.options do -  banner "#{ME}: Back up this system to a byteback-enabled server\n " - -  opt :destination, 'Backup destination (i.e. user@host:/path)', -      :type => :string - -  opt :source, 'Source paths', -      :type => :strings, -      :default => ['/'] - -  opt :exclude, 'Paths to exclude', -      :type => :strings, -      :short => 'x' - -  opt :verbose, 'Show debugging messages' - -  opt :retry_number, 'Number of retries on error', -      :type => :integer, -      :default => 3 - -  opt :io_timeout, 'Number of seconds to allow I/O timeout for', -      :type => :integer, -      :default => 10800 - -  opt :retry_delay, 'Number of seconds between retries after an error', -      :type => :integer, -      :default => 300 - -  opt :ssh_key, 'SSH key filename', -      :type => :string, -      :default => '/etc/byteback/key', -      :short => 'k' - -  opt :help, 'Show this message', -      :short => 'h' - -  banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file.  See the rsync man page for information on how this works.\n" -end - -lock_out_other_processes('byteback-backup') - -@ssh_key = opts[:ssh_key] -@verbose = opts[:verbose] ? '--verbose' : nil -@sources = opts[:source] if opts[:source] -@excludes = opts[:exclude] if opts[:exclude] -@destination = opts[:destination] -@retry_number = opts[:retry_number] -@retry_delay = opts[:retry_delay] -@io_timeout = opts[:io_timeout] if opts[:io_timeout] - - -# Read the default destination -if File.exist?('/etc/byteback/destination') -  @destination = File.read('/etc/byteback/destination').chomp -end - -# Set the default SSH key -if File.exist?('/etc/byteback/key') -  @ssh_key = '/etc/byteback/key' -end - -# If we have a local timeout-file then use that -if File.exist?('/etc/byteback/io_timeout') -  @io_timeout = File.foreach('/etc/byteback/io_timeout').first.to_i -end - - -# -# Check our destination -# -if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ -  @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] -else -  fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') -end - -# -# Validate & normalise source directories -# -@sources = ['/'] if @sources.nil? - -fatal('No sources specified') if @sources.empty? - -@sources = @sources.map do |s| -  s = s.gsub(/\/+/, '/') -  fatal("Can't read directory #{s}") unless File.readable?(s) -  s -end - -# Automatically exclude anything mounted on a non-local filesystem, plus -# various cache and temporary directories common on Bytemark & Debian -# systems -# -if @excludes.nil? - -  PROBABLY_LOCAL = %w( -    btrfs -    ext2 -    ext3 -    ext4 -    reiserfs -    xfs -    nilfs -    jfs -    reiser4 -    zfs -    rootfs -  ) - -  COMMON_JUNK = %w( -    /swap.file -    /tmp -    /var/backups/localhost -    /var/cache/apt/archives -    /var/lib/php5 -    /var/tmp -  ) - -  MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ).map(&:to_sym) - -  mounts = File.read('/proc/mounts').split("\n").map do |line| -    Hash[MOUNT_HEADINGS.zip(line.split(' '))] -  end - -  @excludes = - -    mounts.select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) }.map { |m| m[:file] } + COMMON_JUNK.select { |f| File.exist?(f) } - -end - -@excludes = @excludes.map do |e| -  e.gsub(/\/+/, '/') -end - -fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination  # -# Test ssh connection is good before we start +#  Run an ssh-command.  # -fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) -  def ssh(*ssh_args)    args = ['ssh',            '-o', 'BatchMode=yes', @@ -178,8 +40,7 @@ def ssh(*ssh_args)    log_system(*args)  end -fatal("Could not connect to #{@destination}") unless -  ssh('byteback-receive', '--ping', @verbose) == 0 +  #  # Call rsync to copy certain sources, returns exit status (see man rsync) @@ -235,43 +96,214 @@ def rsync(*sources)    log_system(*args)  end -# -# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial -# transfer due to vanished source files", which we treat as success otherwise -# on some hosts the backup process never finishes. -# -RSYNC_EXIT_STATUSES_TO_ACCEPT   = [0, 24] -RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] -# Run the file copy, retrying if necessary -# -loop do -  status = rsync(*@sources) -  if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } -    break -  elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } -    warn "rsync exited with status #{status}" -    if @retry_number > 0 -      warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" -      @retry_number -= 1 -      sleep @retry_delay -      redo +## +##  Entry-point to our code. +## +if __FILE__ == $PROGRAM_NAME + + +  ME = $PROGRAM_NAME.split('/').last + +  opts = Trollop.options do +    banner "#{ME}: Back up this system to a byteback-enabled server\n " + +    opt :destination, 'Backup destination (i.e. user@host:/path)', +        :type => :string + +    opt :source, 'Source paths', +        :type => :strings, +        :default => ['/'] + +    opt :exclude, 'Paths to exclude', +        :type => :strings, +        :short => 'x' + +    opt :verbose, 'Show debugging messages' + +    opt :retry_number, 'Number of retries on error', +        :type => :integer, +        :default => 3 + +    opt :io_timeout, 'Number of seconds to allow I/O timeout for', +        :type => :integer, +        :default => 10800 + +    opt :retry_delay, 'Number of seconds between retries after an error', +        :type => :integer, +        :default => 300 + +    opt :ssh_key, 'SSH key filename', +        :type => :string, +        :default => '/etc/byteback/key', +        :short => 'k' + +    opt :help, 'Show this message', +        :short => 'h' + +    banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file.  See the rsync man page for information on how this works.\n" +  end + + +  # +  #  Abort if we're already running. +  # +  lock_out_other_processes('byteback-backup') + +  @ssh_key = opts[:ssh_key] +  @verbose = opts[:verbose] ? '--verbose' : nil +  @sources = opts[:source] if opts[:source] +  @excludes = opts[:exclude] if opts[:exclude] +  @destination = opts[:destination] +  @retry_number = opts[:retry_number] +  @retry_delay = opts[:retry_delay] +  @io_timeout = opts[:io_timeout] if opts[:io_timeout] + + +  # Read the default destination +  if File.exist?('/etc/byteback/destination') +    @destination = File.read('/etc/byteback/destination').chomp +  end + +  # Set the default SSH key +  if File.exist?('/etc/byteback/key') +    @ssh_key = '/etc/byteback/key' +  end + +  # If we have a local timeout-file then use that +  if File.exist?('/etc/byteback/io_timeout') +    @io_timeout = File.foreach('/etc/byteback/io_timeout').first.to_i +  end + + +  # +  # Check our destination +  # +  if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ +    @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)] +  else +    fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups') +  end + +  # +  # Validate & normalise source directories +  # +  @sources = ['/'] if @sources.nil? + +  fatal('No sources specified') if @sources.empty? + +  @sources = @sources.map do |s| +    s = s.gsub(/\/+/, '/') +    fatal("Can't read directory #{s}") unless File.readable?(s) +    s +  end + +  # Automatically exclude anything mounted on a non-local filesystem, plus +  # various cache and temporary directories common on Bytemark & Debian +  # systems +  # +  if @excludes.nil? + +    PROBABLY_LOCAL = %w( +    btrfs +    ext2 +    ext3 +    ext4 +    reiserfs +    xfs +    nilfs +    jfs +    reiser4 +    zfs +    rootfs +  ) + +    COMMON_JUNK = %w( +    /swap.file +    /tmp +    /var/backups/localhost +    /var/cache/apt/archives +    /var/lib/php5 +    /var/tmp +  ) + +    MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ).map(&:to_sym) + +    mounts = File.read('/proc/mounts').split("\n").map do |line| +      Hash[MOUNT_HEADINGS.zip(line.split(' '))] +    end + +    @excludes = + +      mounts.select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) }.map { |m| m[:file] } + COMMON_JUNK.select { |f| File.exist?(f) } + +  end + +  @excludes = @excludes.map do |e| +    e.gsub(/\/+/, '/') +  end + +  fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination + +  # +  # Test that we have an SSH-key which we can read. +  # +  fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) + +  # +  # Test ssh connection is good before we start +  #   +  fatal("Could not connect to #{@destination}") unless +    ssh('byteback-receive', '--ping', @verbose) == 0 + + +  # +  # We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial +  # transfer due to vanished source files", which we treat as success otherwise +  # on some hosts the backup process never finishes. +  # +  RSYNC_EXIT_STATUSES_TO_ACCEPT   = [0, 24] +  RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30] + +  # Run the file copy, retrying if necessary +  # +  loop do +    status = rsync(*@sources) + +    if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status } +      break +    elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status } + +      warn "rsync exited with status #{status}" + +      if @retry_number > 0 +        warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s" +        @retry_number -= 1 +        sleep @retry_delay +        redo +      else +        fatal('Maximum number of rsync retries reached') +      end      else -      fatal('Maximum number of rsync retries reached') +      fatal("Fatal rsync error occurred (#{status})")      end -  else -    fatal("Fatal rsync error occurred (#{status})")    end -end -info('Backup completed, requesting snapshot') +  info('Backup completed, requesting snapshot') -# Mark the backup as done on the other end -# -fatal('Backup could not be marked complete') unless -  ssh('byteback-receive', '--complete', @verbose) == 0 +  # Mark the backup as done on the other end +  # +  fatal('Backup could not be marked complete') unless +    ssh('byteback-receive', '--complete', @verbose) == 0 -info('Finished') + +  # +  # Run our completion-actions, if any. +  # +  run_parts( "/etc/byteback/post-backup.d/" ) + +  info('Finished') +end | 
