path: root/bin/byteback-backup

#!/usr/bin/ruby
#
# Back up this system to a byteback-enabled server (just some command line
# tools and SSH setup).  We aim to make sure this backups are easy, complete
# and safe for most types of hosting customer.
#
# See 'man byteback' for more information.

require 'resolv'

$LOAD_PATH.unshift('/usr/lib/byteback')

require 'trollop'
require 'byteback/util'
require 'byteback/log'
include Byteback::Util
include Byteback::Log

ME = $PROGRAM_NAME.split('/').last

opts = Trollop.options do
  banner "#{ME}: Back up this system to a byteback-enabled server\n "

  opt :destination, 'Backup destination (i.e. user@host:/path)',
      :type => :string

  opt :source, 'Source paths',
      :type => :strings,
      :default => ['/']

  opt :exclude, 'Paths to exclude',
      :type => :strings,
      :short => 'x'

  opt :verbose, 'Show debugging messages'

  opt :retry_number, 'Number of retries on error',
      :type => :integer,
      :default => 3

  opt :io_timeout, 'Number of seconds to allow I/O timeout for',
      :type => :integer,
      :default => 10800

  opt :retry_delay, 'Number of seconds between retries after an error',
      :type => :integer,
      :default => 300

  opt :ssh_key, 'SSH key filename',
      :type => :string,
      :default => '/etc/byteback/key',
      :short => 'k'

  opt :help, 'Show this message',
      :short => 'h'

  banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file.  See the rsync man page for information on how this works.\n"
end

lock_out_other_processes('byteback-backup')

@ssh_key = opts[:ssh_key]
@verbose = opts[:verbose] ? '--verbose' : nil
@sources = opts[:source] if opts[:source]
@excludes = opts[:exclude] if opts[:exclude]
@destination = opts[:destination]
@retry_number = opts[:retry_number]
@retry_delay = opts[:retry_delay]
@io_timeout = opts[:io_timeout] if opts[:io_timeout]


# Read the default destination
if File.exist?('/etc/byteback/destination')
  @destination = File.read('/etc/byteback/destination').chomp
end

# Set the default SSH key
if File.exist?('/etc/byteback/key')
  @ssh_key = '/etc/byteback/key'
end

# If we have a local timeout-file then use that
if File.exist?('/etc/byteback/io_timeout')
  @io_timeout = File.foreach('/etc/byteback/io_timeout').first.to_i
end


#
# Check our destination
#
if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/
  @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)]
else
  fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups')
end

#
# Validate & normalise source directories
#
@sources = ['/'] if @sources.nil?

fatal('No sources specified') if @sources.empty?

@sources = @sources.map do |s|
  s = s.gsub(/\/+/, '/')
  fatal("Can't read directory #{s}") unless File.readable?(s)
  s
end

# Automatically exclude anything mounted on a non-local filesystem, plus
# various cache and temporary directories common on Bytemark & Debian
# systems
#
if @excludes.nil?

  PROBABLY_LOCAL = %w(
    btrfs
    ext2
    ext3
    ext4
    reiserfs
    xfs
    nilfs
    jfs
    reiser4
    zfs
    rootfs
  )

  COMMON_JUNK = %w(
    /swap.file
    /tmp
    /var/backups/localhost
    /var/cache/apt/archives
    /var/lib/php5
    /var/tmp
  )

  MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno ).map(&:to_sym)

  mounts = File.read('/proc/mounts').split("\n").map do |line|
    Hash[MOUNT_HEADINGS.zip(line.split(' '))]
  end

  @excludes =

    mounts.select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) }.map { |m| m[:file] } + COMMON_JUNK.select { |f| File.exist?(f) }

end

@excludes = @excludes.map do |e|
  e.gsub(/\/+/, '/')
end

fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination

#
# Test ssh connection is good before we start
#
fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key)

def ssh(*ssh_args)
  args = ['ssh',
          '-o', 'BatchMode=yes',
          '-o', 'ConnectionAttempts=5',
          '-o', 'ConnectTimeout=30',
          '-o', 'Compression=yes',
          '-o', 'CompressionLevel=3',
          '-o', 'ServerAliveInterval=60',
          '-o', 'TCPKeepAlive=yes',
          '-x', '-a',
          '-i', @ssh_key,
          '-l', @destination_user,
          @destination_host
         ] +
         ssh_args.map { |a| a ? a : '' }

  log_system(*args)
end

fatal("Could not connect to #{@destination}") unless
  ssh('byteback-receive', '--ping', @verbose) == 0

#
# Call rsync to copy certain sources, returns exit status (see man rsync)
#
def rsync(*sources)
  # Default options include --inplace because we only care about consistency
  # at the end of the job, and rsync will do more work for big files without
  # it.
  #
  # rsync can spend a long time at the far end checking over its files at
  # the far end without transfer, so we want to wait as long as possible
  # without jeopardising the timing of the next backup run.
  #
  #
  args = %w( rsync --archive --numeric-ids --delete-delay --inplace --relative )

  #
  #  Add on the I/O-timeout
  #
  args += ['--timeout', @io_timeout ] unless ( @io_timeout.nil? )


  args += ['--rsh', "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"]
  args << '--verbose' if @verbose
  args += @excludes.map { |x| ['--exclude', x] }.flatten

  #
  # Add in the rsync excludes and sources files, if present.
  #
  if File.exist?('/etc/byteback/excludes')
    args += ['--exclude-from', '/etc/byteback/excludes']
  end

  #
  # Add in an rsync_filter if required.
  #
  if File.exist?('/etc/byteback/rsync_filter')
    args += ['--filter', 'merge /etc/byteback/rsync_filter']
  end

  #
  # To add extra rsync flags, a file can be used.  This can have flags all on one line, or one per line.
  #
  if File.exist?('/etc/byteback/rsync_flags')
    args += File.readlines('/etc/byteback/rsync_flags').map(&:chomp)
  end

  args += ['--rsync-path', 'rsync --fake-super']

  args += sources
  args << @destination

  log_system(*args)
end

#
# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial
# transfer due to vanished source files", which we treat as success otherwise
# on some hosts the backup process never finishes.
#
RSYNC_EXIT_STATUSES_TO_ACCEPT   = [0, 24]
RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30]

# Run the file copy, retrying if necessary
#
loop do
  status = rsync(*@sources)

  if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status }
    break
  elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status }

    warn "rsync exited with status #{status}"

    if @retry_number > 0
      warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s"
      @retry_number -= 1
      sleep @retry_delay
      redo
    else
      fatal('Maximum number of rsync retries reached')
    end
  else
    fatal("Fatal rsync error occurred (#{status})")
  end
end

info('Backup completed, requesting snapshot')

# Mark the backup as done on the other end
#
fatal('Backup could not be marked complete') unless
  ssh('byteback-receive', '--complete', @verbose) == 0

info('Finished')