Relocated the binaries to bin/

1 files changed, 264 insertions, 0 deletions

diff --git a/bin/byteback-backup b/bin/byteback-backup
new file mode 100755
index 0000000..ac9165a
--- /dev/null
+++ b/bin/byteback-backup
@@ -0,0 +1,264 @@
+#!/usr/bin/ruby
+#
+# Back up this system to a byteback-enabled server (just some command line
+# tools and SSH setup).  We aim to make sure this backups are easy, complete
+# and safe for most types of hosting customer.
+#
+# See 'man byteback' for more information.
+
+require 'resolv'
+
+$LOAD_PATH.unshift('/usr/lib/byteback')
+
+require 'trollop'
+require 'byteback/util'
+require 'byteback/log'
+include Byteback::Util
+include Byteback::Log
+
+ME = $PROGRAM_NAME.split('/').last
+
+opts = Trollop.options do
+  banner "#{ME}: Back up this system to a byteback-enabled server\n "
+
+  opt :destination, 'Backup destination (i.e. user@host:/path)',
+      type: :string
+
+  opt :source, 'Source paths',
+      type: :strings,
+      default: ['/']
+
+  opt :exclude, 'Paths to exclude',
+      type: :strings,
+      short: 'x'
+
+  opt :verbose, 'Show debugging messages'
+
+  opt :retry_number, 'Number of retries on error',
+      type: :integer,
+      default: 3
+
+  opt :retry_delay, 'Number of seconds between retries after an error',
+      type: :integer,
+      default: 300
+
+  opt :ssh_key, 'SSH key filename',
+      type: :string,
+      default: '/etc/byteback/key',
+      short: 'k'
+
+  opt :help, 'Show this message',
+      short: 'h'
+
+  banner "\nAdditional excludes can be specified using /etc/byteback/rsync_filter, which is an rsync filter file.  See the rsync man page for information on how this works.\n"
+end
+
+lock_out_other_processes('byteback-backup')
+
+@ssh_key = opts[:ssh_key]
+@verbose = opts[:verbose] ? '--verbose' : nil
+@sources = opts[:source] if opts[:source]
+@excludes = opts[:exclude] if opts[:exclude]
+@destination = opts[:destination]
+@retry_number = opts[:retry_number]
+@retry_delay = opts[:retry_delay]
+
+# Read the default destination
+if File.exist?('/etc/byteback/destination')
+  @destination = File.read('/etc/byteback/destination').chomp
+end
+
+# Set the default SSH key
+if File.exist?('/etc/byteback/key')
+  @ssh_key = '/etc/byteback/key'
+end
+
+#
+# Check our destination
+#
+if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/
+  @destination_user, @destination_host, @destination_path = [Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3)]
+else
+  fatal('Destination must be a remote path, e.g. ssh@host.com:/store/backups')
+end
+
+#
+# Validate & normalise source directories
+#
+@sources = ['/'] if @sources.nil?
+
+fatal('No sources specified') if @sources.empty?
+
+@sources = @sources.map do |s|
+  s = s.gsub(/\/+/, '/')
+  fatal("Can't read directory #{s}") unless File.readable?(s)
+  s
+end
+
+# Automatically exclude anything mounted on a non-local filesystem, plus
+# various cache and temporary directories common on Bytemark & Debian
+# systems
+#
+if @excludes.nil?
+
+  PROBABLY_LOCAL = %w(
+    btrfs
+    ext2
+    ext3
+    ext4
+    reiserfs
+    xfs
+    nilfs
+    jfs
+    reiser4
+    zfs
+    rootfs
+  )
+
+  COMMON_JUNK = %w(
+    /swap.file
+    /tmp
+    /var/backups/localhost
+    /var/cache/apt/archives
+    /var/lib/php5
+    /var/tmp
+  )
+
+  MOUNT_HEADINGS = %w( spec file vfstype mntops freq passno )
+                   .map(&:to_sym)
+
+  mounts = File.read('/proc/mounts').split("\n").map do |line|
+    Hash[MOUNT_HEADINGS.zip(line.split(' '))]
+  end
+
+  @excludes =
+
+    mounts
+    .select { |m| !PROBABLY_LOCAL.include?(m[:vfstype]) }
+    .map { |m| m[:file] } +
+
+    COMMON_JUNK.select { |f| File.exist?(f) }
+
+end
+
+@excludes = @excludes.map do |e|
+  e.gsub(/\/+/, '/')
+end
+
+fatal('Must suply --destination or put it into /etc/bytebackup/destination') unless @destination
+
+#
+# Test ssh connection is good before we start
+#
+fatal("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key)
+
+def ssh(*ssh_args)
+  args = ['ssh',
+          '-o', 'BatchMode=yes',
+          '-o', 'ConnectionAttempts=5',
+          '-o', 'ConnectTimeout=30',
+          '-o', 'ServerAliveInterval=60',
+          '-o', 'TCPKeepAlive=yes',
+          '-x', '-a',
+          '-i', @ssh_key,
+          '-l', @destination_user,
+          @destination_host
+         ] +
+         ssh_args
+         .map { |a| a ? a : '' }
+
+  log_system(*args)
+end
+
+fatal("Could not connect to #{@destination}") unless
+  ssh('byteback-receive', '--ping', @verbose) == 0
+
+#
+# Call rsync to copy certain sources, returns exit status (see man rsync)
+#
+def rsync(*sources)
+  # Default options include --inplace because we only care about consistency
+  # at the end of the job, and rsync will do more work for big files without
+  # it.
+  #
+  # The timeout is set to 12 hours - rsync can spend a long time at the
+  # far end checking over its files at the far end without transfer, so we
+  # want to wait as long as possible without jeopardising the timing of the
+  # next backup run.  Obviously if rsync itself takes nearly 24 hours for a
+  # given filesystem, daily backups (with this tool) are out of the question.
+  #
+  args = %w( rsync --archive --numeric-ids --delete-delay --inplace --relative --timeout 43200 )
+
+  args += ['--rsh', "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"]
+  args << '--verbose' if @verbose
+  args += @excludes.map { |x| ['--exclude', x] }.flatten
+
+  #
+  # Add in the rsync excludes and sources files, if present.
+  #
+  if File.exist?('/etc/byteback/excludes')
+    args += ['--exclude-from', '/etc/byteback/excludes']
+  end
+
+  #
+  # Add in an rsync_filter if required.
+  #
+  if File.exist?('/etc/byteback/rsync_filter')
+    args += ['--filter', 'merge /etc/byteback/rsync_filter']
+  end
+
+  #
+  # To add extra rsync flags, a file can be used.  This can have flags all on one line, or one per line.
+  #
+  if File.exist?('/etc/byteback/rsync_flags')
+    args += File.readlines('/etc/byteback/rsync_flags').map(&:chomp)
+  end
+
+  args += ['--rsync-path', 'rsync --fake-super']
+
+  args += sources
+  args << @destination
+
+  log_system(*args)
+end
+
+#
+# We treat exit statuses 0 and 24 as success; 0 is "Success"; 24 is "Partial
+# transfer due to vanished source files", which we treat as success otherwise
+# on some hosts the backup process never finishes.
+#
+RSYNC_EXIT_STATUSES_TO_ACCEPT   = [0, 24]
+RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10, 11, 20, 21, 22, 23, 30]
+
+# Run the file copy, retrying if necessary
+#
+loop do
+  status = rsync(*@sources)
+
+  if RSYNC_EXIT_STATUSES_TO_ACCEPT.any? { |s| s === status }
+    break
+  elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.any? { |s| s === status }
+
+    warn "rsync exited with status #{status}"
+
+    if @retry_number > 0
+      warn "rsync will retry #{@retry_number} more times, sleeping #{@retry_delay}s"
+      @retry_number -= 1
+      sleep @retry_delay
+      redo
+    else
+      fatal('Maximum number of rsync retries reached')
+    end
+  else
+    fatal("Fatal rsync error occurred (#{status})")
+  end
+end
+
+info('Backup completed, requesting snapshot')
+
+# Mark the backup as done on the other end
+#
+fatal('Backup could not be marked complete') unless
+  ssh('byteback-receive', '--complete', @verbose) == 0
+
+info('Finished')