diff options
Diffstat (limited to 'debian/byteback')
| -rw-r--r-- | debian/byteback/DEBIAN/control | 26 | ||||
| -rw-r--r-- | debian/byteback/DEBIAN/md5sums | 9 | ||||
| -rwxr-xr-x | debian/byteback/usr/sbin/byteback | 65 | ||||
| -rwxr-xr-x | debian/byteback/usr/sbin/byteback-backup | 216 | ||||
| -rwxr-xr-x | debian/byteback/usr/sbin/byteback-receive | 54 | ||||
| -rwxr-xr-x | debian/byteback/usr/sbin/byteback-setup-client | 54 | ||||
| -rwxr-xr-x | debian/byteback/usr/sbin/byteback-setup-client-receive | 42 | ||||
| -rwxr-xr-x | debian/byteback/usr/sbin/byteback-snapshot | 234 | ||||
| -rw-r--r-- | debian/byteback/usr/share/doc/byteback/README.md.gz | bin | 0 -> 3193 bytes | |||
| -rw-r--r-- | debian/byteback/usr/share/doc/byteback/changelog.Debian.gz | bin | 0 -> 203 bytes | |||
| -rw-r--r-- | debian/byteback/usr/share/doc/byteback/copyright | 29 |
11 files changed, 729 insertions, 0 deletions
diff --git a/debian/byteback/DEBIAN/control b/debian/byteback/DEBIAN/control new file mode 100644 index 0000000..ee20a2e --- /dev/null +++ b/debian/byteback/DEBIAN/control @@ -0,0 +1,26 @@ +Package: byteback +Version: 0.2.0-1 +Architecture: all +Maintainer: Patrick J Cherry <patrick@bytemark.co.uk> +Installed-Size: 48 +Depends: ruby | ruby-interpreter, rsync, openssh-client +Recommends: ruby-trollop | libtrollop-ruby +Section: ruby +Priority: optional +Homepage: https://projects.bytemark.co.uk/projects/byteback +Description: Maintenance-free client & server backup scripts for Linux + byteback encapsulates Bytemark's "best practice" for maintenance-free backups + with easy client and server setup. + . + "Maintenance-free" means that we'd rather make full use of a fixed amount of + disc space. Management of disc space must be completely automatic, so the + process never grinds to a halt for reasons that could be automatically + resolved. Failed backups can be restarted in case of network problems. + . + We use the standard OpenSSH on the server for encrypted transport & access + control, btrfs for simple snapshots and rsync for efficient data transfer + across the network. + . + Backups should require as little configuration as possible to be safe - just + the server address should be enough. +Ruby-Versions: ruby1.9.1 ruby2.0 diff --git a/debian/byteback/DEBIAN/md5sums b/debian/byteback/DEBIAN/md5sums new file mode 100644 index 0000000..e9641f9 --- /dev/null +++ b/debian/byteback/DEBIAN/md5sums @@ -0,0 +1,9 @@ +6f600d576b18b93e3bf43afd043f156e usr/sbin/byteback +21a8bb0fb9fecf44bbf58e9bbceda2b7 usr/sbin/byteback-backup +cee04f5b5f344d8ecb6ae0af488c5e0d usr/sbin/byteback-receive +7a7899e4bc3c0ba13280b8cd6eca827c usr/sbin/byteback-setup-client +cc78a3fd9b662e3db1e7fa801c4011dd usr/sbin/byteback-setup-client-receive +54a8cd3305209c36446b63707e68d070 usr/sbin/byteback-snapshot +538e288de3c0b5fdddcfb65840017482 usr/share/doc/byteback/README.md.gz +09620c298fa07a984827d0bac4dd9cc3 usr/share/doc/byteback/changelog.Debian.gz +4e261bda29e8364ad551d709a97162a0 usr/share/doc/byteback/copyright diff --git a/debian/byteback/usr/sbin/byteback b/debian/byteback/usr/sbin/byteback new file mode 100755 index 0000000..7418006 --- /dev/null +++ b/debian/byteback/usr/sbin/byteback @@ -0,0 +1,65 @@ +#!/usr/bin/ruby +# +# byteback backup script prototype +# +# (c) Bytemark Hosting 2013 +# +# +VERSION='prototype' + +HOSTNAME=`hostname -f` + +mode = ARGV.shift +case mode +when 'backup' + + @destination_host = HOSTNAME.split(".")[2..-1].join(".") + + system *<<-CMD.split(/\s+/) + +rsync + + --rsync-path + rsync --fake-super + --rsh + ssh -i /etc/bytebackup/bytebackup.key + --delete + --one-file-system + --archive + --exclude + /swap.file + + / + + #{@destination_ssh} + + CMD +when 'backup-receive' + +else + print <<-SYNTAX +byteback v#{VERSION}, a focused backup tool + +Usage: byteback <mode> + +Modes: + server-setup + client-setup + backup + backup-receive + +Type 'bytebackup help <mode>' for more information on a mode, or +see the man page. + SYNTAX + exit 1 +end + +require 'trollop' +opts = Trollop::options do + opt :mode, "Program mode to run", + :default => :backup, + :required, + :type => String + + opt +:end
\ No newline at end of file diff --git a/debian/byteback/usr/sbin/byteback-backup b/debian/byteback/usr/sbin/byteback-backup new file mode 100755 index 0000000..22fe0fb --- /dev/null +++ b/debian/byteback/usr/sbin/byteback-backup @@ -0,0 +1,216 @@ +#!/usr/bin/ruby +# +# Back up this system to a byteback-enabled server (just some command line +# tools and SSH setup). We aim to make sure this backups are easy, complete +# and safe for most types of hosting customer. +# +# See 'man byteback' for more information. + +require 'getoptlong' +require 'resolv' + + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +def verbose(message) + print "#{message}\n" +end + +def help + puts <<EOF +#{$0}: Back up this system to a byteback-enabled server + +Options: + --destination, -d <s>: Backup destination (i.e. user@host:/path) + --source, -s <s>: Source paths (defaults: / and /boot) + --exclude, -x <s>: Exclude paths (defaults: /swap.file, /var/backups/localhost, /var/cache) + --verbose, -v: Show rsync command and progress + --retry-number, -r <n>: Number of retries on error (default: 3) + --retry-delay, -e <n>: Wait number of seconds between retries (default: 1800) + --ssh-key, -k <s>: SSH key for connection (default: /etc/byteback/key) + --help, -h: Show this message +EOF + exit 0 +end + + +opts = GetoptLong.new( + [ '--help', '-h', GetoptLong::NO_ARGUMENT ], + [ '--verbose', '-v', GetoptLong::NO_ARGUMENT ], + [ '--source', '-s', GetoptLong::REQUIRED_ARGUMENT ], + [ '--destination', '-d', GetoptLong::REQUIRED_ARGUMENT ], + [ '--retry-number', '-r', GetoptLong::REQUIRED_ARGUMENT ], + [ '--retry-delay', '-e', GetoptLong::REQUIRED_ARGUMENT ], + [ '--ssh-key' ,'-k', GetoptLong::REQUIRED_ARGUMENT ] +) + +@ssh_key = nil +@destination = nil +@retry_number = 3 +@retry_delay = 1800 +@sources = nil +@excludes = nil + +# Read the default destination +if File.exists?("/etc/byteback/destination") + @destination = File.read("/etc/byteback/destination").chomp +end + +# Set the default SSH key +if File.exists?("/etc/byteback/key") + @ssh_key = "/etc/byteback/key" +end + +# Read in the default sources +if File.exists?("/etc/byteback/sources") + @sources = File.readlines("/etc/byteback/sources").map{|m| m.chomp} +end + +# Read in the default excludes +if File.exists?("/etc/byteback/excludes") + @excludes = File.readlines("/etc/byteback/excludes").map{|m| m.chomp} +end + +begin + opts.each do |opt,arg| + case opt + when '--help' + help = true + when '--verbose' + $VERBOSE = true + when "--source" + @sources ||= [] + @sources << arg + when "--exclude" + @excludes ||= [] + @excludes << arg + when "--destination" + @destination = arg + when "--retry-number" + @retry_number = arg.to_i + when "--retry-delay" + @retry_delay = arg.to_i + when "--ssh-key" + @ssh_key = arg + end + end +rescue => err + # any errors, show the help + warn err.to_s + help = true +end + + +# +# Check our destination +# +if @destination =~ /^(?:(.+)@)?([^@:]+):(.+)?$/ + @destination_user, @destination_host, @destination_path = [$1, $2, $3] +else + error("Destination must be a remote path, e.g. ssh@host.com:/store/backups") +end + +# +# Validate & normalise source directories +# +@sources = ["/"] if @sources.nil? + +error("No sources specified") if @sources.empty? + +@sources = @sources.map do |s| + s = s.gsub(/\/+/,"/") + error("Can't read directory #{s}") unless File.readable?(s) + s +end + +# +# Validate and normalise excludes +# +if @excludes.nil? + @excludes = ["/swap.file", "/var/backups/localhost"] + @excludes << "/var/cache/apt/archives" if File.directory?("/var/cache/apt/archives") +end + +@excludes = @excludes.map do |e| + e.gsub(/\/+/,"/") +end + +error("Must suply --destination or put it into /etc/bytebackup/destination") unless @destination + +# +# Test ssh connection is good before we start +# +error("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) + +def ssh(*ssh_args) + args = ["ssh", + "-o", "BatchMode=yes", + "-x", "-a", + "-i", @ssh_key, + "-l", @destination_user, + @destination_host + ] + + ssh_args. + map { |a| a ? a : "" } + + print args.map { |a| / /.match(a) ? "\"#{a}\"" : a }.join(" ")+"\n" if $VERBOSE + + system(*args) +end + +error("Could not connect to #{@destination}") unless + ssh("byteback-receive", "--ping", ($VERBOSE ? "--verbose" : "" )) + +# +# Call rsync to copy certain sources, returns exit status (see man rsync) +# +def rsync(*sources) + # Default options include --inplace because we only care about consistency + # at the end of the job, and rsync will do more work for big files without + # it. + # + args = %w(rsync --archive --numeric-ids --delete --inplace --delete --one-file-system --relative) + args += [ "--rsync-path", "rsync --fake-super"] + args += [ "--rsh", "ssh -o BatchMode=yes -x -a -i #{@ssh_key} -l #{@destination_user}"] + args << "--verbose" if $VERBOSE + args += @excludes.map { |x| ["--exclude", x] }.flatten + args += sources + args << @destination + + print args.map { |a| / /.match(a) ? "\"#{a}\"" : a }.join(" ")+"\n" if $VERBOSE + + system(*args) + + return $?.exitstatus +end + +RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10,11,20,21,22,23,24,30] + +# Run the file copy, retrying if necessary +# +loop do + status = rsync(*@sources) + + if status === 0 + break + elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.include?(status) + if @retry_number > 0 + @retry_number -= 1 + sleep @retry_delay + redo + else + error("Maximum number of rsync retries reached") + end + else + error("Fatal rsync error occurred (#{status})") + end +end + +# Mark the backup as done on the other end +# +error("Backup could not be marked complete") unless + ssh("sudo", "byteback-snapshot", "--snapshot", ($VERBOSE ? "--verbose" : "")) + diff --git a/debian/byteback/usr/sbin/byteback-receive b/debian/byteback/usr/sbin/byteback-receive new file mode 100755 index 0000000..05e6a7c --- /dev/null +++ b/debian/byteback/usr/sbin/byteback-receive @@ -0,0 +1,54 @@ +#!/usr/bin/ruby +# +# Program to receive backups and run rsync in receive mode. Must check that +# user as authorised by SSH is allowed to access particular directory. + +#STDERR.print ARGV.inspect + "\n" + +require 'trollop' + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +#STDERR.print "ARGV=#{ARGV.inspect}\nSSH_ORIGINAL_COMMAND=#{ENV['SSH_ORIGINAL_COMMAND']}\n" + +if ENV['SSH_ORIGINAL_COMMAND'] + ARGV.concat(ENV['SSH_ORIGINAL_COMMAND'].split(" ")) +end + +#STDERR.print "after ARGV=#{ARGV.inspect}\n" + +byteback_host = ENV['BYTEBACK_HOST'] +error("BYTEBACK_HOST environment not set") unless byteback_host + +byteback_root = ENV['HOME'] + "/" + ENV["BYTEBACK_HOST"] +error("#{byteback_root} does not exist") unless File.directory?(byteback_root) + +# force destination to be where we expect +# +if ARGV[0] == 'rsync' + ARGV[-1] = "#{byteback_root}/current" + exec(*ARGV) +elsif ARGV[0] == 'byteback-snapshot' || (ARGV[0] == 'sudo' && ARGV[1] == 'byteback-snapshot') + ARGV.concat(["--root", "#{byteback_root}"]) + exec(*ARGV) +end + +opts = Trollop::options do + opt :verbose, "Print diagnostics" + opt :ping, "Check connection parameters and exit" + opt :complete, "Mark current backup as complete" +end + +error("Please only choose one mode") if opts[:ping] && opts[:complete] +if opts[:complete] + system("byteback-snapshot", byteback_root) +elsif opts[:ping] + exit 0 +else + STDERR.print "byteback-receive failed\n" + exit 9 +end + diff --git a/debian/byteback/usr/sbin/byteback-setup-client b/debian/byteback/usr/sbin/byteback-setup-client new file mode 100755 index 0000000..ddb6672 --- /dev/null +++ b/debian/byteback/usr/sbin/byteback-setup-client @@ -0,0 +1,54 @@ +#!/usr/bin/ruby +# +# Run on a client machine to set up backups for the first time + +require 'fileutils' +require 'trollop' + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +def verbose(message) + print "#{message}\n" +end + +opts = Trollop::options do + + opt :hostname, "Set host name for backups", + :type => :string + + opt :destination, "Backup destination (i.e. user@host:/path)", + :type => :string + +end + +@destination = opts[:destination] +@hostname = opts[:hostname] + +_dummy, @destination_user, @destination_host, colon, @destination_path = + /^(.*)?(?:@)([^:]+)(:)(.*)?$/.match(@destination).to_a + +error("Must be a remote path") unless colon +if !@hostname + @hostname = `hostname -f` + print "No hostname set, using #{@hostname}" +end + +error "This host already appears set up - you need to delete /etc/byteback if not" if + File.readable?("/etc/byteback/key") + +FileUtils.mkdir_p("/etc/byteback") + +error "Couldn't generate SSH key" unless + system("ssh-keygen -q -t rsa -C \"byteback client key\" -N \"\" -f /etc/byteback/key") + +key_pub = File.read("/etc/byteback/key.pub") + +error "Setup didn't work" unless + system("ssh -i /etc/byteback/key -l #{@destination_user} #{@destination_host} byteback-setup-client-receive #{@hostname} #{key_pub}") + +File.open("/etc/byteback/destination", "w") { |f| f.print @destination } + +print "Setup worked! To take your first backup run: byteback-backup --verbose\n" diff --git a/debian/byteback/usr/sbin/byteback-setup-client-receive b/debian/byteback/usr/sbin/byteback-setup-client-receive new file mode 100755 index 0000000..35a3b65 --- /dev/null +++ b/debian/byteback/usr/sbin/byteback-setup-client-receive @@ -0,0 +1,42 @@ +#!/usr/bin/ruby +# +# Called by byteback-setup-client to set up a new byteback-setup-client + +require 'fileutils' + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +@hostname = ARGV.shift +@pubkey = ARGV.join(" ") + +error("You must call this from byteback-setup-client on remote host") unless + @hostname && + /^ssh/.match(@pubkey) && + ENV['SSH_CONNECTION'] + +@client_ip = ENV['SSH_CONNECTION'].split(" ").first + +Dir.chdir(ENV['HOME']) # don't know why we wouldn't be here + +Dir.mkdir(@hostname) + +error("Couldn't create btrfs subvolume (needs sudo)") unless + system("sudo btrfs subvolume create #{@hostname}/current") + +FileUtils.mkdir_p(".ssh") + +error("This key already exists in .ssh/authorized_keys on server") if + File.exists?(".ssh/authorized_keys") && + File.read(".ssh/authorized_keys").match(@pubkey.split(/\s+/)[1]) + +File.open(".ssh/authorized_keys", "a+") do |fh| + fh.print <<-LINE.gsub(/\n/,"") +command="byteback-receive", +from="#{@client_ip}", +environment="BYTEBACK_HOST=#{@hostname}" + #{@pubkey} + LINE +end diff --git a/debian/byteback/usr/sbin/byteback-snapshot b/debian/byteback/usr/sbin/byteback-snapshot new file mode 100755 index 0000000..0e5a362 --- /dev/null +++ b/debian/byteback/usr/sbin/byteback-snapshot @@ -0,0 +1,234 @@ +#!/usr/bin/ruby +# +# Program to create a snapshot and/or rotate a directory of backup snapshots +# using btrfs subvolume commands. + +require 'trollop' +require 'time' + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +def verbose(message) + print "#{Time.now}: #{message}\n" if @verbose +end + +# Icky way to find out free disc space on our mount +# +class DiskFree + def initialize(mount) + @mount = mount + end + + def total + all[2] + end + + def used + all[3] + end + + def available + all[4] + end + + def fraction_used + disk_device, disk_fs, disk_total, disk_used, disk_available, *rest = all + disk_used.to_f / disk_available + end + + protected + + def all + disk_device, disk_fs, disk_total, disk_used, disk_available, *rest = + df. + split("\n")[1]. + split(/\s+/). + map { |i| /^[0-9]+$/.match(i) ? i.to_i : i } + end + + def df + `/bin/df -T -P -B1 #{@mount}` + end +end + +# Represent a directory full of backups where "current" is a subvolume +# which is snapshotted to frozen backup directories called e.g. +# "yyyy-mm-ddThh:mm+zzzz". +# +class BackupDirectory + attr_reader :dir + + def initialize(dir) + @dir = Dir.new(dir) + current + end + + # Return total amount of free space in backup directory (bytes) + # + def free + df = DiskFree.new(@dir.path) + df.total - df.used + end + + # Return an array of Times representing the current list of + # snapshots. + # + def snapshot_times + @dir.entries.map do |entry| + begin + Time.parse(entry) + rescue ArgumentError => error + nil + end + end. + compact. + sort + end + + # What order to remove snapshots in to regain disk space? + # + # Order backups by their closeness to defined backup times, which are + # listed in a set order (i.e. today's backup is more important than yesterday's). + # + BACKUP_IMPORTANCE = [0, 1, 2, 3, 7, 14, 21, 28, 56, 112] + def snapshot_times_by_importance + now = Time.now + snapshot_times_unsorted = snapshot_times + snapshot_times_sorted = [] + while !snapshot_times_unsorted.empty? + BACKUP_IMPORTANCE.each do |days| + target_time = now + (days*86400) + closest = snapshot_times_unsorted.inject(nil) do |best, time| + if best.nil? || (time-target_time).abs < (best-target_time).abs + time + else + best + end + end + break unless closest + snapshot_times_sorted << snapshot_times_unsorted.delete(closest) + end + end + snapshot_times_sorted + end + + # Returns the size of the given snapshot (runs du, may be slow) + # + # Would much prefer to take advantage of this feature: + # http://dustymabe.com/2013/09/22/btrfs-how-big-are-my-snapshots/ + # but it's not currently in Debian/wheezy. + # + def snapshot_size(time=snapshot_times.latest) + `du -s -b #{snapshot_path(time)}`.to_i + end + + def average_snapshot_size(number=10) + snapshot_times.sort[0..number].inject(0) { |time, total| snapshot_size(time) } / number + end + + # Create a new snapshot of 'current' + # + def new_snapshot! + system_no_error("btrfs subvolume snapshot -r #{current.path} #{snapshot_path}") + end + + def delete_snapshot!(time) + system_no_error("btrfs subvolume delete #{snapshot_path(time)}") + end + + def current + Dir.new("#{dir.path}/current") + end + + def snapshot_path(time=Time.now) + "#{dir.path}/#{time.strftime("%Y-%m-%dT%H:%M%z")}" + end + + protected + + def system_no_error(*args) + raise RuntimeError.new("Command failed: "+args.join(" ")) unless + system(*args) + end +end + +opts = Trollop::options do + + opt :root, "Backups directory (must be a btrfs subvolume)", + :type => :string + + opt :snapshot, "Take a new snapshot" + + opt :prune, "Prune old backups", + :type => :string + + opt :list, "List backups (by 'age' or 'importance')", + :type => :string + + opt :verbose, "Print diagnostics" + +end + +@root = opts[:root] +@verbose = opts[:verbose] +@do_snapshot = opts[:snapshot] +@do_list = opts[:list] +@do_prune = opts[:prune] + +error("Must specify snapshot, prune or list") unless @do_snapshot || @do_prune || @do_list + +error("--root not readable") unless File.directory?(@root) + +@backups = BackupDirectory.new(@root) + +def get_snapshots_by(method) + if method == 'importance' + @backups.snapshot_times_by_importance.reverse # least important first + elsif method == 'age' + @backups.snapshot_times + else + raise ArgumentError.new("Unknown snapshot sort method #{method}") + end +end + +if @do_snapshot + last_snapshot_time = @backups.snapshot_times.last + error("Last snapshot was less than six hours ago") unless + !last_snapshot_time || + Time.now - @backups.snapshot_times.last >= 6*60*60 # FIXME: make configurable + + verbose "Making new snapshot" + @backups.new_snapshot! +end + +if @do_list + list = get_snapshots_by(@do_list) + print "Backups in #{@root} by #{@do_list}:\n" + list.each_with_index do |time, index| + print "#{sprintf('% 3d',index)}: #{time}\n" + end +end + +if @do_prune + verbose "Counting last 10 backups" + target_free_space = 1.5 * @backups.average_snapshot_size(10) + verbose "Want to ensure we have #{target_free_space}" + + if @backups.free >= target_free_space + verbose "(we have #{@backups.free} so no action needed)" + else + list = get_snapshots_by(@do_prune) + + while @backups.free < target_free_space && !list.empty? + to_delete = list.pop + verbose "Deleting #{to_delete}" + @backups.delete_snapshot!(to_delete) + verbose "Leaves us with #{@backups.free}" + end + end +end + +verbose "Finished" diff --git a/debian/byteback/usr/share/doc/byteback/README.md.gz b/debian/byteback/usr/share/doc/byteback/README.md.gz Binary files differnew file mode 100644 index 0000000..6d5e92a --- /dev/null +++ b/debian/byteback/usr/share/doc/byteback/README.md.gz diff --git a/debian/byteback/usr/share/doc/byteback/changelog.Debian.gz b/debian/byteback/usr/share/doc/byteback/changelog.Debian.gz Binary files differnew file mode 100644 index 0000000..0fa0c0d --- /dev/null +++ b/debian/byteback/usr/share/doc/byteback/changelog.Debian.gz diff --git a/debian/byteback/usr/share/doc/byteback/copyright b/debian/byteback/usr/share/doc/byteback/copyright new file mode 100644 index 0000000..daf38ac --- /dev/null +++ b/debian/byteback/usr/share/doc/byteback/copyright @@ -0,0 +1,29 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: byteback +Source: <https://projects.bytemark.co.uk/projects/byteback> + +Files: * +Copyright: 2013-2014 Bytemark Computer Consulting Ltd +License: GPL-2+ + +License: GPL-2+ + This program is free software; you can redistribute it + and/or modify it under the terms of the GNU General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later + version. + . + This program is distributed in the hope that it will be + useful, but WITHOUT ANY WARRANTY; without even the implied + warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the GNU General Public License for more + details. + . + You should have received a copy of the GNU General Public + License along with this package; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, + Boston, MA 02110-1301 USA + . + On Debian systems, the full text of the GNU General Public + License version 2 can be found in the file + `/usr/share/common-licenses/GPL-2'. |