From ae4967f4cf354a76af18e8d4af91f73af7d648d2 Mon Sep 17 00:00:00 2001 From: Matthew Bloch Date: Wed, 8 Jan 2014 13:19:01 +0000 Subject: Normalized line endings --- byteback-backup | 358 ++++++++++++++++++++--------------------- byteback-snapshot | 468 +++++++++++++++++++++++++++--------------------------- 2 files changed, 413 insertions(+), 413 deletions(-) diff --git a/byteback-backup b/byteback-backup index ffc4186..430975e 100755 --- a/byteback-backup +++ b/byteback-backup @@ -1,179 +1,179 @@ -#!/usr/bin/ruby -# -# Back up this system to a byteback-enabled server (just some command line -# tools and SSH setup). We aim to make sure this backups are easy, complete -# and safe for most types of hosting customer. -# -# See 'man byteback' for more information. - -require 'trollop' -require 'resolv' - -@sources = ["/"] -@exclude = ["/swap.file", "/var/backups/localhost"] - -def error(message) - STDERR.print "*** #{message}\n" - exit 1 -end - -def verbose(message) - print "#{message}\n" -end - -opts = Trollop::options do - - opt :destination, "Backup destination (i.e. user@host:/path)", - :type => :string - - opt :source, "Source paths", - :type => :strings - - opt :verbose, "Show rsync command and progress" - - opt :retry_number, "Number of retries on error", - :type => :integer, - :default => 3 - - opt :retry_delay, "Wait number of seconds between retries", - :type => :integer, - :default => 1800 - - opt :ssh_key, "SSH key for connection", - :type => :string, - :default => "/etc/byteback/key" -end - -@ssh_key = opts[:ssh_key] -@verbose = opts[:verbose] ? "--verbose" : nil -@sources = opts[:source] if opts[:source] -@destination = opts[:destination] -@retry_number = opts[:retry_number] -@retry_delay = opts[:retry_delay] - -if !@destination && File.exists?("/etc/byteback/destination") - @destination = File.read("/etc/byteback/destination").chomp -end -error("Must suply --destination or put it into /etc/bytebackup/destination") unless @destination - -_dummy, @destination_user, @destination_host, colon, @destination_path = - /^(.*)?(?:@)([^:]+)(:)(.*)?$/.match(@destination).to_a - -error("Must be a remote path") unless colon - -# Validate & normalise source directories -# -error("No sources specified") if @sources.empty? -@sources = @sources.map do |s| - s = s.gsub(/\/+/,"/") - error("Can't read directory #{s}") unless File.readable?(s) - s -end - -# Guess destination for backup -# -if !@destination - guesses = [] - hostname = `hostname -f` - Resolv::DNS.open do |dns| - suffix = hostname.split(".")[2..-1].join(".") - ["byteback." + suffix].each do |name| - [Resolv::DNS::Resource::IN::AAAA, - Resolv::DNS::Resource::IN::A].each do |record_type| - next if !guesses.empty? # only care about first result - guesses += dns.getresources(name, record_type) - end - end - end - - if guesses.empty? - error "Couldn't guess at backup host, please specify --destination" - end - - # ick, do I really have to do this to get a string represnetion of - # the IP address? - # - guess = guesses.first.inspect - match = / (.*)>$/.match(guess)[1] - error "Result #{guesses} is not an IP" if !match - @destination = "byteback@#{match[1]}:#{HOSTNAME}/current/" - - verbose "Guessed destination=#{@destination} from #{guess}" -end - -# Test ssh connection is good before we start -# -error("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) - -def ssh(*args) - ["ssh", - "-o", "BatchMode=yes", - "-x", "-a", - "-i", @ssh_key, - "-l", @destination_user - ] + - args -end - -error("Could not connect to #{@destination}") unless - system(*ssh("byteback-receive", "--ping", @verbose)) - -# Call rsync to copy certain sources, returns exit status (see man rsync) -# -def rsync(*sources) - # Default options include --inplace because we only care about consistency - # at the end of the job, and rsync will do more work for big files without - # it. - # - args = [ - "rsync", - "--archive", - "--numeric-ids", - "--delete", - "--inplace", - "--rsync-path", - "rsync --fake-super", - "--rsh", - ssh.join(" "), - "--delete", - "--one-file-system", - "--relative" - ] - - args << "--verbose" if @verbose - args += @exclude.map { |x| ["--exclude", x] }.flatten - args += sources - args << @destination - - print args.map { |a| / /.match(a) ? "\"#{a}\"" : a }.join(" ")+"\n" if @verbose - system(*args) - - return $?.exitstatus -end - -RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10,11,20,21,22,23,24,30] - -# Run the file copy, retrying if necessary -# -loop do - status = rsync(*@sources) - - if status === 0 - break - elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.include?(status) - if @retry_number > 0 - @retry_number -= 1 - sleep @retry_delay - redo - else - error("Maximum number of rsync retries reached") - end - else - error("Fatal rsync error occurred (#{status})") - end -end - -# Mark the backup as done on the other end -# -error("Backup could not be marked complete") unless - system(*ssh("sudo", "byteback-snapshot", "--snapshot", @verbose)) +#!/usr/bin/ruby +# +# Back up this system to a byteback-enabled server (just some command line +# tools and SSH setup). We aim to make sure this backups are easy, complete +# and safe for most types of hosting customer. +# +# See 'man byteback' for more information. + +require 'trollop' +require 'resolv' + +@sources = ["/"] +@exclude = ["/swap.file", "/var/backups/localhost"] + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +def verbose(message) + print "#{message}\n" +end + +opts = Trollop::options do + + opt :destination, "Backup destination (i.e. user@host:/path)", + :type => :string + + opt :source, "Source paths", + :type => :strings + + opt :verbose, "Show rsync command and progress" + + opt :retry_number, "Number of retries on error", + :type => :integer, + :default => 3 + + opt :retry_delay, "Wait number of seconds between retries", + :type => :integer, + :default => 1800 + + opt :ssh_key, "SSH key for connection", + :type => :string, + :default => "/etc/byteback/key" +end + +@ssh_key = opts[:ssh_key] +@verbose = opts[:verbose] ? "--verbose" : nil +@sources = opts[:source] if opts[:source] +@destination = opts[:destination] +@retry_number = opts[:retry_number] +@retry_delay = opts[:retry_delay] + +if !@destination && File.exists?("/etc/byteback/destination") + @destination = File.read("/etc/byteback/destination").chomp +end +error("Must suply --destination or put it into /etc/bytebackup/destination") unless @destination + +_dummy, @destination_user, @destination_host, colon, @destination_path = + /^(.*)?(?:@)([^:]+)(:)(.*)?$/.match(@destination).to_a + +error("Must be a remote path") unless colon + +# Validate & normalise source directories +# +error("No sources specified") if @sources.empty? +@sources = @sources.map do |s| + s = s.gsub(/\/+/,"/") + error("Can't read directory #{s}") unless File.readable?(s) + s +end + +# Guess destination for backup +# +if !@destination + guesses = [] + hostname = `hostname -f` + Resolv::DNS.open do |dns| + suffix = hostname.split(".")[2..-1].join(".") + ["byteback." + suffix].each do |name| + [Resolv::DNS::Resource::IN::AAAA, + Resolv::DNS::Resource::IN::A].each do |record_type| + next if !guesses.empty? # only care about first result + guesses += dns.getresources(name, record_type) + end + end + end + + if guesses.empty? + error "Couldn't guess at backup host, please specify --destination" + end + + # ick, do I really have to do this to get a string represnetion of + # the IP address? + # + guess = guesses.first.inspect + match = / (.*)>$/.match(guess)[1] + error "Result #{guesses} is not an IP" if !match + @destination = "byteback@#{match[1]}:#{HOSTNAME}/current/" + + verbose "Guessed destination=#{@destination} from #{guess}" +end + +# Test ssh connection is good before we start +# +error("Could not read ssh key #{@ssh_key}") unless File.readable?(@ssh_key) + +def ssh(*args) + ["ssh", + "-o", "BatchMode=yes", + "-x", "-a", + "-i", @ssh_key, + "-l", @destination_user + ] + + args +end + +error("Could not connect to #{@destination}") unless + system(*ssh("byteback-receive", "--ping", @verbose)) + +# Call rsync to copy certain sources, returns exit status (see man rsync) +# +def rsync(*sources) + # Default options include --inplace because we only care about consistency + # at the end of the job, and rsync will do more work for big files without + # it. + # + args = [ + "rsync", + "--archive", + "--numeric-ids", + "--delete", + "--inplace", + "--rsync-path", + "rsync --fake-super", + "--rsh", + ssh.join(" "), + "--delete", + "--one-file-system", + "--relative" + ] + + args << "--verbose" if @verbose + args += @exclude.map { |x| ["--exclude", x] }.flatten + args += sources + args << @destination + + print args.map { |a| / /.match(a) ? "\"#{a}\"" : a }.join(" ")+"\n" if @verbose + system(*args) + + return $?.exitstatus +end + +RSYNC_EXIT_STATUSES_TO_RETRY_ON = [10,11,20,21,22,23,24,30] + +# Run the file copy, retrying if necessary +# +loop do + status = rsync(*@sources) + + if status === 0 + break + elsif RSYNC_EXIT_STATUSES_TO_RETRY_ON.include?(status) + if @retry_number > 0 + @retry_number -= 1 + sleep @retry_delay + redo + else + error("Maximum number of rsync retries reached") + end + else + error("Fatal rsync error occurred (#{status})") + end +end + +# Mark the backup as done on the other end +# +error("Backup could not be marked complete") unless + system(*ssh("sudo", "byteback-snapshot", "--snapshot", @verbose)) diff --git a/byteback-snapshot b/byteback-snapshot index daf368b..4e1be92 100755 --- a/byteback-snapshot +++ b/byteback-snapshot @@ -1,234 +1,234 @@ -#!/usr/bin/ruby -# -# Program to create a snapshot and/or rotate a directory of backup snapshots -# using btrfs subvolume commands. - -require 'trollop' -require 'time' - -def error(message) - STDERR.print "*** #{message}\n" - exit 1 -end - -def verbose(message) - print "#{Time.now}: #{message}\n" if @verbose -end - -# Icky way to find out free disc space on our mount -# -class DiskFree - def initialize(mount) - @mount = mount - end - - def total - all[2] - end - - def used - all[3] - end - - def available - all[4] - end - - def fraction_used - disk_device, disk_fs, disk_total, disk_used, disk_available, *rest = all - disk_used.to_f / disk_available - end - - protected - - def all - disk_device, disk_fs, disk_total, disk_used, disk_available, *rest = - df. - split("\n")[1]. - split(/\s+/). - map { |i| /^[0-9]+$/.match(i) ? i.to_i : i } - end - - def df - `/bin/df -T -P -B1 #{@mount}` - end -end - -# Represent a directory full of backups where "current" is a subvolume -# which is snapshotted to frozen backup directories called e.g. -# "yyyy-mm-ddThh:mm". -# -class BackupDirectory - attr_reader :dir - - def initialize(dir) - @dir = Dir.new(dir) - current - end - - # Return total amount of free space in backup directory (bytes) - # - def free - df = DiskFree.new(@dir.path) - df.total - df.used - end - - # Return an array of Times representing the current list of - # snapshots. - # - def snapshot_times - @dir.entries.map do |entry| - begin - Time.parse(entry) - rescue ArgumentError => error - nil - end - end. - compact. - sort - end - - # What order to remove snapshots in to regain disk space? - # - # Order backups by their closeness to defined backup times, which are - # listed in a set order (i.e. today's backup is more important than yesterday's). - # - BACKUP_IMPORTANCE = [0, 1, 2, 3, 7, 14, 21, 28, 56, 112] - def snapshot_times_by_importance - now = Time.now - snapshot_times_unsorted = snapshot_times - snapshot_times_sorted = [] - while !snapshot_times_unsorted.empty? - BACKUP_IMPORTANCE.each do |days| - target_time = now + (days*86400) - closest = snapshot_times_unsorted.inject(nil) do |best, time| - if best.nil? || (time-target_time).abs < (best-target_time).abs - time - else - best - end - end - break unless closest - snapshot_times_sorted << snapshot_times_unsorted.delete(closest) - end - end - snapshot_times_sorted - end - - # Returns the size of the given snapshot (runs du, may be slow) - # - # Would much prefer to take advantage of this feature: - # http://dustymabe.com/2013/09/22/btrfs-how-big-are-my-snapshots/ - # but it's not currently in Debian/wheezy. - # - def snapshot_size(time=snapshot_times.latest) - `du -s -b #{snapshot_path(time)}`.to_i - end - - def average_snapshot_size(number=10) - snapshot_times.sort[0..number].inject(0) { |time, total| snapshot_size(time) } / number - end - - # Create a new snapshot of 'current' - # - def new_snapshot! - system_no_error("btrfs subvolume snapshot -r #{current.path} #{snapshot_path}") - end - - def delete_snapshot!(time) - system_no_error("btrfs subvolume delete #{snapshot_path(time)}") - end - - def current - Dir.new("#{dir.path}/current") - end - - def snapshot_path(time=Time.now) - "#{dir.path}/#{time.strftime("%Y-%m-%dT%H:%M")}" - end - - protected - - def system_no_error(*args) - raise RuntimeError.new("Command failed: "+args.join(" ")) unless - system(*args) - end -end - -opts = Trollop::options do - - opt :root, "Backups directory (must be a btrfs subvolume)", - :type => :string - - opt :snapshot, "Take a new snapshot" - - opt :prune, "Prune old backups" - :type => :string - - opt :list, "List backups (by 'age' or 'importance')", - :type => :string - - opt :verbose, "Print diagnostics" - -end - -@root = opts[:root] -@verbose = opts[:verbose] -@do_snapshot = opts[:snapshot] -@do_list = opts[:list] -@do_prune = opts[:prune] - -error("Must specify snapshot, prune or list") unless @do_snapshot || @do_prune || @do_list - -error("--root not readable") unless File.directory?(@root) - -@backups = BackupDirectory.new(@root) - -def get_snapshots_by(method) - if method == 'importance' - @backups.snapshot_times_by_importance.reverse # least important first - elsif method == 'age' - @backups.snapshot_times - else - raise ArgumentError.new("Unknown snapshot sort method #{method}") - end -end - -if @do_snapshot - last_snapshot_time = @backups.snapshots.last - error("Last snapshot was less than six hours ago") unless - !last_snapshot_time || - Time.now - @backups.snapshots.last >= 6*60*60 # FIXME: make configurable - - verbose "Making new snapshot" - @backups.new_snapshot! -end - -if @do_list - list = get_snapshots_by(@do_list) - print "Backups in #{@root} by #{@do_list}:\n" - list.each_with_index do |time, index| - print "#{sprintf('% 3d',index)}: #{time}\n" - end -end - -if @do_prune - verbose "Counting last 10 backups" - target_free_space = 1.5 * @backups.average_snapshot_size(10) - verbose "Want to ensure we have #{target_free_space}" - - if @backups.free >= target_free_space - verbose "(we have #{@backups.free} so no action needed)" - else - list = get_snapshots_by(@do_prune) - - while @backups.free < target_free_space && !list.empty? - to_delete = list.pop - verbose "Deleting #{to_delete}" - @backups.delete_snapshot!(to_delete) - verbose "Leaves us with #{@backups.free}" - end - end -end - -verbose "Finished" +#!/usr/bin/ruby +# +# Program to create a snapshot and/or rotate a directory of backup snapshots +# using btrfs subvolume commands. + +require 'trollop' +require 'time' + +def error(message) + STDERR.print "*** #{message}\n" + exit 1 +end + +def verbose(message) + print "#{Time.now}: #{message}\n" if @verbose +end + +# Icky way to find out free disc space on our mount +# +class DiskFree + def initialize(mount) + @mount = mount + end + + def total + all[2] + end + + def used + all[3] + end + + def available + all[4] + end + + def fraction_used + disk_device, disk_fs, disk_total, disk_used, disk_available, *rest = all + disk_used.to_f / disk_available + end + + protected + + def all + disk_device, disk_fs, disk_total, disk_used, disk_available, *rest = + df. + split("\n")[1]. + split(/\s+/). + map { |i| /^[0-9]+$/.match(i) ? i.to_i : i } + end + + def df + `/bin/df -T -P -B1 #{@mount}` + end +end + +# Represent a directory full of backups where "current" is a subvolume +# which is snapshotted to frozen backup directories called e.g. +# "yyyy-mm-ddThh:mm". +# +class BackupDirectory + attr_reader :dir + + def initialize(dir) + @dir = Dir.new(dir) + current + end + + # Return total amount of free space in backup directory (bytes) + # + def free + df = DiskFree.new(@dir.path) + df.total - df.used + end + + # Return an array of Times representing the current list of + # snapshots. + # + def snapshot_times + @dir.entries.map do |entry| + begin + Time.parse(entry) + rescue ArgumentError => error + nil + end + end. + compact. + sort + end + + # What order to remove snapshots in to regain disk space? + # + # Order backups by their closeness to defined backup times, which are + # listed in a set order (i.e. today's backup is more important than yesterday's). + # + BACKUP_IMPORTANCE = [0, 1, 2, 3, 7, 14, 21, 28, 56, 112] + def snapshot_times_by_importance + now = Time.now + snapshot_times_unsorted = snapshot_times + snapshot_times_sorted = [] + while !snapshot_times_unsorted.empty? + BACKUP_IMPORTANCE.each do |days| + target_time = now + (days*86400) + closest = snapshot_times_unsorted.inject(nil) do |best, time| + if best.nil? || (time-target_time).abs < (best-target_time).abs + time + else + best + end + end + break unless closest + snapshot_times_sorted << snapshot_times_unsorted.delete(closest) + end + end + snapshot_times_sorted + end + + # Returns the size of the given snapshot (runs du, may be slow) + # + # Would much prefer to take advantage of this feature: + # http://dustymabe.com/2013/09/22/btrfs-how-big-are-my-snapshots/ + # but it's not currently in Debian/wheezy. + # + def snapshot_size(time=snapshot_times.latest) + `du -s -b #{snapshot_path(time)}`.to_i + end + + def average_snapshot_size(number=10) + snapshot_times.sort[0..number].inject(0) { |time, total| snapshot_size(time) } / number + end + + # Create a new snapshot of 'current' + # + def new_snapshot! + system_no_error("btrfs subvolume snapshot -r #{current.path} #{snapshot_path}") + end + + def delete_snapshot!(time) + system_no_error("btrfs subvolume delete #{snapshot_path(time)}") + end + + def current + Dir.new("#{dir.path}/current") + end + + def snapshot_path(time=Time.now) + "#{dir.path}/#{time.strftime("%Y-%m-%dT%H:%M")}" + end + + protected + + def system_no_error(*args) + raise RuntimeError.new("Command failed: "+args.join(" ")) unless + system(*args) + end +end + +opts = Trollop::options do + + opt :root, "Backups directory (must be a btrfs subvolume)", + :type => :string + + opt :snapshot, "Take a new snapshot" + + opt :prune, "Prune old backups" + :type => :string + + opt :list, "List backups (by 'age' or 'importance')", + :type => :string + + opt :verbose, "Print diagnostics" + +end + +@root = opts[:root] +@verbose = opts[:verbose] +@do_snapshot = opts[:snapshot] +@do_list = opts[:list] +@do_prune = opts[:prune] + +error("Must specify snapshot, prune or list") unless @do_snapshot || @do_prune || @do_list + +error("--root not readable") unless File.directory?(@root) + +@backups = BackupDirectory.new(@root) + +def get_snapshots_by(method) + if method == 'importance' + @backups.snapshot_times_by_importance.reverse # least important first + elsif method == 'age' + @backups.snapshot_times + else + raise ArgumentError.new("Unknown snapshot sort method #{method}") + end +end + +if @do_snapshot + last_snapshot_time = @backups.snapshots.last + error("Last snapshot was less than six hours ago") unless + !last_snapshot_time || + Time.now - @backups.snapshots.last >= 6*60*60 # FIXME: make configurable + + verbose "Making new snapshot" + @backups.new_snapshot! +end + +if @do_list + list = get_snapshots_by(@do_list) + print "Backups in #{@root} by #{@do_list}:\n" + list.each_with_index do |time, index| + print "#{sprintf('% 3d',index)}: #{time}\n" + end +end + +if @do_prune + verbose "Counting last 10 backups" + target_free_space = 1.5 * @backups.average_snapshot_size(10) + verbose "Want to ensure we have #{target_free_space}" + + if @backups.free >= target_free_space + verbose "(we have #{@backups.free} so no action needed)" + else + list = get_snapshots_by(@do_prune) + + while @backups.free < target_free_space && !list.empty? + to_delete = list.pop + verbose "Deleting #{to_delete}" + @backups.delete_snapshot!(to_delete) + verbose "Leaves us with #{@backups.free}" + end + end +end + +verbose "Finished" -- cgit v1.2.1