From 668b9871e64cb82ac30c8defb29d56d774f3c140 Mon Sep 17 00:00:00 2001 From: Patrick J Cherry Date: Tue, 1 Dec 2015 22:41:06 +0000 Subject: Completely re-vamped restore command. Fixes #12403 The byteback-restore command now uses the rsync xattrs to display information about the files due to be restored. It can handle filenames with spaces (!) and other characters. --- lib/byteback/restore.rb | 122 ++++++++++++++++++++ lib/byteback/restore_file.rb | 215 +++++++++++++++++++++++++++++++++++ lib/ffi-xattr.rb | 78 +++++++++++++ lib/ffi-xattr/darwin_lib.rb | 47 ++++++++ lib/ffi-xattr/error.rb | 23 ++++ lib/ffi-xattr/extensions.rb | 3 + lib/ffi-xattr/extensions/file.rb | 9 ++ lib/ffi-xattr/extensions/pathname.rb | 9 ++ lib/ffi-xattr/linux_lib.rb | 52 +++++++++ lib/ffi-xattr/version.rb | 3 + lib/ffi-xattr/windows_lib.rb | 45 ++++++++ 11 files changed, 606 insertions(+) create mode 100644 lib/byteback/restore.rb create mode 100644 lib/byteback/restore_file.rb create mode 100644 lib/ffi-xattr.rb create mode 100644 lib/ffi-xattr/darwin_lib.rb create mode 100644 lib/ffi-xattr/error.rb create mode 100644 lib/ffi-xattr/extensions.rb create mode 100644 lib/ffi-xattr/extensions/file.rb create mode 100644 lib/ffi-xattr/extensions/pathname.rb create mode 100644 lib/ffi-xattr/linux_lib.rb create mode 100644 lib/ffi-xattr/version.rb create mode 100644 lib/ffi-xattr/windows_lib.rb (limited to 'lib') diff --git a/lib/byteback/restore.rb b/lib/byteback/restore.rb new file mode 100644 index 0000000..e2df3fe --- /dev/null +++ b/lib/byteback/restore.rb @@ -0,0 +1,122 @@ + +require 'byteback/restore_file' + +module Byteback + + class Restore + + def self.find(byteback_root, revision, paths) + x = Byteback::Restore.new(byteback_root) + x.revision = revision + x.find(paths) + return x + end + + # + # This takes a string or array of strings as an argument, and QP encodes + # each argument. This is for safe parsing of spaces etc at the remote end. + # + # Returns an array of encoded strings. + # + def self.encode_args(args) + [args].flatten.collect{|s| [s].pack("M").gsub(" ","=20").gsub("=\n","")} + end + + # + # This takes a string or array of strings, each of which is quoted + # printable, and unpacks it. + # + # Returns an array of decoded strings. + # + def self.decode_args(args) + [args].flatten.collect{|s| (s + "=\n").unpack("M")}.flatten + end + + def initialize(byteback_root) + # + # We use expand_path here to make sure we have a full path, with no + # trailing slash. + # + @byteback_root = File.expand_path(byteback_root) + @now = Time.now + @revision = "*" + @results = [] + end + + def revision=(r) + if r =~ /^[a-z0-9:\+\*\-]+$/i + @revision = r + else + puts "*** Warning: Bad revision #{r.inspect}" + end + end + + def results + @results + end + + def find(paths, full = false) + results = [] + # + # Make sure we've an array, and that we get rid of any ".." nonsense. + # + paths = [paths].flatten.collect{|p| File.expand_path(p, "/")} + seen = [] + + @results = paths.collect do |path| + Dir.glob(File.expand_path(File.join(@byteback_root, @revision, path))).collect do |f| + restore_file = Byteback::RestoreFile.new(f, @byteback_root, @now) + end + end.flatten.sort{|a,b| [a.path, a.snapshot_time] <=> [b.path, b.snapshot_time]} + + # + # If we want an unpruned list, return it now. + # + return @results if full + + pruned_results = [] + + @results.each do |r| + pruned_results << r unless pruned_results.include?(r) + end + + @results = pruned_results + end + + def list + heading = %w(snapshot modestring size uid gid mtime path) + listings = [heading] + @results.sort.each do |r| + listing = heading.collect{|m| r.__send__(m.to_sym).to_s } + if r.symlink? + listing[-1] << " -> "+r.readlink + end + listings << listing + end + + field_sizes = [0]*heading.length + + listings.each do |fields| + fields.each_with_index do |field, i| + field_sizes[i] = (field_sizes[i] > field.length) ? field_sizes[i] : field.length + end + end + + fmt = field_sizes.collect{|i| "%-#{i}.#{i}s"}.join(" ") + + bar = "-"*field_sizes.inject(field_sizes.length){|m,s| m+=s} + + output = [] + listings.each do |fields| + output << sprintf(fmt, *fields) + if bar + output << bar + bar = nil + end + end + + return output.join("\n") + end + + end +end diff --git a/lib/byteback/restore_file.rb b/lib/byteback/restore_file.rb new file mode 100644 index 0000000..e9282d0 --- /dev/null +++ b/lib/byteback/restore_file.rb @@ -0,0 +1,215 @@ +require 'ffi-xattr' +require 'scanf' +require 'pp' + +module Byteback + class RestoreFile + S_IFMT = 0170000 # bit mask for the file type bit fields + + S_IFSOCK = 0140000 # socket + S_IFLNK = 0120000 # symbolic link + S_IFREG = 0100000 # regular file + S_IFBLK = 0060000 # block device + S_IFDIR = 0040000 # directory + S_IFCHR = 0020000 # character device + S_IFIFO = 0010000 # FIFO + + S_ISUID = 0004000 # set-user-ID bit + S_ISGID = 0002000 # set-group-ID bit (see below) + S_ISVTX = 0001000 # sticky bit (see below) + + S_IRWXU = 00700 # mask for file owner permissions + S_IRUSR = 00400 # owner has read permission + S_IWUSR = 00200 # owner has write permission + S_IXUSR = 00100 # owner has execute permission + + S_IRWXG = 00070 # mask for group permissions + S_IRGRP = 00040 # group has read permission + S_IWGRP = 00020 # group has write permission + S_IXGRP = 00010 # group has execute permission + + S_IRWXO = 00007 # mask for permissions for others (not in group) + S_IROTH = 00004 # others have read permission + S_IWOTH = 00002 # others have write permission + S_IXOTH = 00001 # others have execute permission + + include Comparable + + def initialize(full_path, byteback_root=".", now = Time.now) + @full_path = full_path + @byteback_root = byteback_root + @now = now + + # + # The snapshot is the first directory after the byteback_root + # + @snapshot = full_path.sub(%r(^#{Regexp.escape @byteback_root}),'').split("/")[1] + + if @snapshot == "current" + @snapshot_time = @now + else + @snapshot_time = Time.parse(@snapshot) + end + + # + # Restore path + # + @path = full_path.sub(%r(^#{Regexp.escape @byteback_root}/#{Regexp.escape @snapshot}),'') + + @stat = @mode = @dev_major = @dev_minor = @uid = @gid = nil + end + + def <=>(other) + [self.path, self.mtime.to_i, self.size] <=> [other.path, other.mtime.to_i, other.size] + end + + def stat + @stat = ::File.lstat(@full_path) unless @stat.is_a?(File::Stat) + @stat + end + + def snapshot + @snapshot + end + + def snapshot_time + @snapshot_time + end + + def path + @path + end + + def to_s + sprintf("%10s %i %4i %4i %s %s %s", self.modestring, self.size, self.uid, self.gid, self.mtime.strftime("%b %2d %H:%M"), @snapshot, @path) + end + + def read_rsync_xattrs + xattr = Xattr.new(@full_path, :no_follow => false) + rsync_xattrs = xattr["user.rsync.%stat"] + if rsync_xattrs + @mode, @dev_major, @dev_minor, @uid, @gid = rsync_xattrs.scanf("%o %d,%d %d:%d") + raise ArgumentError, "Corrupt rsync stat xattr found for #{@full_path} (#{rsync_xattrs})" unless [@mode, @dev_major, @dev_minor, @uid, @gid].all?{|i| i.is_a?(Integer)} + else + warn "No rsync stat xattr found for #{@full_path}" + @mode, @dev_major, @dev_minor, @uid, @gid = %w(mode dev_major dev_minor uid gid).collect{|m| self.stat.__send__(m.to_sym)} + end + end + + def mode + return self.stat.mode if self.stat.symlink? + read_rsync_xattrs unless @mode + @mode + end + + def dev_minor + read_rsync_xattrs unless @dev_minor + @dev_minor + end + + def dev_major + read_rsync_xattrs unless @dev_major + @dev_major + end + + def uid + read_rsync_xattrs unless @uid + @uid + end + + def gid + read_rsync_xattrs unless @gid + @gid + end + + # + # This returns the type of file as a single character. + # + def ftypelet + if file? + "-" + elsif directory? + "d" + elsif blockdev? + "b" + elsif chardev? + "c" + elsif symlink? + "l" + elsif fifo? + "p" + elsif socket? + "s" + else + "?" + end + end + + # + # This returns a modestring from the octal, like drwxr-xr-x. + # This has mostly been copied from strmode from filemode.h in coreutils. + # + def modestring + str = "" + str << ftypelet + str << ((mode & S_IRUSR == S_IRUSR) ? 'r' : '-') + str << ((mode & S_IWUSR == S_IWUSR) ? 'w' : '-') + str << ((mode & S_ISUID == S_ISUID) ? + ((mode & S_IXUSR == S_IXUSR) ? 's' : 'S') : + ((mode & S_IXUSR == S_IXUSR) ? 'x' : '-')) + str << ((mode & S_IRGRP == S_IRGRP) ? 'r' : '-') + str << ((mode & S_IWGRP == S_IWGRP) ? 'w' : '-') + str << ((mode & S_ISGID == S_ISGID) ? + ((mode & S_IXGRP == S_IXGRP) ? 's' : 'S') : + ((mode & S_IXGRP == S_IXGRP) ? 'x' : '-')) + str << ((mode & S_IROTH == S_IROTH) ? 'r' : '-') + str << ((mode & S_IWOTH == S_IWOTH) ? 'w' : '-') + str << ((mode & S_ISVTX == S_ISVTX) ? + ((mode & S_IXOTH == S_IXOTH) ? 't' : 'T') : + ((mode & S_IXOTH == S_IXOTH) ? 'x' : '-')) + return str + end + + def socket? + (mode & S_IFMT) == S_IFSOCK + end + + def symlink? + self.stat.symlink? || (mode & S_IFMT) == S_IFLNK + end + + def file? + (mode & S_IFMT) == S_IFREG + end + + def blockdev? + (mode & S_IFMT) == S_IFBLK + end + + def directory? + (mode & S_IFMT) == S_IFDIR + end + + def chardev? + (mode & S_IFMT) == S_IFCHR + end + + def fifo? + (mode & S_IFMT) == S_IFIFO + end + + def readlink + if self.stat.symlink? + File.readlink(@full_path) + else + File.read(@full_path).chomp + end + end + + def method_missing(m, *args, &blk) + return self.stat.__send__(m) if self.stat.respond_to?(m) + + raise NoMethodError, m + end + end +end diff --git a/lib/ffi-xattr.rb b/lib/ffi-xattr.rb new file mode 100644 index 0000000..2dd1e23 --- /dev/null +++ b/lib/ffi-xattr.rb @@ -0,0 +1,78 @@ +require 'ffi' +require 'ffi-xattr/version' +require 'ffi-xattr/error' + +case RUBY_PLATFORM +when /linux/ + require 'ffi-xattr/linux_lib' +when /darwin|bsd/ + require 'ffi-xattr/darwin_lib' +when /mingw/ + require 'ffi-xattr/windows_lib' +else + raise NotImplementedError, "ffi-xattr not supported on #{RUBY_PLATFORM}" +end + +class Xattr + include Enumerable + + # Create a new Xattr instance with path. + # Use :no_follow => true in options to work on symlink itself instead of following it. + def initialize(path, options = {}) + @path = + if path.respond_to?(:to_path) + path.to_path + elsif path.respond_to?(:to_str) + path.to_str + else + path + end + raise Errno::ENOENT, @path unless File.exist?(@path) + + @no_follow = !!options[:no_follow] + end + + # List extended attribute names + def list + Lib.list @path, @no_follow + end + + # Get an extended attribute value + def get(key) + Lib.get @path, @no_follow, key.to_s + end + alias_method :[], :get + + # Set an extended attribute value + def set(key, value) + Lib.set @path, @no_follow, key.to_s, value.to_s + end + alias_method :[]=, :set + + # Remove an extended attribute value + def remove(key) + Lib.remove @path, @no_follow, key.to_s + end + + # Iterates over pairs of extended attribute names and values + def each(&blk) + list.each do |key| + yield key, get(key) + end + end + + # Returns hash of extended attributes + + def to_hash + res = {} + each { |k,v| res[k] = v } + + res + end + + alias_method :to_h, :to_hash + + def as_json(*args) + to_hash + end +end diff --git a/lib/ffi-xattr/darwin_lib.rb b/lib/ffi-xattr/darwin_lib.rb new file mode 100644 index 0000000..59f0810 --- /dev/null +++ b/lib/ffi-xattr/darwin_lib.rb @@ -0,0 +1,47 @@ +class Xattr # :nodoc: all + module Lib + extend FFI::Library + + ffi_lib "System" + + attach_function :listxattr, [:string, :pointer, :size_t, :int], :ssize_t + attach_function :getxattr, [:string, :string, :pointer, :size_t, :uint, :int], :ssize_t + attach_function :setxattr, [:string, :string, :pointer, :size_t, :uint, :int], :int + attach_function :removexattr, [:string, :string, :int], :int + + XATTR_NOFOLLOW = 0x0001 + + class << self + def list(path, no_follow) + options = no_follow ? XATTR_NOFOLLOW : 0 + size = listxattr(path, nil, 0, options) + res_ptr = FFI::MemoryPointer.new(:pointer, size) + listxattr(path, res_ptr, size, options) + + res_ptr.read_string(size).split("\000") + end + + def get(path, no_follow, key) + options = no_follow ? XATTR_NOFOLLOW : 0 + size = getxattr(path, key, nil, 0, 0, options) + return unless size > 0 + + str_ptr = FFI::MemoryPointer.new(:char, size) + getxattr(path, key, str_ptr, size, 0, options) + + str_ptr.read_string(size) + end + + def set(path, no_follow, key, value) + options = no_follow ? XATTR_NOFOLLOW : 0 + Error.check setxattr(path, key, value, value.bytesize, 0, options) + end + + def remove(path, no_follow, key) + options = no_follow ? XATTR_NOFOLLOW : 0 + Error.check removexattr(path, key, options) + end + end + + end +end diff --git a/lib/ffi-xattr/error.rb b/lib/ffi-xattr/error.rb new file mode 100644 index 0000000..602053d --- /dev/null +++ b/lib/ffi-xattr/error.rb @@ -0,0 +1,23 @@ +class Xattr # :nodoc: all + module Error + extend FFI::Library + + ffi_lib "c" + + attach_function :strerror_r, [:int, :pointer, :size_t], :int unless RUBY_PLATFORM =~ /mingw/ + + class << self + def last + errno = FFI.errno + ptr = FFI::MemoryPointer.new(:char, 256) + strerror_r(errno, ptr, 256) + + [ ptr.read_string, errno ] + end + + def check(int) + raise SystemCallError.new(*last) if int != 0 + end + end + end +end diff --git a/lib/ffi-xattr/extensions.rb b/lib/ffi-xattr/extensions.rb new file mode 100644 index 0000000..4ecad8a --- /dev/null +++ b/lib/ffi-xattr/extensions.rb @@ -0,0 +1,3 @@ +require 'ffi-xattr/extensions/file' +require 'ffi-xattr/extensions/pathname' + diff --git a/lib/ffi-xattr/extensions/file.rb b/lib/ffi-xattr/extensions/file.rb new file mode 100644 index 0000000..200f024 --- /dev/null +++ b/lib/ffi-xattr/extensions/file.rb @@ -0,0 +1,9 @@ +require 'ffi-xattr' + +class File + + # Returns an Xattr object for the named file (see Xattr). + def self.xattr(file_name) + Xattr.new(file_name) + end +end diff --git a/lib/ffi-xattr/extensions/pathname.rb b/lib/ffi-xattr/extensions/pathname.rb new file mode 100644 index 0000000..c2ed11e --- /dev/null +++ b/lib/ffi-xattr/extensions/pathname.rb @@ -0,0 +1,9 @@ +require 'ffi-xattr/extensions/file' + +class Pathname + # Returns an Xattr object. + # See File.xattr. + def xattr + File.xattr(self) + end +end diff --git a/lib/ffi-xattr/linux_lib.rb b/lib/ffi-xattr/linux_lib.rb new file mode 100644 index 0000000..e90a8ad --- /dev/null +++ b/lib/ffi-xattr/linux_lib.rb @@ -0,0 +1,52 @@ +class Xattr # :nodoc: all + module Lib + extend FFI::Library + + ffi_lib "c" + + attach_function :strerror, [:int], :string + + attach_function :listxattr, [:string, :pointer, :size_t], :size_t + attach_function :setxattr, [:string, :string, :pointer, :size_t, :int], :int + attach_function :getxattr, [:string, :string, :pointer, :size_t], :int + attach_function :removexattr, [:string, :string], :int + + attach_function :llistxattr, [:string, :pointer, :size_t], :size_t + attach_function :lsetxattr, [:string, :string, :pointer, :size_t, :int], :int + attach_function :lgetxattr, [:string, :string, :pointer, :size_t], :int + attach_function :lremovexattr, [:string, :string], :int + + class << self + def list(path, no_follow) + method = no_follow ? :llistxattr : :listxattr + size = send(method, path, nil, 0) + res_ptr = FFI::MemoryPointer.new(:pointer, size) + send(method, path, res_ptr, size) + + res_ptr.read_string(size).split("\000") + end + + def get(path, no_follow, key) + method = no_follow ? :lgetxattr : :getxattr + size = send(method, path, key, nil, 0) + return unless size > 0 + + str_ptr = FFI::MemoryPointer.new(:char, size) + send(method, path, key, str_ptr, size) + + str_ptr.read_string(size) + end + + def set(path, no_follow, key, value) + method = no_follow ? :lsetxattr : :setxattr + Error.check send(method, path, key, value, value.bytesize, 0) + end + + def remove(path, no_follow, key) + method = no_follow ? :lremovexattr : :removexattr + Error.check send(method, path, key) + end + end + + end +end diff --git a/lib/ffi-xattr/version.rb b/lib/ffi-xattr/version.rb new file mode 100644 index 0000000..eaafe38 --- /dev/null +++ b/lib/ffi-xattr/version.rb @@ -0,0 +1,3 @@ +class Xattr + VERSION = "0.1.2" +end diff --git a/lib/ffi-xattr/windows_lib.rb b/lib/ffi-xattr/windows_lib.rb new file mode 100644 index 0000000..d4199cc --- /dev/null +++ b/lib/ffi-xattr/windows_lib.rb @@ -0,0 +1,45 @@ +# encoding: utf-8 +require 'ffi-xattr' + +class Xattr # :nodoc: all + module Lib + class << self + + def list(path, no_follow) + lines = `dir /r "#{path}"`.split("\n") + + xattrs = [] + lines.each { |line| + if line =~ /\:\$DATA$/ + size = line.split(' ')[0].gsub(/[^0-9]/,'').to_i + + if size > 0 + xattrs << line.split(':')[1] + end + end + } + xattrs + end + + def get(path, no_follow, key) + fp = "#{path}:#{key}" + if File.exists?(fp) + File.binread(fp) + else + raise "No such key. #{key.inspect} #{path.inspect}" + end + end + + def set(path, no_follow, key, value) + File.open("#{path}:#{key}",'wb') { |io| io << value } + end + + def remove(path, no_follow, key) + # done this way because Windows have no function to remove Alternate Data Stream + # quickest way is to set the value to 0 byte length instead of trying to create another file then apply the attributes, especially when dealing with a big file + self.set(path, false, key, '') + end + end + + end +end -- cgit v1.2.1