From 7aa6160448d6a009d1bdd1f7f7d1aa454fc2bf6e Mon Sep 17 00:00:00 2001 From: Markus Kern Date: Thu, 14 Mar 2013 20:43:04 +0000 Subject: [PATCH 1/5] Added --no-meta option to prevent uploading of meta data to S3 --- s3sync.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/s3sync.rb b/s3sync.rb index c80e828..29c1319 100644 --- a/s3sync.rb +++ b/s3sync.rb @@ -68,7 +68,8 @@ def S3sync.main [ '--cache-control', GetoptLong::REQUIRED_ARGUMENT ], [ '--exclude', GetoptLong::REQUIRED_ARGUMENT ], [ '--make-dirs', GetoptLong::NO_ARGUMENT ], - [ '--no-md5', GetoptLong::NO_ARGUMENT ] + [ '--no-md5', GetoptLong::NO_ARGUMENT ], + [ '--no-meta', GetoptLong::NO_ARGUMENT ] ) def S3sync.usage(message = nil) @@ -80,7 +81,7 @@ def S3sync.usage(message = nil) --ssl -s --recursive -r --delete --public-read -p --expires="" --cache-control="" --exclude="" --progress --debug -d - --make-dirs --no-md5 + --make-dirs --no-md5 --no-meta One of or must be of S3 format, the other a local path. Reminders: * An S3 formatted item with bucket 'mybucket' and prefix 'mypre' looks like: @@ -489,11 +490,17 @@ def permissions g ? g.to_i : 600 # default to owner only end def updateFrom(fromNode) + # Don't upload directory meta files to S3. Requires use of --make-dirs when downloading. + if $S3syncOptions['--no-meta'] and fromNode.directory? + return + end if fromNode.respond_to?(:stream) meta = Hash.new - meta['owner'] = fromNode.owner.to_s - meta['group'] = fromNode.group.to_s - meta['permissions'] = fromNode.permissions.to_s + if not $S3syncOptions['--no-meta'] + meta['owner'] = fromNode.owner.to_s + meta['group'] = fromNode.group.to_s + meta['permissions'] = fromNode.permissions.to_s + end meta['symlink'] = 'true' if fromNode.symlink? begin theStream = fromNode.stream From 2bfeb0b5e3ef93d12dafbfddd0fb20ac183c4f0d Mon Sep 17 00:00:00 2001 From: Markus Kern Date: Thu, 14 Mar 2013 21:14:06 +0000 Subject: [PATCH 2/5] Move check outside updateFrom() to also supress verbose output. --- s3sync.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/s3sync.rb b/s3sync.rb index 29c1319..b9b6564 100644 --- a/s3sync.rb +++ b/s3sync.rb @@ -373,8 +373,11 @@ def S3sync.s3TreeRecurse(g, bucket, prefix, path) else LocalNode.new(localPrefix, sourceNode.name) end - puts "Create node #{sourceNode.name}" if $S3syncOptions['--verbose'] - dNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun'] + # Don't upload directory meta files to S3. Requires use of --make-dirs when downloading. + if dNode.kind_of? LocalNode or not $S3syncOptions['--no-meta'] or not sourceNode.directory? + puts "Create node #{sourceNode.name}" if $S3syncOptions['--verbose'] + dNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun'] + end sourceNode = sourceTree.next? ? sourceTree.next : nil elsif (!sourceNode) or (destinationNode and (sourceNode.name > destinationNode.name)) $stderr.puts "Source does not have #{destinationNode.name}" if $S3syncOptions['--debug'] @@ -490,10 +493,6 @@ def permissions g ? g.to_i : 600 # default to owner only end def updateFrom(fromNode) - # Don't upload directory meta files to S3. Requires use of --make-dirs when downloading. - if $S3syncOptions['--no-meta'] and fromNode.directory? - return - end if fromNode.respond_to?(:stream) meta = Hash.new if not $S3syncOptions['--no-meta'] From 31deb4a2c5b57489e4e3eb936b2d172ae00e03f7 Mon Sep 17 00:00:00 2001 From: Markus Kern Date: Fri, 15 Mar 2013 12:24:06 +0000 Subject: [PATCH 3/5] Make main files executable --- s3cmd.rb | 0 s3sync.rb | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 s3cmd.rb mode change 100644 => 100755 s3sync.rb diff --git a/s3cmd.rb b/s3cmd.rb old mode 100644 new mode 100755 diff --git a/s3sync.rb b/s3sync.rb old mode 100644 new mode 100755 From 95921c69c954c111493ee9b4f83f76e8b2c6e701 Mon Sep 17 00:00:00 2001 From: Markus Kern Date: Thu, 21 Mar 2013 18:42:13 +0000 Subject: [PATCH 4/5] If user/group meta data is missing then set to current euid/egid instead of root --- s3sync.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/s3sync.rb b/s3sync.rb index b9b6564..a8d547a 100755 --- a/s3sync.rb +++ b/s3sync.rb @@ -480,13 +480,15 @@ def owner @result = S3sync.S3try(:head, @bucket, @path) end debug("Owner of this s3 node is #{@result.object.metadata['owner']}") - @result.object.metadata['owner'].to_i # if not there, will be nil => 0 which == root so good default + g = @result.object.metadata['owner'] + g ? g.to_i : Process.euid # if not there, set to effective user id of this process end def group unless @result @result = S3sync.S3try(:head, @bucket, @path) end - @result.object.metadata['group'].to_i # 0 default ok + g = @result.object.metadata['group'] + g ? g.to_i : Process.egid # if not there, set to effective group id of this process end def permissions g = @result.object.metadata['permissions'] From 04c892224eafaf337a6916cc3e55b063bcb8c2eb Mon Sep 17 00:00:00 2001 From: Markus Kern Date: Fri, 22 Mar 2013 20:14:30 +0000 Subject: [PATCH 5/5] File permissions are octal numbers! --- s3sync.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3sync.rb b/s3sync.rb index a8d547a..1135542 100755 --- a/s3sync.rb +++ b/s3sync.rb @@ -492,7 +492,7 @@ def group end def permissions g = @result.object.metadata['permissions'] - g ? g.to_i : 600 # default to owner only + g ? g.to_i : 0600 # default to owner only end def updateFrom(fromNode) if fromNode.respond_to?(:stream) @@ -621,7 +621,7 @@ def group self.exist? ? self.stat().gid : 0 end def permissions - self.exist? ? self.stat().mode : 600 + self.exist? ? self.stat().mode : 0600 end def updateFrom(fromNode) if fromNode.respond_to?(:to_stream)