Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Follow symbolic links #135

Merged
merged 9 commits into from
Mar 30, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@ This is a major release with many breaking changes.
- `-ds`, `-dw`, `-us` and `-uw` global flags are no longer available. Multipart
concurrency and part size flags are now part of the `cp/mv` command. New
replacement flags are `--concurrency | -c` and `--part-size | -p`. ([#110](https://github.com/peak/s5cmd/pull/110))
- s5cmd `cp` command follows symbolic links by default (only when uploading to s3 from local filesystem). Use `--no-follow-symlinks` flag to disable this feature.
- Dropped `-parents` flag from copy command. Copy behaviour has changed to preserve the directory hierarchy as a default.
Optional `-flatten` flag is added to flatten directory structure.

#### Features

- Added `mb` command to make buckets. ([#25](https://github.com/peak/s5cmd/issues/25))
Expand Down
33 changes: 19 additions & 14 deletions command/cp.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@ var copyCommandFlags = []cli.Flag{
Aliases: []string{"f"},
Usage: "flatten directory structure of source, starting from the first wildcard",
},
&cli.BoolFlag{
Name: "no-follow-symlinks",
Usage: "do not follow symbolic links",
},
&cli.StringFlag{
Name: "storage-class",
Usage: "set storage class for target ('STANDARD','REDUCED_REDUNDANCY','GLACIER','STANDARD_IA')",
Expand Down Expand Up @@ -116,15 +120,15 @@ var CopyCommand = &cli.Command{
fullCommand: givenCommand(c),
deleteSource: false, // don't delete source
// flags
noClobber: c.Bool("no-clobber"),
ifSizeDiffer: c.Bool("if-size-differ"),
ifSourceNewer: c.Bool("if-source-newer"),
flatten: c.Bool("flatten"),
storageClass: storage.LookupClass(c.String("storage-class")),
concurrency: c.Int("concurrency"),
partSize: c.Int64("part-size") * megabytes,
noClobber: c.Bool("no-clobber"),
ifSizeDiffer: c.Bool("if-size-differ"),
ifSourceNewer: c.Bool("if-source-newer"),
flatten: c.Bool("flatten"),
followSymlinks: !c.Bool("no-follow-symlinks"),
storageClass: storage.LookupClass(c.String("storage-class")),
concurrency: c.Int("concurrency"),
partSize: c.Int64("part-size") * megabytes,
}

return copyCommand.Run(c.Context)
},
}
Expand All @@ -138,11 +142,12 @@ type Copy struct {
deleteSource bool

// flags
noClobber bool
ifSizeDiffer bool
ifSourceNewer bool
flatten bool
storageClass storage.StorageClass
noClobber bool
ifSizeDiffer bool
ifSourceNewer bool
flatten bool
followSymlinks bool
storageClass storage.StorageClass

// s3 options
concurrency int
Expand All @@ -165,7 +170,7 @@ func (c Copy) Run(ctx context.Context) error {
return err
}

objch, err := expandSource(ctx, client, srcurl)
objch, err := expandSource(ctx, client, c.followSymlinks, srcurl)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion command/du.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func Size(

var merror error

for object := range client.List(ctx, srcurl) {
for object := range client.List(ctx, srcurl, false) {
if object.Type.IsDir() || errorpkg.IsCancelation(object.Err) {
continue
}
Expand Down
10 changes: 7 additions & 3 deletions command/expand.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
func expandSource(
ctx context.Context,
client storage.Storage,
followSymlinks bool,
srcurl *url.URL,
) (<-chan *storage.Object, error) {
var isDir bool
Expand All @@ -30,11 +31,13 @@ func expandSource(

// call storage.List for only walking operations.
if srcurl.HasGlob() || isDir {
return client.List(ctx, srcurl), nil
return client.List(ctx, srcurl, followSymlinks), nil
}

ch := make(chan *storage.Object, 1)
ch <- &storage.Object{URL: srcurl}
if storage.ShouldProcessUrl(srcurl, followSymlinks) {
ch <- &storage.Object{URL: srcurl}
}
close(ch)
return ch, nil
}
Expand All @@ -46,6 +49,7 @@ func expandSource(
func expandSources(
ctx context.Context,
client storage.Storage,
followSymlinks bool,
srcurls ...*url.URL,
) <-chan *storage.Object {
ch := make(chan *storage.Object)
Expand All @@ -61,7 +65,7 @@ func expandSources(
go func(origSrc *url.URL) {
defer wg.Done()

objch, err := expandSource(ctx, client, origSrc)
objch, err := expandSource(ctx, client, followSymlinks, origSrc)
if err != nil {
ch <- &storage.Object{Err: err}
return
Expand Down
154 changes: 152 additions & 2 deletions command/expand_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ import (
"sort"
"testing"

"github.com/stretchr/testify/assert"
"gotest.tools/v3/fs"

"github.com/stretchr/testify/mock"

"github.com/peak/s5cmd/storage"
Expand Down Expand Up @@ -148,11 +151,11 @@ func TestExpandSources(t *testing.T) {
ch := generateObjects(objects)

if src != "s3://bucket/key" {
client.On("List", mock.Anything, srcurl).Once().Return(ch)
client.On("List", mock.Anything, srcurl, mock.Anything).Once().Return(ch)
}
}

gotChan := expandSources(context.Background(), client, srcurls...)
gotChan := expandSources(context.Background(), client, false, srcurls...)

var objects []string
for obj := range gotChan {
Expand All @@ -175,6 +178,153 @@ func TestExpandSources(t *testing.T) {
}
}

func TestExpandSource_Follow_Link_To_Single_File(t *testing.T) {
folderLayout := []fs.PathOp{
fs.WithDir(
"a",
fs.WithFile("f1.txt", ""),
),
fs.WithDir(
"b",
),
fs.WithSymlink("b/my_link", "a/f1.txt"),
}

workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
defer workdir.Remove()

ctx := context.Background()
workdirUrl, _ := url.New(workdir.Join("b/my_link"))

//follow symbolic links
ch, _ := expandSource(ctx, storage.NewFilesystem(), true, workdirUrl)
var expected []string
for obj := range ch {
expected = append(expected, obj.URL.Absolute())
}
assert.Equal(t, []string{workdir.Join("b/my_link")}, expected)
}

func TestExpandSource_Do_Not_Follow_Link_To_Single_File(t *testing.T) {
folderLayout := []fs.PathOp{
fs.WithDir(
"a",
fs.WithFile("f1.txt", ""),
),
fs.WithDir(
"b",
),
fs.WithSymlink("b/my_link", "a/f1.txt"),
}

workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
defer workdir.Remove()

ctx := context.Background()
workdirUrl, _ := url.New(workdir.Join("b/my_link"))

//do not follow symbolic links
ch, _ := expandSource(ctx, storage.NewFilesystem(), false, workdirUrl)
var expected []string
for obj := range ch {
expected = append(expected, obj.URL.Absolute())
}
assert.Empty(t, expected)
}

func TestExpandSource_Follow_Link_To_Directory(t *testing.T) {
folderLayout := []fs.PathOp{
fs.WithDir(
"a",
fs.WithFile("f1.txt", ""),
fs.WithFile("f2.txt", ""),
fs.WithDir("b",
fs.WithFile("f3.txt", "")),
),
fs.WithDir(
"c",
),
fs.WithSymlink("c/my_link", "a"),
}

workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
defer workdir.Remove()

ctx := context.Background()
workdirUrl, _ := url.New(workdir.Join("c/my_link"))

//follow symbolic links
ch, _ := expandSource(ctx, storage.NewFilesystem(), true, workdirUrl)
var expected []string
for obj := range ch {
expected = append(expected, obj.URL.Absolute())
}
sort.Strings(expected)
assert.Equal(t, []string{
workdir.Join("c/my_link/b/f3.txt"),
workdir.Join("c/my_link/f1.txt"),
workdir.Join("c/my_link/f2.txt"),
}, expected)
}

func TestExpandSource_Do_Not_Follow_Link_To_Directory(t *testing.T) {
folderLayout := []fs.PathOp{
fs.WithDir(
"a",
fs.WithFile("f1.txt", ""),
fs.WithFile("f2.txt", ""),
fs.WithDir("b",
fs.WithFile("f3.txt", "")),
),
fs.WithDir(
"c",
),
fs.WithSymlink("c/my_link", "a"),
}

workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
defer workdir.Remove()

ctx := context.Background()
workdirUrl, _ := url.New(workdir.Join("c/my_link"))

//do not follow symbolic links
ch, _ := expandSource(ctx, storage.NewFilesystem(), false, workdirUrl)
var expected []string
for obj := range ch {
expected = append(expected, obj.URL.Absolute())
}
assert.Empty(t, expected)
}

func TestExpandSource_Do_Not_Follow_Symlinks(t *testing.T) {
ctx := context.Background()
fileContent := "CAFEBABE"
folderLayout := []fs.PathOp{
fs.WithDir(
"a",
fs.WithFile("f1.txt", fileContent),
),
fs.WithDir("b"),
fs.WithDir("c"),
fs.WithSymlink("b/link1", "a/f1.txt"),
fs.WithSymlink("c/link2", "b/link1"),
}

workdir := fs.NewDir(t, t.Name(), folderLayout...)
defer workdir.Remove()

workdirUrl, _ := url.New(workdir.Path())

//do not follow symbolic links
ch, _ := expandSource(ctx, storage.NewFilesystem(), false, workdirUrl)
var expected []string
for obj := range ch {
expected = append(expected, obj.URL.Absolute())
}
assert.Equal(t, []string{workdir.Join("a/f1.txt")}, expected)
}

func keys(urls map[string][]*storage.Object) []string {
var urlKeys []string
for key := range urls {
Expand Down
2 changes: 1 addition & 1 deletion command/ls.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func List(

var merror error

for object := range client.List(ctx, srcurl) {
for object := range client.List(ctx, srcurl, false) {
if errorpkg.IsCancelation(object.Err) {
continue
}
Expand Down
2 changes: 1 addition & 1 deletion command/rm.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func Delete(
return err
}

objChan := expandSources(ctx, client, srcurls...)
objChan := expandSources(ctx, client, false, srcurls...)

// do object->url transformation
urlch := make(chan *url.URL)
Expand Down
Loading