Follow symbolic links

CHANGELOG.md

@@ -30,6 +30,7 @@ This is a major release with many breaking changes.
 - `-ds`, `-dw`, `-us` and `-uw` global flags are no longer available. Multipart
   concurrency and part size flags are now part of the `cp/mv` command. New
   replacement flags are `--concurrency | -c` and `--part-size | -p`. ([#110](https://github.com/peak/s5cmd/pull/110))
+- s5cmd `cp` command follows symbolic links by default (only when uploading to s3 from local filesystem). Use `--no-follow-symlinks` flag to disable this feature.
 
 #### Features
 

command/cp.go

@@ -80,6 +80,10 @@ var copyCommandFlags = []cli.Flag{
 		Name:  "parents",
 		Usage: "create same directory structure of source, starting from the first wildcard",
 	},
+	&cli.BoolFlag{
+		Name:  "no-follow-symlinks",
+		Usage: "do not follow symbolic links",
+	},
 	&cli.StringFlag{
 		Name:  "storage-class",
 		Usage: "set storage class for target ('STANDARD','REDUCED_REDUNDANCY','GLACIER','STANDARD_IA')",
@@ -115,15 +119,15 @@ var CopyCommand = &cli.Command{
 			fullCommand:  givenCommand(c),
 			deleteSource: false, // don't delete source
 			// flags
-			noClobber:     c.Bool("no-clobber"),
-			ifSizeDiffer:  c.Bool("if-size-differ"),
-			ifSourceNewer: c.Bool("if-source-newer"),
-			parents:       c.Bool("parents"),
-			storageClass:  storage.LookupClass(c.String("storage-class")),
-			concurrency:   c.Int("concurrency"),
-			partSize:      c.Int64("part-size") * megabytes,
+			noClobber:      c.Bool("no-clobber"),
+			ifSizeDiffer:   c.Bool("if-size-differ"),
+			ifSourceNewer:  c.Bool("if-source-newer"),
+			parents:        c.Bool("parents"),
+			followSymlinks: !c.Bool("no-follow-symlinks"),
+			storageClass:   storage.LookupClass(c.String("storage-class")),
+			concurrency:    c.Int("concurrency"),
+			partSize:       c.Int64("part-size") * megabytes,
 		}
-
 		return copyCommand.Run(c.Context)
 	},
 }
@@ -137,11 +141,12 @@ type Copy struct {
 	deleteSource bool
 
 	// flags
-	noClobber     bool
-	ifSizeDiffer  bool
-	ifSourceNewer bool
-	parents       bool
-	storageClass  storage.StorageClass
+	noClobber      bool
+	ifSizeDiffer   bool
+	ifSourceNewer  bool
+	parents        bool
+	followSymlinks bool
+	storageClass   storage.StorageClass
 
 	// s3 options
 	concurrency int
@@ -164,7 +169,7 @@ func (c Copy) Run(ctx context.Context) error {
 		return err
 	}
 
-	objch, err := expandSource(ctx, client, srcurl)
+	objch, err := expandSource(ctx, client, c.followSymlinks, srcurl)
 	if err != nil {
 		return err
 	}

command/du.go

@@ -90,7 +90,7 @@ func Size(
 
 	var merror error
 
-	for object := range client.List(ctx, srcurl) {
+	for object := range client.List(ctx, srcurl, false) {
 		if object.Type.IsDir() || errorpkg.IsCancelation(object.Err) {
 			continue
 		}

command/expand.go

@@ -14,6 +14,7 @@ import (
 func expandSource(
 	ctx context.Context,
 	client storage.Storage,
+	followSymlinks bool,
 	srcurl *url.URL,
 ) (<-chan *storage.Object, error) {
 	var isDir bool
@@ -30,11 +31,13 @@ func expandSource(
 
 	// call storage.List for only walking operations.
 	if srcurl.HasGlob() || isDir {
-		return client.List(ctx, srcurl), nil
+		return client.List(ctx, srcurl, followSymlinks), nil
 	}
 
 	ch := make(chan *storage.Object, 1)
-	ch <- &storage.Object{URL: srcurl}
+	if storage.ShouldProcessUrl(srcurl, followSymlinks) {
+		ch <- &storage.Object{URL: srcurl}
+	}
 	close(ch)
 	return ch, nil
 }
@@ -46,6 +49,7 @@ func expandSource(
 func expandSources(
 	ctx context.Context,
 	client storage.Storage,
+	followSymlinks bool,
 	srcurls ...*url.URL,
 ) <-chan *storage.Object {
 	ch := make(chan *storage.Object)
@@ -61,7 +65,7 @@ func expandSources(
 			go func(origSrc *url.URL) {
 				defer wg.Done()
 
-				objch, err := expandSource(ctx, client, origSrc)
+				objch, err := expandSource(ctx, client, followSymlinks, origSrc)
 				if err != nil {
 					ch <- &storage.Object{Err: err}
 					return

command/expand_test.go

@@ -6,6 +6,9 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+	"gotest.tools/v3/fs"
+
 	"github.com/stretchr/testify/mock"
 
 	"github.com/peak/s5cmd/storage"
@@ -152,7 +155,7 @@ func TestExpandSources(t *testing.T) {
 				}
 			}
 
-			gotChan := expandSources(context.Background(), client, srcurls...)
+			gotChan := expandSources(context.Background(), client, false, srcurls...)
 
 			var objects []string
 			for obj := range gotChan {
@@ -175,6 +178,153 @@ func TestExpandSources(t *testing.T) {
 	}
 }
 
+func TestExpandSource_Follow_Link_To_Single_File(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+		),
+		fs.WithDir(
+			"b",
+		),
+		fs.WithSymlink("b/my_link", "a/f1.txt"),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("b/my_link"))
+
+	//follow symbolic links
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), true, workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Equal(t, []string{workdir.Join("b/my_link")}, expected)
+}
+
+func TestExpandSource_Do_Not_Follow_Link_To_Single_File(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+		),
+		fs.WithDir(
+			"b",
+		),
+		fs.WithSymlink("b/my_link", "a/f1.txt"),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("b/my_link"))
+
+	//do not follow symbolic links
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), false, workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Empty(t, expected)
+}
+
+func TestExpandSource_Follow_Link_To_Directory(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+			fs.WithFile("f2.txt", ""),
+			fs.WithDir("b",
+				fs.WithFile("f3.txt", "")),
+		),
+		fs.WithDir(
+			"c",
+		),
+		fs.WithSymlink("c/my_link", "a"),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("c/my_link"))
+
+	//follow symbolic links
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), true, workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	sort.Strings(expected)
+	assert.Equal(t, []string{
+		workdir.Join("c/my_link/b/f3.txt"),
+		workdir.Join("c/my_link/f1.txt"),
+		workdir.Join("c/my_link/f2.txt"),
+	}, expected)
+}
+
+func TestExpandSource_Do_Not_Follow_Link_To_Directory(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+			fs.WithFile("f2.txt", ""),
+			fs.WithDir("b",
+				fs.WithFile("f3.txt", "")),
+		),
+		fs.WithDir(
+			"c",
+		),
+		fs.WithSymlink("c/my_link", "a"),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("c/my_link"))
+
+	//do not follow symbolic links
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), false, workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Empty(t, expected)
+}
+
+func TestExpandSource_Do_Not_Follow_Symlinks(t *testing.T) {
+	ctx := context.Background()
+	fileContent := "CAFEBABE"
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", fileContent),
+		),
+		fs.WithDir("b"),
+		fs.WithDir("c"),
+		fs.WithSymlink("b/link1", "a/f1.txt"),
+		fs.WithSymlink("c/link2", "b/link1"),
+	}
+
+	workdir := fs.NewDir(t, t.Name(), folderLayout...)
+	defer workdir.Remove()
+
+	workdirUrl, _ := url.New(workdir.Path())
+
+	//do not follow symbolic links
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), false, workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Equal(t, []string{workdir.Join("a/f1.txt")}, expected)
+}
+
 func keys(urls map[string][]*storage.Object) []string {
 	var urlKeys []string
 	for key := range urls {

command/ls.go

@@ -116,7 +116,7 @@ func List(
 
 	var merror error
 
-	for object := range client.List(ctx, srcurl) {
+	for object := range client.List(ctx, srcurl, false) {
 		if errorpkg.IsCancelation(object.Err) {
 			continue
 		}

command/rm.go

@@ -75,7 +75,7 @@ func Delete(
 		return err
 	}
 
-	objChan := expandSources(ctx, client, srcurls...)
+	objChan := expandSources(ctx, client, false, srcurls...)
 
 	// do object->url transformation
 	urlch := make(chan *url.URL)