Follow symbolic links

CHANGELOG.md

@@ -30,6 +30,7 @@ This is a major release with many breaking changes.
 - `-ds`, `-dw`, `-us` and `-uw` global flags are no longer available. Multipart
   concurrency and part size flags are now part of the `cp/mv` command. New
   replacement flags are `--concurrency | -c` and `--part-size | -p`. ([#110](https://github.com/peak/s5cmd/pull/110))
+- s5cmd `cp` command follows symbolic links by default (only when uploading to s3 from local filesystem). Use `--no-follow-symlinks` flag to disable this feature.
 
 #### Features
 

command/cp.go

@@ -80,6 +80,10 @@ var copyCommandFlags = []cli.Flag{
 		Name:  "parents",
 		Usage: "create same directory structure of source, starting from the first wildcard",
 	},
+	&cli.BoolFlag{
+		Name:  "no-follow-symlinks",
+		Usage: "do not follow symbolic links (s5cmd follows symbolic links by default)",
+	},
 	&cli.StringFlag{
 		Name:  "storage-class",
 		Usage: "set storage class for target ('STANDARD','REDUCED_REDUNDANCY','GLACIER','STANDARD_IA')",
@@ -124,6 +128,7 @@ var CopyCommand = &cli.Command{
 			partSize:      c.Int64("part-size") * megabytes,
 		}
 
+		storage.FollowSymlinks = !c.Bool("no-follow-symlinks")
 		return copyCommand.Run(c.Context)
 	},
 }

command/expand.go

@@ -34,7 +34,9 @@ func expandSource(
 	}
 
 	ch := make(chan *storage.Object, 1)
-	ch <- &storage.Object{URL: srcurl}
+	if storage.ShouldProcessUrl(srcurl) {
+		ch <- &storage.Object{URL: srcurl}
+	}
 	close(ch)
 	return ch, nil
 }

command/expand_test.go

@@ -2,10 +2,14 @@ package command
 
 import (
 	"context"
+	"os"
 	"reflect"
 	"sort"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+	"gotest.tools/v3/fs"
+
 	"github.com/stretchr/testify/mock"
 
 	"github.com/peak/s5cmd/storage"
@@ -175,6 +179,162 @@ func TestExpandSources(t *testing.T) {
 	}
 }
 
+func TestExpandSource_Follow_Link_To_Single_File(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+		),
+		fs.WithDir(
+			"b",
+		),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	os.Symlink(workdir.Join("a/f1.txt"), workdir.Join("b/my_link"))
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("b/my_link"))
+
+	//follow symbolic links
+	storage.FollowSymlinks = true
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Equal(t, []string{workdir.Join("b/my_link")}, expected)
+}
+
+func TestExpandSource_Do_Not_Follow_Link_To_Single_File(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+		),
+		fs.WithDir(
+			"b",
+		),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	os.Symlink(workdir.Join("a/f1.txt"), workdir.Join("b/my_link"))
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("b/my_link"))
+
+	//do not follow symbolic links
+	storage.FollowSymlinks = false
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Empty(t, expected)
+}
+
+func TestExpandSource_Follow_Link_To_Directory(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+			fs.WithFile("f2.txt", ""),
+			fs.WithDir("b",
+				fs.WithFile("f3.txt", "")),
+		),
+		fs.WithDir(
+			"c",
+		),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	os.Symlink(workdir.Join("a"), workdir.Join("c/my_link"))
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("c/my_link"))
+
+	//follow symbolic links
+	storage.FollowSymlinks = true
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	sort.Strings(expected)
+	assert.Equal(t, []string{
+		workdir.Join("c/my_link/b/f3.txt"),
+		workdir.Join("c/my_link/f1.txt"),
+		workdir.Join("c/my_link/f2.txt"),
+	}, expected)
+}
+
+func TestExpandSource_Do_Not_Follow_Link_To_Directory(t *testing.T) {
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", ""),
+			fs.WithFile("f2.txt", ""),
+			fs.WithDir("b",
+				fs.WithFile("f3.txt", "")),
+		),
+		fs.WithDir(
+			"c",
+		),
+	}
+
+	workdir := fs.NewDir(t, "expandsourcetest", folderLayout...)
+	defer workdir.Remove()
+
+	os.Symlink(workdir.Join("a"), workdir.Join("c/my_link"))
+
+	ctx := context.Background()
+	workdirUrl, _ := url.New(workdir.Join("c/my_link"))
+
+	//do not follow symbolic links
+	storage.FollowSymlinks = false
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Empty(t, expected)
+}
+
+func TestExpandSource_Do_Not_Follow_Symlinks(t *testing.T) {
+	ctx := context.Background()
+	fileContent := "CAFEBABE"
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", fileContent),
+		),
+		fs.WithDir("b"),
+		fs.WithDir("c"),
+	}
+
+	workdir := fs.NewDir(t, t.Name(), folderLayout...)
+	defer workdir.Remove()
+
+	workdirUrl, _ := url.New(workdir.Path())
+	os.Symlink(workdir.Join("a/f1.txt"), workdir.Join("b/link1"))
+	os.Symlink(workdir.Join("b/link1"), workdir.Join("c/link2"))
+
+	//do not follow symbolic links
+	storage.FollowSymlinks = false
+	ch, _ := expandSource(ctx, storage.NewFilesystem(), workdirUrl)
+	var expected []string
+	for obj := range ch {
+		expected = append(expected, obj.URL.Absolute())
+	}
+	assert.Equal(t, []string{workdir.Join("a/f1.txt")}, expected)
+}
+
 func keys(urls map[string][]*storage.Object) []string {
 	var urlKeys []string
 	for key := range urls {

e2e/cp_test.go

@@ -24,6 +24,7 @@ package e2e
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
@@ -2175,3 +2176,127 @@ func TestCopyMultipleLocalNestedFilesToS3WithParents(t *testing.T) {
 	assert.Assert(t, ensureS3Object(s3client, bucket, "prefix/a/file1.txt", "file1"))
 	assert.Assert(t, ensureS3Object(s3client, bucket, "prefix/b/c/file2.txt", "file2"))
 }
+
+// cp --no-follow-symlinks my_link s3://bucket/prefix/
+func TestCopyLinkToASingleFileWithFollowSymlinkDisabled(t *testing.T) {
+	t.Parallel()
+
+	s3client, s5cmd, cleanup := setup(t)
+	defer cleanup()
+
+	const bucket = "bucket"
+	createBucket(t, s3client, bucket)
+
+	fileContent := "CAFEBABE"
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", fileContent),
+		),
+		fs.WithDir("b"),
+	}
+
+	workdir := fs.NewDir(t, t.Name(), folderLayout...)
+	defer workdir.Remove()
+
+	os.Symlink(workdir.Join("a/f1.txt"), workdir.Join("b/my_link"))
+
+	dst := fmt.Sprintf("s3://%v/prefix/", bucket)
+
+	cmd := s5cmd("cp", "--no-follow-symlinks", "b/my_link", dst)
+	result := icmd.RunCmd(cmd, withWorkingDir(workdir))
+
+	result.Assert(t, icmd.Success)
+
+	assertLines(t, result.Stdout(), map[int]compareFunc{
+		0: equals(""),
+	}, sortInput(true))
+}
+
+// cp --parents * s3://bucket/prefix/
+func TestCopyWithFollowSymlink(t *testing.T) {
+	t.Parallel()
+
+	s3client, s5cmd, cleanup := setup(t)
+	defer cleanup()
+
+	const bucket = "bucket"
+	createBucket(t, s3client, bucket)
+
+	fileContent := "CAFEBABE"
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", fileContent),
+		),
+		fs.WithDir("b"),
+		fs.WithDir("c"),
+	}
+
+	workdir := fs.NewDir(t, t.Name(), folderLayout...)
+	defer workdir.Remove()
+
+	os.Symlink(workdir.Join("a/f1.txt"), workdir.Join("b/link1"))
+	os.Symlink(workdir.Join("b/link1"), workdir.Join("c/link2"))
+
+	dst := fmt.Sprintf("s3://%v/prefix/", bucket)
+
+	cmd := s5cmd("cp", "--parents", "*", dst)
+	result := icmd.RunCmd(cmd, withWorkingDir(workdir))
+
+	result.Assert(t, icmd.Success)
+
+	assertLines(t, result.Stdout(), map[int]compareFunc{
+		0: equals(""),
+		1: equals("cp a/f1.txt %va/f1.txt", dst),
+		2: equals("cp b/link1 %vb/link1", dst),
+		3: equals("cp c/link2 %vc/link2", dst),
+	}, sortInput(true))
+
+	// assert s3 objects
+	assert.Assert(t, ensureS3Object(s3client, bucket, "prefix/a/f1.txt", fileContent))
+	assert.Assert(t, ensureS3Object(s3client, bucket, "prefix/b/link1", fileContent))
+	assert.Assert(t, ensureS3Object(s3client, bucket, "prefix/c/link2", fileContent))
+}
+
+// cp --parents --no-follow-symlinks * s3://bucket/prefix/
+func TestCopyWithNoFollowSymlink(t *testing.T) {
+	t.Parallel()
+
+	s3client, s5cmd, cleanup := setup(t)
+	defer cleanup()
+
+	const bucket = "bucket"
+	createBucket(t, s3client, bucket)
+
+	fileContent := "CAFEBABE"
+	folderLayout := []fs.PathOp{
+		fs.WithDir(
+			"a",
+			fs.WithFile("f1.txt", fileContent),
+		),
+		fs.WithDir("b"),
+		fs.WithDir("c"),
+	}
+
+	workdir := fs.NewDir(t, t.Name(), folderLayout...)
+	defer workdir.Remove()
+
+	os.Symlink(workdir.Join("a/f1.txt"), workdir.Join("b/link1"))
+	os.Symlink(workdir.Join("b/link1"), workdir.Join("c/link2"))
+
+	dst := fmt.Sprintf("s3://%v/prefix/", bucket)
+
+	cmd := s5cmd("cp", "--parents", "--no-follow-symlinks", "*", dst)
+	result := icmd.RunCmd(cmd, withWorkingDir(workdir))
+
+	result.Assert(t, icmd.Success)
+
+	assertLines(t, result.Stdout(), map[int]compareFunc{
+		0: equals(""),
+		1: equals("cp a/f1.txt %va/f1.txt", dst),
+	}, sortInput(true))
+
+	// assert s3 objects
+	assert.Assert(t, ensureS3Object(s3client, bucket, "prefix/a/f1.txt", fileContent))
+}

storage/fs.go

@@ -104,6 +104,10 @@ func (f *Filesystem) expandGlob(ctx context.Context, src *url.URL) <-chan *Objec
 }
 
 func walkDir(ctx context.Context, storage Storage, src *url.URL, fn func(o *Object)) {
+	//skip if symlink is pointing to a dir and --no-follow-symlink
+	if !ShouldProcessUrl(src) {
+		return
+	}
 	err := godirwalk.Walk(src.Absolute(), &godirwalk.Options{
 		Callback: func(pathname string, dirent *godirwalk.Dirent) error {
 			// we're interested in files
@@ -119,15 +123,20 @@ func walkDir(ctx context.Context, storage Storage, src *url.URL, fn func(o *Obje
 			fileurl.SetRelative(src.Absolute())
 
 			obj, err := storage.Stat(ctx, fileurl)
+
+			//skip if symlink is pointing to a file and --no-follow-symlink
+			if !ShouldProcessUrl(fileurl) {
+				return nil
+			}
+
 			if err != nil {
 				return err
 			}
 			fn(obj)
 			return nil
 		},
-		// TODO(ig): enable following symlink once we have the necessary cli
 		// flags
-		FollowSymbolicLinks: false,
+		FollowSymbolicLinks: FollowSymlinks,
 	})
 	if err != nil {
 		obj := &Object{Err: err}