add msync and expand sync to support lists

README.md

@@ -326,9 +326,8 @@ This approach is intentional to prevent confusion and make it easier to comprehe
 Spot supports the following command types:
 
 - `script`: can be any valid shell script. The script will be executed on the remote host(s) using SSH, inside a shell.
-- `copy`: copies a file from the local machine to the remote host(s). Example: `copy: {"src": "testdata/conf.yml", "dst": "/tmp/conf.yml", "mkdir": true}`. If `mkdir` is set to `true` the command will create the destination directory if it doesn't exist, same as `mkdir -p` in bash. Note: `copy` command type supports multiple commands too, the same way as `mcopy` below.
-- `mcopy`: copies multiple files from the local machine to the remote host(s). Example: `mcopy: [{"src": "testdata/1.yml", "dst": "/tmp/1.yml", "mkdir": true}, {"src": "testdata/1.txt", "dst": "/tmp/1.txt"}]`. This is just a shortcut for multiple `copy` commands.
-- `sync`: syncs directory from the local machine to the remote host(s). Optionally supports deleting files on the remote host(s) that don't exist locally. Example: `sync: {"src": "testdata", "dst": "/tmp/things", "delete": true}`. Another option is `exclude` which allows to specify a list of files to exclude from the sync. Example: `sync: {"src": "testdata", "dst": "/tmp/things", "exclude": ["*.txt", "*.yml"]}` 
+- `copy`: copies a file from the local machine to the remote host(s). Example: `copy: {"src": "testdata/conf.yml", "dst": "/tmp/conf.yml", "mkdir": true}`. If `mkdir` is set to `true` the command will create the destination directory if it doesn't exist, same as `mkdir -p` in bash. Note: `copy` command type supports multiple commands too presented as a list of copy elements.
+- `sync`: syncs directory from the local machine to the remote host(s). Optionally supports deleting files on the remote host(s) that don't exist locally. Example: `sync: {"src": "testdata", "dst": "/tmp/things", "delete": true}`. Another option is `exclude` which allows to specify a list of files to exclude from the sync. Example: `sync: {"src": "testdata", "dst": "/tmp/things", "exclude": ["*.txt", "*.yml"]}`. Note: `sync` command type supports multiple commands too presented as a list of sync elements.
 - `delete`: deletes a file or directory on the remote host(s), optionally can remove recursively. Example: `delete: {"path": "/tmp/things", "recur": true}`
 - `wait`: waits for the specified command to finish on the remote host(s) with 0 error code. This command is useful when you need to wait for a service to start before executing the next command. Allows to specify the timeout as well as check interval. Example: `wait: {"cmd": "curl -s --fail localhost:8080", "timeout": "30s", "interval": "1s"}`
 - `echo`: prints the specified message to the console. Example: `echo: "Hello World $some_var"`. This command is useful for debugging purposes and also to print the value of variables to the console.

pkg/config/command.go

@@ -19,6 +19,7 @@ type Cmd struct {
 	Copy        CopyInternal      `yaml:"copy" toml:"copy"`
 	MCopy       []CopyInternal    `yaml:"mcopy" toml:"mcopy"`
 	Sync        SyncInternal      `yaml:"sync" toml:"sync"`
+	MSync       []SyncInternal    `yaml:"msync" toml:"msync"`
 	Delete      DeleteInternal    `yaml:"delete" toml:"delete"`
 	Wait        WaitInternal      `yaml:"wait" toml:"wait"`
 	Script      string            `yaml:"script" toml:"script,multiline"`
@@ -300,7 +301,7 @@ func (cmd *Cmd) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		fieldName := field.Tag.Get("yaml")
 
 		// skip copy, processed separately. fields without yaml tag or with "-" are skipped too
-		if fieldName == "copy" || fieldName == "" || fieldName == "-" {
+		if fieldName == "copy" || fieldName == "sync" || fieldName == "" || fieldName == "-" {
 			continue
 		}
 
@@ -316,6 +317,14 @@ func (cmd *Cmd) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			return err
 		}
 	}
+
+	// sync is a special case, as it can be either a struct or a list of structs
+	if err := unmarshalField("sync", &cmd.Sync); err != nil {
+		if err := unmarshalField("sync", &cmd.MSync); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -331,6 +340,7 @@ func (cmd *Cmd) validate() error {
 		{"mcopy", func() bool { return len(cmd.MCopy) > 0 }},
 		{"delete", func() bool { return cmd.Delete.Location != "" }},
 		{"sync", func() bool { return cmd.Sync.Source != "" && cmd.Sync.Dest != "" }},
+		{"msync", func() bool { return len(cmd.MSync) > 0 }},
 		{"wait", func() bool { return cmd.Wait.Command != "" }},
 		{"echo", func() bool { return cmd.Echo != "" }},
 	}

pkg/config/command_test.go

@@ -286,7 +286,7 @@ func TestCmd_UnmarshalYAML(t *testing.T) {
 
 	testCases := []testCase{
 		{
-			name: "Simple case",
+			name: "simple case",
 			yamlInput: `
 name: test
 script: echo "Hello, World!"
@@ -298,19 +298,30 @@ script: echo "Hello, World!"
 		},
 
 		{
-			name: "Copy and Mcopy",
+			name: "copy multiple sets",
 			yamlInput: `
 name: test
 copy:
-  src: source
-  dst: destination
+  - {src: source1, dst: destination1}
+  - {src: source2, dst: destination2}
 `,
 			expectedCmd: Cmd{
-				Name: "test",
-				Copy: CopyInternal{
-					Source: "source",
-					Dest:   "destination",
-				},
+				Name:  "test",
+				MCopy: []CopyInternal{{Source: "source1", Dest: "destination1"}, {Source: "source2", Dest: "destination2"}},
+			},
+		},
+
+		{
+			name: "msync",
+			yamlInput: `
+name: test
+sync:
+  - {src: source1, dst: destination1}
+  - {src: source2, dst: destination2}
+`,
+			expectedCmd: Cmd{
+				Name:  "test",
+				MSync: []SyncInternal{{Source: "source1", Dest: "destination1"}, {Source: "source2", Dest: "destination2"}},
 			},
 		},
 
@@ -412,10 +423,11 @@ func TestCmd_validate(t *testing.T) {
 		{"only mcopy", Cmd{MCopy: []CopyInternal{{Source: "source1", Dest: "dest1"}, {Source: "source2", Dest: "dest2"}}}, ""},
 		{"only delete", Cmd{Delete: DeleteInternal{Location: "location"}}, ""},
 		{"only sync", Cmd{Sync: SyncInternal{Source: "source", Dest: "dest"}}, ""},
+		{"only msync", Cmd{MSync: []SyncInternal{{Source: "source", Dest: "dest"}}}, ""},
 		{"only wait", Cmd{Wait: WaitInternal{Command: "command"}}, ""},
 		{"multiple fields set", Cmd{Script: "example_script", Copy: CopyInternal{Source: "source", Dest: "dest"}},
 			"only one of [script, copy] is allowed"},
-		{"nothing set", Cmd{}, "one of [script, copy, mcopy, delete, sync, wait, echo] must be set"},
+		{"nothing set", Cmd{}, "one of [script, copy, mcopy, delete, sync, msync, wait, echo] must be set"},
 	}
 
 	for _, tt := range tbl {

pkg/executor/remote.go

@@ -437,6 +437,7 @@ func (ex *Remote) getRemoteFilesProperties(ctx context.Context, dir string, excl
 	var processEntry func(ctx context.Context, client *sftp.Client, root string, excl []string, dir string) error
 
 	processEntry = func(ctx context.Context, client *sftp.Client, root string, excl []string, dir string) error {
+		log.Printf("[DEBUG] processing remote directory %s", dir)
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -455,7 +456,7 @@ func (ex *Remote) getRemoteFilesProperties(ctx context.Context, dir string, excl
 			fullPath := filepath.Join(dir, entry.Name())
 			relPath, err := filepath.Rel(root, fullPath)
 			if err != nil {
-				log.Printf("[ERROR] failed to get relative path for %s: %v", fullPath, err)
+				log.Printf("[WARN] failed to get relative path for %s: %v", fullPath, err)
 				continue
 			}
 
@@ -465,8 +466,8 @@ func (ex *Remote) getRemoteFilesProperties(ctx context.Context, dir string, excl
 
 			if entry.IsDir() {
 				err := processEntry(ctx, client, root, excl, fullPath)
-				if err != nil {
-					log.Printf("[ERROR] failed to process directory %s: %v", fullPath, err)
+				if err != nil && err.Error() != "context canceled" {
+					log.Printf("[WARN] failed to process directory %s: %v", fullPath, err)
 				}
 				continue
 			}

pkg/runner/commands.go

@@ -162,6 +162,24 @@ func (ec *execCmd) Sync(ctx context.Context) (details string, vars map[string]st
 	return details, nil, nil
 }
 
+// Msync synchronizes multiple locations from a source to a destination on a target host.
+func (ec *execCmd) Msync(ctx context.Context) (details string, vars map[string]string, err error) {
+	msgs := []string{}
+	tmpl := templater{hostAddr: ec.hostAddr, hostName: ec.hostName, task: ec.tsk, command: ec.cmd.Name, env: ec.cmd.Environment}
+	for _, c := range ec.cmd.MSync {
+		src := tmpl.apply(c.Source)
+		dst := tmpl.apply(c.Dest)
+		msgs = append(msgs, fmt.Sprintf("%s -> %s", src, dst))
+		ecSingle := ec
+		ecSingle.cmd.Sync = config.SyncInternal{Source: src, Dest: dst, Exclude: c.Exclude, Delete: c.Delete}
+		if _, _, err := ecSingle.Sync(ctx); err != nil {
+			return details, nil, fmt.Errorf("can't sync %s to %s %s: %w", src, ec.hostAddr, dst, err)
+		}
+	}
+	details = fmt.Sprintf(" {sync: %s}", strings.Join(msgs, ", "))
+	return details, nil, nil
+}
+
 // Delete deletes files on a target host. If sudo option is set, it will execute a sudo rm commands.
 func (ec *execCmd) Delete(ctx context.Context) (details string, vars map[string]string, err error) {
 	tmpl := templater{hostAddr: ec.hostAddr, hostName: ec.hostName, task: ec.tsk, command: ec.cmd.Name, env: ec.cmd.Environment}

pkg/runner/commands_test.go

@@ -314,4 +314,28 @@ func Test_execCmd(t *testing.T) {
 		assert.Contains(t, details, "conf.yml")
 		assert.NotContains(t, details, "conf2.yml")
 	})
+
+	t.Run("msync command", func(t *testing.T) {
+		ec := execCmd{exec: sess, tsk: &config.Task{Name: "test"}, cmd: config.Cmd{MSync: []config.SyncInternal{
+			{Source: "testdata", Dest: "/tmp/sync.testdata_m1", Exclude: []string{"conf2.yml"}},
+			{Source: "testdata", Dest: "/tmp/sync.testdata_m2"},
+		}, Name: "test"}}
+		details, _, err := ec.Msync(ctx)
+		require.NoError(t, err)
+		assert.Equal(t, " {sync: testdata -> /tmp/sync.testdata_m1, testdata -> /tmp/sync.testdata_m2}", details)
+
+		ec = execCmd{exec: sess, tsk: &config.Task{Name: "test"}, cmd: config.Cmd{Echo: "$(ls -la /tmp/sync.testdata_m1)",
+			Name: "test"}}
+		details, _, err = ec.Echo(ctx)
+		require.NoError(t, err)
+		assert.Contains(t, details, "conf.yml")
+		assert.NotContains(t, details, "conf2.yml")
+
+		ec = execCmd{exec: sess, tsk: &config.Task{Name: "test"}, cmd: config.Cmd{Echo: "$(ls -la /tmp/sync.testdata_m2)",
+			Name: "test"}}
+		details, _, err = ec.Echo(ctx)
+		require.NoError(t, err)
+		assert.Contains(t, details, "conf.yml")
+		assert.Contains(t, details, "conf2.yml")
+	})
 }

pkg/runner/runner.go

@@ -184,8 +184,11 @@ func (p *Process) execCommand(ctx context.Context, ec execCmd) (details string,
 		log.Printf("[DEBUG] copy multiple files to %s", ec.hostAddr)
 		return ec.Mcopy(ctx)
 	case ec.cmd.Sync.Source != "" && ec.cmd.Sync.Dest != "":
-		log.Printf("[DEBUG] sync files on %s", ec.hostAddr)
+		log.Printf("[DEBUG] sync files to %s", ec.hostAddr)
 		return ec.Sync(ctx)
+	case len(ec.cmd.MSync) > 0:
+		log.Printf("[DEBUG] sync multiple locations to %s", ec.hostAddr)
+		return ec.Msync(ctx)
 	case ec.cmd.Delete.Location != "":
 		log.Printf("[DEBUG] delete files on %s", ec.hostAddr)
 		return ec.Delete(ctx)