refactor compose and support multi compose files

Signed-off-by: ye.sijun <junnplus@gmail.com>

cmd/nerdctl/compose.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"errors"
 
-	composecli "github.com/compose-spec/compose-go/cli"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/platforms"
@@ -45,7 +44,7 @@ func newComposeCommand() *cobra.Command {
 		TraverseChildren: true, // required for global short hands like -f
 	}
 	// `-f` is a nonPersistentAlias, as it conflicts with `nerdctl compose logs --follow`
-	AddPersistentStringFlag(composeCommand, "file", nil, []string{"f"}, "", "", "Specify an alternate compose file")
+	AddPersistentStringArrayFlag(composeCommand, "file", nil, []string{"f"}, nil, "", "Specify an alternate compose file")
 	composeCommand.PersistentFlags().String("project-directory", "", "Specify an alternate working directory")
 	composeCommand.PersistentFlags().StringP("project-name", "p", "", "Specify an alternate project name")
 	composeCommand.PersistentFlags().String("env-file", "", "Specify an alternate environment file")
@@ -83,7 +82,7 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 	if err != nil {
 		return nil, err
 	}
-	file, err := cmd.Flags().GetString("file")
+	files, err := cmd.Flags().GetStringArray("file")
 	if err != nil {
 		return nil, err
 	}
@@ -109,21 +108,15 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 	}
 
 	o := composer.Options{
-		ProjectOptions: composecli.ProjectOptions{
-			WorkingDir:  projectDirectory,
-			ConfigPaths: []string{},
-			Environment: map[string]string{},
-			EnvFile:     envFile,
-		},
-		Project:        projectName,
-		NerdctlCmd:     nerdctlCmd,
-		NerdctlArgs:    nerdctlArgs,
-		DebugPrintFull: debugFull,
+		Project:          projectName,
+		ProjectDirectory: projectDirectory,
+		ConfigPaths:      files,
+		EnvFile:          envFile,
+		NerdctlCmd:       nerdctlCmd,
+		NerdctlArgs:      nerdctlArgs,
+		DebugPrintFull:   debugFull,
 	}
 
-	if file != "" {
-		o.ProjectOptions.ConfigPaths = append([]string{file}, o.ProjectOptions.ConfigPaths...)
-	}
 	cniEnv := &netutil.CNIEnv{
 		Path:        cniPath,
 		NetconfPath: cniNetconfpath,

cmd/nerdctl/compose_config_test.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
 
 	"github.com/containerd/nerdctl/pkg/testutil"
@@ -75,3 +76,31 @@ services:
 	newHash := base.ComposeCmd("-f", newComp.YAMLFullPath(), "config", "--hash=hello1").Out()
 	assert.Assert(t, hash != newHash)
 }
+
+func TestComposeConfigWithMultipleFile(t *testing.T) {
+	base := testutil.NewBase(t)
+
+	var dockerComposeYAML = `
+services:
+  hello1:
+    image: alpine:3.13
+`
+
+	comp := testutil.NewComposeDir(t, dockerComposeYAML)
+	defer comp.CleanUp()
+
+	comp.WriteFile("docker-compose.test.yml", `
+services:
+  hello2:
+    image: alpine:3.14
+`)
+	comp.WriteFile("docker-compose.override.yml", `
+services:
+  hello1:
+    image: alpine:3.14
+`)
+
+	base.ComposeCmd("-f", comp.YAMLFullPath(), "-f", filepath.Join(comp.Dir(), "docker-compose.test.yml"), "config").AssertOutContains("alpine:3.14")
+	base.ComposeCmd("--project-directory", comp.Dir(), "config", "--services").AssertOutExactly("hello1\n")
+	base.ComposeCmd("--project-directory", comp.Dir(), "config").AssertOutContains("alpine:3.14")
+}

cmd/nerdctl/main.go

@@ -413,3 +413,36 @@ func AddPersistentStringFlag(cmd *cobra.Command, name string, aliases, nonPersis
 		}
 	}
 }
+
+// AddPersistentStringArrayFlag is similar to cmd.Flags().StringArray but supports aliases and env var and persistent.
+// See https://github.com/spf13/cobra/blob/master/user_guide.md#persistent-flags to learn what is "persistent".
+func AddPersistentStringArrayFlag(cmd *cobra.Command, name string, aliases, nonPersistentAliases []string, value []string, env string, usage string) {
+	if env != "" {
+		usage = fmt.Sprintf("%s [$%s]", usage, env)
+	}
+	if envV, ok := os.LookupEnv(env); ok {
+		value = []string{envV}
+	}
+	aliasesUsage := fmt.Sprintf("Alias of --%s", name)
+	p := new([]string)
+	flags := cmd.Flags()
+	for _, a := range nonPersistentAliases {
+		if len(a) == 1 {
+			// pflag doesn't support short-only flags, so we have to register long one as well here
+			flags.StringArrayVarP(p, a, a, value, aliasesUsage)
+		} else {
+			flags.StringArrayVar(p, a, value, aliasesUsage)
+		}
+	}
+
+	persistentFlags := cmd.PersistentFlags()
+	persistentFlags.StringArrayVar(p, name, value, usage)
+	for _, a := range aliases {
+		if len(a) == 1 {
+			// pflag doesn't support short-only flags, so we have to register long one as well here
+			persistentFlags.StringArrayVarP(p, a, a, value, aliasesUsage)
+		} else {
+			persistentFlags.StringArrayVar(p, a, value, aliasesUsage)
+		}
+	}
+}

pkg/composer/composer.go

@@ -21,15 +21,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
 	"os/exec"
-	"path/filepath"
 
 	composecli "github.com/compose-spec/compose-go/cli"
 	compose "github.com/compose-spec/compose-go/types"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/identifiers"
-	"github.com/containerd/nerdctl/pkg/composer/projectloader"
 	"github.com/containerd/nerdctl/pkg/composer/serviceparser"
 	"github.com/containerd/nerdctl/pkg/reflectutil"
 
@@ -38,16 +35,17 @@ import (
 
 // Options groups the command line options recommended for a Compose implementation (ProjectOptions) and extra options for nerdctl
 type Options struct {
-	//TODO Specifying multiple Compose files
-	ProjectOptions composecli.ProjectOptions
-	Project        string // empty for default
-	NerdctlCmd     string
-	NerdctlArgs    []string
-	NetworkExists  func(string) (bool, error)
-	VolumeExists   func(string) (bool, error)
-	ImageExists    func(ctx context.Context, imageName string) (bool, error)
-	EnsureImage    func(ctx context.Context, imageName, pullMode, platform string, quiet bool) error
-	DebugPrintFull bool // full debug print, may leak secret env var to logs
+	Project          string // empty for default
+	ProjectDirectory string
+	ConfigPaths      []string
+	EnvFile          string
+	NerdctlCmd       string
+	NerdctlArgs      []string
+	NetworkExists    func(string) (bool, error)
+	VolumeExists     func(string) (bool, error)
+	ImageExists      func(ctx context.Context, imageName string) (bool, error)
+	EnsureImage      func(ctx context.Context, imageName, pullMode, platform string, quiet bool) error
+	DebugPrintFull   bool // full debug print, may leak secret env var to logs
 }
 
 func New(o Options, client *containerd.Client) (*Composer, error) {
@@ -57,41 +55,28 @@ func New(o Options, client *containerd.Client) (*Composer, error) {
 	if o.NetworkExists == nil || o.VolumeExists == nil || o.EnsureImage == nil {
 		return nil, errors.New("got empty functions")
 	}
-	// actually we support single ConfigPath
-	// TODO support multiple ConfigPaths
-	var err error
-	if len(o.ProjectOptions.ConfigPaths) == 0 {
-		composeYaml, err := findComposeYAML(&o)
-		if err != nil {
-			return nil, err
-		}
-		o.ProjectOptions.ConfigPaths = append(o.ProjectOptions.ConfigPaths, composeYaml)
-	}
-
-	if err := composecli.WithOsEnv(&o.ProjectOptions); err != nil {
-		return nil, err
-	}
 
-	if err := composecli.WithDotEnv(&o.ProjectOptions); err != nil {
-		return nil, err
-	}
-
-	for i := 0; i < len(o.ProjectOptions.ConfigPaths); i++ {
-		o.ProjectOptions.ConfigPaths[i], err = filepath.Abs(o.ProjectOptions.ConfigPaths[i])
-		if err != nil {
-			return nil, err
+	if o.Project != "" {
+		if err := identifiers.Validate(o.Project); err != nil {
+			return nil, fmt.Errorf("got invalid project name %q: %w", o.Project, err)
 		}
 	}
 
-	if o.Project == "" {
-		o.Project = filepath.Base(filepath.Dir(o.ProjectOptions.ConfigPaths[0]))
-	}
+	var optionsFn []composecli.ProjectOptionsFn
+	optionsFn = append(optionsFn,
+		composecli.WithWorkingDirectory(o.ProjectDirecotry),
+		composecli.WithEnvFile(o.EnvFile),
+		composecli.WithDefaultConfigPath,
+		composecli.WithOsEnv,
+		composecli.WithDotEnv,
+		composecli.WithName(o.Project),
+	)
 
-	if err := identifiers.Validate(o.Project); err != nil {
-		return nil, fmt.Errorf("got invalid project name %q: %w", o.Project, err)
+	projectOptions, err := composecli.NewProjectOptions(o.ConfigPaths, optionsFn...)
+	if err != nil {
+		return nil, err
 	}
-
-	project, err := projectloader.Load(o.ProjectOptions.ConfigPaths[0], o.Project, o.ProjectOptions.Environment)
+	project, err := composecli.ProjectFromOptions(projectOptions)
 	if err != nil {
 		return nil, err
 	}
@@ -145,30 +130,6 @@ func (c *Composer) runNerdctlCmd(ctx context.Context, args ...string) error {
 	return nil
 }
 
-//find compose Yaml file in current directory and its parents
-func findComposeYAML(o *Options) (string, error) {
-	pwd, err := o.ProjectOptions.GetWorkingDir()
-	if err != nil {
-		return "", err
-	}
-	for {
-		yamlNames := []string{"docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"}
-		for _, candidate := range yamlNames {
-			f := filepath.Join(pwd, candidate)
-			if _, err := os.Stat(f); err == nil {
-				return f, nil
-			} else if !os.IsNotExist(err) {
-				return "", err
-			}
-		}
-		parent := filepath.Dir(pwd)
-		if parent == pwd {
-			return "", fmt.Errorf("cannot find a compose YAML, supported file names: %+v in this directory or any parent", yamlNames)
-		}
-		pwd = parent
-	}
-}
-
 func (c *Composer) Services(ctx context.Context) ([]*serviceparser.Service, error) {
 	var services []*serviceparser.Service
 	if err := c.project.WithServices(nil, func(svc compose.ServiceConfig) error {

pkg/composer/up_network.go

@@ -49,7 +49,7 @@ func (c *Composer) upNetwork(ctx context.Context, shortName string) error {
 		logrus.Infof("Creating network %s", fullName)
 		//add metadata labels to network https://github.com/compose-spec/compose-spec/blob/master/spec.md#labels-1
 		createArgs := []string{
-			fmt.Sprintf("--label=%s=%s", labels.ComposeProject, c.Options.Project),
+			fmt.Sprintf("--label=%s=%s", labels.ComposeProject, c.project.Name),
 			fmt.Sprintf("--label=%s=%s", labels.ComposeNetwork, shortName),
 		}
 

pkg/composer/up_service.go

@@ -148,7 +148,7 @@ func (c *Composer) upServiceContainer(ctx context.Context, service *serviceparse
 
 	//add metadata labels to container https://github.com/compose-spec/compose-spec/blob/master/spec.md#labels
 	container.RunArgs = append([]string{
-		fmt.Sprintf("-l=%s=%s", labels.ComposeProject, c.Options.Project),
+		fmt.Sprintf("-l=%s=%s", labels.ComposeProject, c.project.Name),
 		fmt.Sprintf("-l=%s=%s", labels.ComposeService, service.Unparsed.Name),
 	}, container.RunArgs...)
 

pkg/composer/up_volume.go

@@ -49,7 +49,7 @@ func (c *Composer) upVolume(ctx context.Context, shortName string) error {
 		logrus.Infof("Creating volume %s", fullName)
 		//add metadata labels to volume https://github.com/compose-spec/compose-spec/blob/master/spec.md#labels-2
 		createArgs := []string{
-			fmt.Sprintf("--label=%s=%s", labels.ComposeProject, c.Options.Project),
+			fmt.Sprintf("--label=%s=%s", labels.ComposeProject, c.project.Name),
 			fmt.Sprintf("--label=%s=%s", labels.ComposeVolume, shortName),
 			fullName,
 		}

pkg/testutil/compose.go

@@ -38,6 +38,10 @@ func (cd *ComposeDir) YAMLFullPath() string {
 	return filepath.Join(cd.dir, cd.yamlBasePath)
 }
 
+func (cd *ComposeDir) Dir() string {
+	return cd.dir
+}
+
 func (cd *ComposeDir) ProjectName() string {
 	return filepath.Base(cd.dir)
 }