Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use docker compose v2 by default in test-integration-docker-compatibility #1644

Merged
merged 1 commit into from
Jun 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmd/nerdctl/compose_down_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"testing"
"time"

"github.com/containerd/nerdctl/pkg/composer/serviceparser"
"github.com/containerd/nerdctl/pkg/testutil"
)

Expand Down Expand Up @@ -92,7 +93,7 @@ services:
projectName := compFull.ProjectName()
t.Logf("projectName=%q", projectName)

orphanContainer := fmt.Sprintf("%s_orphan_1", projectName)
orphanContainer := serviceparser.DefaultContainerName(projectName, "orphan", "1")

base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "up", "-d").AssertOK()
defer base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "down", "-v").Run()
Expand Down
4 changes: 4 additions & 0 deletions cmd/nerdctl/compose_kill_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ import (
)

func TestComposeKill(t *testing.T) {
// docker-compose v2 hides exited/killed containers in `compose ps`, and shows
// them if `-a` is passed, which is not supported yet by `nerdctl compose`.
testutil.DockerIncompatible(t)

base := testutil.NewBase(t)
var dockerComposeYAML = fmt.Sprintf(`
version: '3.1'
Expand Down
3 changes: 3 additions & 0 deletions cmd/nerdctl/compose_restart_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ import (
)

func TestComposeRestart(t *testing.T) {
// docker-compose v2 hides exited containers in `compose ps`, and shows
// them if `-a` is passed, which is not supported yet by `nerdctl compose`.
testutil.DockerIncompatible(t)
base := testutil.NewBase(t)
var dockerComposeYAML = fmt.Sprintf(`
version: '3.1'
Expand Down
10 changes: 10 additions & 0 deletions cmd/nerdctl/compose_run_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
Copy link
Member

@AkihiroSuda AkihiroSuda Jun 8, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run() (L52) enough?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It didn't work in docker (compose), e.g., containers from docker compose run (i.e., not from docker compose up) won't be covered by docker compose down:

$ cat docker-compose.yaml
version: '3.1'

services:
  svc0:
    image: nginx
  svc1:
    image: rabbitmq

# docker compose down cannot down container from docker compose run
$ docker compose run -d svc0
[+] Running 1/0
 ✔ Network tmp_default  Created                                                                                                                                                                                                                                                                                                       0.0s
103f148596f2d629f2b875d83da6ac5ea3682ad85b8c61d58e73ddc7931f3b11
$ docker compose down -v
[+] Running 1/0
 ✘ Network tmp_default  Error                                                                                                                                                                                                                                                                                                         0.0s
failed to remove network tmp_default: Error response from daemon: error while removing network: network tmp_default id 3795360bd3e1e9a11def2a76f140597f328009fb101bd7d67490bdeebb8b54f9 has active endpoints
$ docker ps
CONTAINER ID   IMAGE     COMMAND                  CREATED          STATUS          PORTS     NAMES
103f148596f2   nginx     "/docker-entrypoint.…"   24 seconds ago   Up 23 seconds   80/tcp    tmp-svc0-run-db8a05f1f3e6

# works fine if from docker compose up
$ docker compose up -d
[+] Running 2/2
 ✔ Container tmp-svc1-1  Started                                                                                                                                                                                                                                                                                                      0.2s
 ✔ Container tmp-svc0-1  Started                                                                                                                                                                                                                                                                                                      0.3s
$ docker compose down -v
[+] Running 3/2
 ✔ Container tmp-svc0-1  Removed                                                                                                                                                                                                                                                                                                      0.1s
 ✔ Container tmp-svc1-1  Removed                                                                                                                                                                                                                                                                                                      6.3s
 ✔ Network tmp_default   Removed

const sttyPartialOutput = "speed 38400 baud"
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -80,6 +81,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
const sttyPartialOutput = "speed 38400 baud"
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -119,6 +121,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
go func() {
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -178,6 +181,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
go func() {
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -241,6 +245,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
const partialOutput = "bar"
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -271,6 +276,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
const partialOutput = "5000"
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -301,6 +307,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
unbuffer := []string{"unbuffer"}
Expand Down Expand Up @@ -335,6 +342,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
const partialOutput = "hello world"
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -364,6 +372,7 @@ services:
t.Logf("projectName=%q", projectName)
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

defer base.Cmd("rm", "-f", "-v", containerName).Run()
const partialOutput = "hello world"
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
Expand Down Expand Up @@ -396,6 +405,7 @@ services:
destinationDir := "/data"
volumeFlagStr := fmt.Sprintf("%s:%s", tmpDir, destinationDir)

defer base.Cmd("rm", "-f", "-v", containerName).Run()
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
// unbuffer(1) can be installed with `apt-get install expect`.
unbuffer := []string{"unbuffer"}
Expand Down
3 changes: 3 additions & 0 deletions cmd/nerdctl/compose_stop_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ import (
)

func TestComposeStop(t *testing.T) {
// docker-compose v2 hides exited/killed containers in `compose ps`, and shows
// them if `-a` is passed, which is not supported yet by `nerdctl compose`.
testutil.DockerIncompatible(t)
base := testutil.NewBase(t)
var dockerComposeYAML = fmt.Sprintf(`
version: '3.1'
Expand Down
15 changes: 8 additions & 7 deletions cmd/nerdctl/compose_up_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"testing"
"time"

"github.com/containerd/nerdctl/pkg/composer/serviceparser"
"github.com/containerd/nerdctl/pkg/rootlessutil"
"github.com/docker/go-connections/nat"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -187,7 +188,7 @@ networks:
base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d").AssertOK()
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

svc0 := fmt.Sprintf("%s_svc0_1", projectName)
svc0 := serviceparser.DefaultContainerName(projectName, "svc0", "1")
inspectCmd := base.Cmd("inspect", svc0, "--format", "\"{{range .NetworkSettings.Networks}} {{.IPAddress}}{{end}}\"")
result := inspectCmd.Run()
stdoutContent := result.Stdout() + result.Stderr()
Expand Down Expand Up @@ -236,9 +237,9 @@ networks:
base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d").AssertOK()
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

svc0 := fmt.Sprintf("%s_svc0_1", projectName)
svc1 := fmt.Sprintf("%s_svc1_1", projectName)
svc2 := fmt.Sprintf("%s_svc2_1", projectName)
svc0 := serviceparser.DefaultContainerName(projectName, "svc0", "1")
svc1 := serviceparser.DefaultContainerName(projectName, "svc1", "1")
svc2 := serviceparser.DefaultContainerName(projectName, "svc2", "1")

base.Cmd("exec", svc0, "ping", "-c", "1", "svc0").AssertOK()
base.Cmd("exec", svc0, "ping", "-c", "1", "svc1").AssertOK()
Expand Down Expand Up @@ -348,7 +349,7 @@ services:
base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d", "--scale", "test=2").AssertOK()
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

base.ComposeCmd("-f", comp.YAMLFullPath(), "ps").AssertOutContains(fmt.Sprintf("%s_test_2", projectName))
base.ComposeCmd("-f", comp.YAMLFullPath(), "ps").AssertOutContains(serviceparser.DefaultContainerName(projectName, "test", "2"))
}

func TestComposeIPAMConfig(t *testing.T) {
Expand Down Expand Up @@ -377,7 +378,7 @@ networks:
base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d").AssertOK()
defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()

base.Cmd("inspect", "-f", `{{json .NetworkSettings.Networks }}`, projectName+"_foo_1").AssertOutContains("10.1.100.")
base.Cmd("inspect", "-f", `{{json .NetworkSettings.Networks }}`, serviceparser.DefaultContainerName(projectName, "foo", "1")).AssertOutContains("10.1.100.")
}

func TestComposeUpRemoveOrphans(t *testing.T) {
Expand Down Expand Up @@ -409,7 +410,7 @@ services:
projectName := fmt.Sprintf("nerdctl-compose-test-%d", time.Now().Unix())
t.Logf("projectName=%q", projectName)

orphanContainer := fmt.Sprintf("%s_orphan_1", projectName)
orphanContainer := serviceparser.DefaultContainerName(projectName, "orphan", "1")

base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "up", "-d").AssertOK()
defer base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "down", "-v").Run()
Expand Down
2 changes: 0 additions & 2 deletions cmd/nerdctl/compose_version_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

func TestComposeVersion(t *testing.T) {
base := testutil.NewBase(t)
base.ComposeBinary = "" // test with docker compose version, not docker-compose version
base.ComposeCmd("version").AssertOutContains("Compose version ")
}

Expand All @@ -35,6 +34,5 @@ func TestComposeVersionShort(t *testing.T) {

func TestComposeVersionJson(t *testing.T) {
base := testutil.NewBase(t)
base.ComposeBinary = "" // test with docker compose version, not docker-compose version
base.ComposeCmd("version", "--format", "json").AssertOutContains("{\"version\":\"")
}
3 changes: 2 additions & 1 deletion pkg/composer/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/compose-spec/compose-go/types"
"github.com/containerd/containerd"
"github.com/containerd/nerdctl/pkg/composer/pipetagger"
"github.com/containerd/nerdctl/pkg/composer/serviceparser"
"github.com/containerd/nerdctl/pkg/labels"

"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -70,7 +71,7 @@ func (c *Composer) logs(ctx context.Context, containers []containerd.Container,
return err
}
name := info.Labels[labels.Name]
logTag := strings.TrimPrefix(name, c.project.Name+"_")
logTag := strings.TrimPrefix(name, c.project.Name+serviceparser.Separator)
if l := len(logTag); l > logTagMaxLen {
logTagMaxLen = l
}
Expand Down
3 changes: 2 additions & 1 deletion pkg/composer/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ func (c *Composer) Run(ctx context.Context, ro RunOptions) error {
}

for i := range svcs {
svcs[i].ContainerName = fmt.Sprintf("%s_%s_run_%s", c.project.Name, svcs[i].Name, idgen.TruncateID(idgen.GenerateID()))
// FYI: https://github.com/docker/compose/blob/v2.18.1/pkg/compose/run.go#L65
svcs[i].ContainerName = fmt.Sprintf("%[1]s%[4]s%[2]s%[4]srun%[4]s%[3]s", c.project.Name, svcs[i].Name, idgen.TruncateID(idgen.GenerateID()), serviceparser.Separator)
}

targetSvc.Tty = ro.Tty
Expand Down
2 changes: 1 addition & 1 deletion pkg/composer/serviceparser/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ secrets:
assert.NilError(t, err)

t.Logf("foo: %+v", foo)
assert.Equal(t, project.Name+"_foo", foo.Image)
assert.Equal(t, DefaultImageName(project.Name, "foo"), foo.Image)
assert.Equal(t, false, foo.Build.Force)
assert.Equal(t, project.RelativePath("fooctx"), lastOf(foo.Build.BuildArgs))

Expand Down
19 changes: 17 additions & 2 deletions pkg/composer/serviceparser/serviceparser.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"

Expand All @@ -47,6 +48,10 @@ const (
ComposeCosignCertificateOidcIssuerRegexp = "x-nerdctl-cosign-certificate-oidc-issuer-regexp"
)

// Separator is used for naming components (e.g., service image or container)
// https://github.com/docker/compose/blob/8c39b5b7fd4210a69d07885835f7ff826aaa1cd8/pkg/api/api.go#L483
const Separator = "-"

func warnUnknownFields(svc types.ServiceConfig) {
if unknown := reflectutil.UnknownNonEmptyFields(&svc,
"Name",
Expand Down Expand Up @@ -423,7 +428,7 @@ func Parse(project *types.Project, svc types.ServiceConfig) (*Service, error) {
}
} else {
if parsed.Image == "" {
parsed.Image = fmt.Sprintf("%s_%s", project.Name, svc.Name)
parsed.Image = DefaultImageName(project.Name, svc.Name)
}
parsed.Build, err = parseBuildConfig(svc.Build, project, parsed.Image)
if err != nil {
Expand Down Expand Up @@ -460,7 +465,7 @@ func Parse(project *types.Project, svc types.ServiceConfig) (*Service, error) {
func newContainer(project *types.Project, parsed *Service, i int) (*Container, error) {
svc := *parsed.Unparsed
var c Container
c.Name = fmt.Sprintf("%s_%s_%d", project.Name, svc.Name, i+1)
c.Name = DefaultContainerName(project.Name, svc.Name, strconv.Itoa(i+1))
if svc.ContainerName != "" {
if i != 0 {
return nil, errors.New("container_name must not be specified when replicas != 1")
Expand Down Expand Up @@ -884,3 +889,13 @@ func fileReferenceConfigToFlagV(c types.FileReferenceConfig, project *types.Proj
s := fmt.Sprintf("%s:%s:ro", src, target)
return s, nil
}

// DefaultImageName returns the image name following compose naming logic.
func DefaultImageName(projectName string, serviceName string) string {
return projectName + Separator + serviceName
}

// DefaultContainerName returns the service container name following compose naming logic.
func DefaultContainerName(projectName, serviceName, suffix string) string {
return DefaultImageName(projectName, serviceName) + Separator + suffix
}
9 changes: 5 additions & 4 deletions pkg/composer/serviceparser/serviceparser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"testing"

"github.com/compose-spec/compose-go/types"
Expand Down Expand Up @@ -151,7 +152,7 @@ volumes:
assert.Assert(t, wp.Image == "wordpress:5.7")
assert.Assert(t, len(wp.Containers) == 1)
wp1 := wp.Containers[0]
assert.Assert(t, wp1.Name == fmt.Sprintf("%s_wordpress_1", project.Name))
assert.Assert(t, wp1.Name == DefaultContainerName(project.Name, "wordpress", "1"))
assert.Assert(t, in(wp1.RunArgs, "--name="+wp1.Name))
assert.Assert(t, in(wp1.RunArgs, "--hostname=wordpress"))
assert.Assert(t, in(wp1.RunArgs, fmt.Sprintf("--net=%s_default", project.Name)))
Expand Down Expand Up @@ -183,7 +184,7 @@ volumes:
t.Logf("db: %+v", db)
assert.Assert(t, len(db.Containers) == 1)
db1 := db.Containers[0]
assert.Assert(t, db1.Name == fmt.Sprintf("%s_db_1", project.Name))
assert.Assert(t, db1.Name == DefaultContainerName(project.Name, "db", "1"))
assert.Assert(t, in(db1.RunArgs, "--hostname=db"))
assert.Assert(t, in(db1.RunArgs, fmt.Sprintf("-v=%s_db:/var/lib/mysql", project.Name)))
assert.Assert(t, in(db1.RunArgs, "--stop-signal=SIGUSR1"))
Expand Down Expand Up @@ -218,7 +219,7 @@ services:
t.Logf("foo: %+v", foo)
assert.Assert(t, len(foo.Containers) == 2)
for i, c := range foo.Containers {
assert.Assert(t, c.Name == fmt.Sprintf("%s_foo_%d", project.Name, i+1))
assert.Assert(t, c.Name == DefaultContainerName(project.Name, "foo", strconv.Itoa(i+1)))
assert.Assert(t, in(c.RunArgs, "--name="+c.Name))
assert.Assert(t, in(c.RunArgs, fmt.Sprintf("--cpus=%f", 0.42)))
assert.Assert(t, in(c.RunArgs, "-m=44040192"))
Expand Down Expand Up @@ -275,7 +276,7 @@ services:
t.Logf("foo: %+v", foo)
assert.Assert(t, len(foo.Containers) == 3)
for i, c := range foo.Containers {
assert.Assert(t, c.Name == fmt.Sprintf("%s_foo_%d", project.Name, i+1))
assert.Assert(t, c.Name == DefaultContainerName(project.Name, "foo", strconv.Itoa(i+1)))
assert.Assert(t, in(c.RunArgs, "--name="+c.Name))

assert.Assert(t, in(c.RunArgs, "--restart=no"))
Expand Down
32 changes: 6 additions & 26 deletions pkg/testutil/testutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ type Base struct {
Target Target
DaemonIsKillable bool
Binary string
ComposeBinary string // "docker-compose"
Args []string
Env []string
}
Expand All @@ -72,17 +71,8 @@ func (b *Base) Cmd(args ...string) *Cmd {

// ComposeCmd executes `nerdctl -n nerdctl-test compose` or `docker-compose`
func (b *Base) ComposeCmd(args ...string) *Cmd {
var (
binary string
binaryArgs []string
)
if b.ComposeBinary != "" {
binary = b.ComposeBinary
binaryArgs = append(b.Args, args...)
} else {
binary = b.Binary
binaryArgs = append(b.Args, append([]string{"compose"}, args...)...)
}
binary := b.Binary
binaryArgs := append(b.Args, append([]string{"compose"}, args...)...)
icmdCmd := icmd.Command(binary, binaryArgs...)
icmdCmd.Env = b.Env
cmd := &Cmd{
Expand All @@ -97,18 +87,10 @@ func (b *Base) ComposeCmdWithHelper(helper []string, args ...string) *Cmd {
if err != nil {
b.T.Skipf("helper binary %q not found", helper[0])
}
binary := b.Binary
binaryArgs := append(b.Args, append([]string{"compose"}, args...)...)

helperArgs := helper[1:]
var (
binary string
binaryArgs []string
)
if b.ComposeBinary != "" {
binary = b.ComposeBinary
binaryArgs = append(b.Args, args...)
} else {
binary = b.Binary
binaryArgs = append(b.Args, append([]string{"compose"}, args...)...)
}
helperArgs = append(helperArgs, binary)
helperArgs = append(helperArgs, binaryArgs...)
icmdCmd := icmd.Command(helperBin, helperArgs...)
Expand Down Expand Up @@ -657,14 +639,12 @@ func newBase(t *testing.T, ns string) *Base {
t.Fatal(err)
}
base.Args = []string{"--namespace=" + ns}
base.ComposeBinary = ""
case Docker:
base.Binary, err = exec.LookPath("docker")
if err != nil {
t.Fatal(err)
}
base.ComposeBinary, err = exec.LookPath("docker-compose")
if err != nil {
if err := exec.Command("docker", "compose", "version").Run(); err != nil {
t.Fatal(err)
}
default:
Expand Down