From 5f57d8e010ce4c6f05da02a226360f249ac93fed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Carvalho?= Date: Tue, 10 Apr 2018 15:26:33 -0300 Subject: [PATCH] all: adds image deploy support for kubernetes This commit adds support for sidecar image deployment. When SOURCE_IMAGE is provided, the sidecar will inspect the given image and also a potential Procfile and tsuru.yaml on the primary container. This is sent to stdout so tsuru API can parse and process them. The primary container is commited and it's image is pushed. --- deploy.go | 41 +++++++++++++- integration_test.go | 88 +++++++++++++++++++++++++++++ internal/docker/docker.go | 10 ++++ internal/docker/docker_test.go | 12 ++++ internal/docker/integration_test.go | 80 +++----------------------- internal/docker/testing/sidecar.go | 70 +++++++++++++++++++++++ main.go | 40 ++++++++----- 7 files changed, 252 insertions(+), 89 deletions(-) create mode 100644 integration_test.go create mode 100644 internal/docker/testing/sidecar.go diff --git a/deploy.go b/deploy.go index 82421af..da21ddf 100644 --- a/deploy.go +++ b/deploy.go @@ -6,6 +6,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "os" @@ -64,7 +65,7 @@ func setupSidecar(dockerClient *docker.Client, config Config) (*docker.Sidecar, } err = sideCar.UploadToPrimaryContainer(context.Background(), config.InputFile) if err != nil { - fatal("failed to upload input file: %v", err) + fatalf("failed to upload input file: %v", err) } return sideCar, nil } @@ -79,6 +80,10 @@ func pushSidecar(dockerClient *docker.Client, sideCar *docker.Sidecar, config Co if err != nil { return fmt.Errorf("failed to commit main container: %v", err) } + return tagAndPushDestinations(dockerClient, imgID, config, w) +} + +func tagAndPushDestinations(dockerClient *docker.Client, srcImgID string, config Config, w io.Writer) error { authConfig := docker.AuthConfig{ Username: config.RegistryAuthUser, Password: config.RegistryAuthPass, @@ -86,7 +91,7 @@ func pushSidecar(dockerClient *docker.Client, sideCar *docker.Sidecar, config Co ServerAddress: config.RegistryAddress, } for _, destImg := range config.DestinationImages { - if err := tagAndPush(dockerClient, imgID, destImg, authConfig, config.RegistryPushRetries, w); err != nil { + if err := tagAndPush(dockerClient, srcImgID, destImg, authConfig, config.RegistryPushRetries, w); err != nil { return err } } @@ -113,3 +118,35 @@ func tagAndPush(dockerClient *docker.Client, imgID, imageName string, auth docke } return nil } + +func inspect(dockerClient *docker.Client, image string, filesystem Filesystem, w io.Writer, errW io.Writer) error { + imgInspect, err := dockerClient.Inspect(context.Background(), image) + if err != nil { + return fmt.Errorf("failed to inspect image %q: %v", image, err) + } + tsuruYaml, err := loadTsuruYaml(filesystem) + if err != nil { + return fmt.Errorf("failed to load tsuru yaml: %v", err) + } + procfileDirs := []string{defaultWorkingDir, "/app/user", ""} + var procfile string + for _, d := range procfileDirs { + procfile, err = readProcfile(d, filesystem) + if err != nil { + // we can safely ignore this error since tsuru may use the image CMD/Entrypoint + fmt.Fprintf(errW, "Unable to read procfile in %v: %v", d, err) + continue + } + break + } + m := map[string]interface{}{ + "image": imgInspect, + "tsuruYaml": tsuruYaml, + "procfile": procfile, + } + err = json.NewEncoder(w).Encode(m) + if err != nil { + return fmt.Errorf("failed to encode inspected data: %v", err) + } + return nil +} diff --git a/integration_test.go b/integration_test.go new file mode 100644 index 0000000..60b4e06 --- /dev/null +++ b/integration_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + + "github.com/tsuru/deploy-agent/internal/docker" + "github.com/tsuru/deploy-agent/internal/docker/testing" + "github.com/tsuru/deploy-agent/internal/tsuru" + "github.com/tsuru/tsuru/exec" + "gopkg.in/check.v1" +) + +const primaryImage = "tsuru/base-platform" + +func checkSkip(c *check.C) { + if os.Getenv("DEPLOYAGENT_INTEGRATION") == "" { + c.Skip("skipping integration tests") + } +} + +func (s *S) TestInspect(c *check.C) { + checkSkip(c) + + dClient, err := docker.NewClient("") + c.Assert(err, check.IsNil) + + _, cleanup, err := testing.SetupPrimaryContainer(c) + defer cleanup() + + sidecar, err := docker.NewSidecar(dClient, "root") + c.Assert(err, check.IsNil) + + outW := new(bytes.Buffer) + errW := new(bytes.Buffer) + + yamlData := ` +hooks: + build: + - ps +` + + err = sidecar.Execute(exec.ExecuteOptions{ + Cmd: "/bin/sh", + Args: []string{"-c", fmt.Sprintf("mkdir -p /home/application/current/ && echo '%s' > /home/application/current/tsuru.yaml", yamlData)}, + Stdout: outW, + Stderr: errW, + }) + c.Assert(err, check.IsNil) + c.Assert(outW.String(), check.DeepEquals, "") + c.Assert(errW.String(), check.DeepEquals, "") + + for _, loc := range []string{"/", "/app/user/", "/home/application/current/"} { + outW.Reset() + errW.Reset() + + err = sidecar.ExecuteAsUser("root", exec.ExecuteOptions{ + Cmd: "/bin/sh", + Args: []string{"-c", fmt.Sprintf(`mkdir -p %s && echo '%s' > %sProcfile`, loc, loc, loc)}, + Stdout: outW, + Stderr: errW, + }) + c.Assert(err, check.IsNil) + c.Assert(outW.String(), check.DeepEquals, "") + c.Assert(errW.String(), check.DeepEquals, "") + + outW.Reset() + errW.Reset() + + err = inspect(dClient, "tsuru/base-platform", &executorFS{executor: sidecar}, outW, errW) + c.Assert(err, check.IsNil) + + m := struct { + Procfile string + TsuruYaml tsuru.TsuruYaml + Image docker.ImageInspect + }{} + err = json.NewDecoder(outW).Decode(&m) + c.Assert(err, check.IsNil) + c.Assert(outW.String(), check.DeepEquals, "") + c.Assert(m.Procfile, check.DeepEquals, loc+"\n") + c.Assert(m.TsuruYaml, check.DeepEquals, tsuru.TsuruYaml{Hooks: tsuru.Hook{BuildHooks: []string{"ps"}}}) + c.Assert(m.Image.ID, check.Not(check.DeepEquals), "") + } + +} diff --git a/internal/docker/docker.go b/internal/docker/docker.go index 5b246ef..3ce3106 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -38,6 +38,8 @@ type Image struct { tag string } +type ImageInspect docker.Image + func (i Image) Name() string { return i.registry + "/" + i.repository } @@ -141,6 +143,14 @@ func (c *Client) Upload(ctx context.Context, containerID, path string, inputStre return c.api.UploadToContainer(containerID, opts) } +func (c *Client) Inspect(ctx context.Context, img string) (ImageInspect, error) { + inspect, err := c.api.InspectImage(img) + if err != nil { + return ImageInspect{}, err + } + return ImageInspect(*inspect), err +} + func splitImageName(imageName string) (registry, repo, tag string) { imgNameSplit := strings.Split(imageName, ":") switch len(imgNameSplit) { diff --git a/internal/docker/docker_test.go b/internal/docker/docker_test.go index cd63cf9..b521c03 100644 --- a/internal/docker/docker_test.go +++ b/internal/docker/docker_test.go @@ -183,3 +183,15 @@ func (s *S) TestClientUpload(c *check.C) { err = client.api.DownloadFromContainer(contID, docker.DownloadFromContainerOptions{Path: "/myfile.txt"}) c.Assert(err, check.IsNil) } + +func (s *S) TestInspect(c *check.C) { + client, err := NewClient(s.dockerserver.URL()) + c.Assert(err, check.IsNil) + + err = client.api.PullImage(docker.PullImageOptions{Repository: "image"}, docker.AuthConfiguration{}) + c.Assert(err, check.IsNil) + + img, err := client.Inspect(context.Background(), "image") + c.Assert(err, check.IsNil) + c.Assert(img.ID, check.Not(check.DeepEquals), "") +} diff --git a/internal/docker/integration_test.go b/internal/docker/integration_test.go index 24c4498..49a9a59 100644 --- a/internal/docker/integration_test.go +++ b/internal/docker/integration_test.go @@ -3,17 +3,13 @@ package docker import ( "bytes" "context" - "fmt" "os" - "time" - "github.com/fsouza/go-dockerclient" + "github.com/tsuru/deploy-agent/internal/docker/testing" "github.com/tsuru/tsuru/exec" "gopkg.in/check.v1" ) -const primaryImage = "tsuru/base-platform" - func checkSkip(c *check.C) { if os.Getenv("DEPLOYAGENT_INTEGRATION") == "" { c.Skip("skipping integration tests") @@ -26,12 +22,8 @@ func (s *S) TestSidecarUploadToPrimaryContainerIntegration(c *check.C) { dClient, err := NewClient("") c.Assert(err, check.IsNil) - pContID, err := setupPrimaryContainer(c, dClient) - defer func(id string) { - if id != "" { - dClient.api.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true}) - } - }(pContID) + _, cleanup, err := testing.SetupPrimaryContainer(c) + defer cleanup() c.Assert(err, check.IsNil) sidecar, err := NewSidecar(dClient, "") @@ -60,12 +52,8 @@ func (s *S) TestSidecarExecuteIntegration(c *check.C) { dClient, err := NewClient("") c.Assert(err, check.IsNil) - pContID, err := setupPrimaryContainer(c, dClient) - defer func(id string) { - if id != "" { - dClient.api.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true}) - } - }(pContID) + _, cleanup, err := testing.SetupPrimaryContainer(c) + defer cleanup() c.Assert(err, check.IsNil) sidecar, err := NewSidecar(dClient, "") @@ -162,12 +150,8 @@ func (s *S) TestSidecarExecuteAsUserIntegration(c *check.C) { dClient, err := NewClient("") c.Assert(err, check.IsNil) - pContID, err := setupPrimaryContainer(c, dClient) - defer func(id string) { - if id != "" { - dClient.api.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true}) - } - }(pContID) + _, cleanup, err := testing.SetupPrimaryContainer(c) + defer cleanup() c.Assert(err, check.IsNil) sidecar, err := NewSidecar(dClient, "") @@ -197,53 +181,3 @@ func (s *S) TestSidecarExecuteAsUserIntegration(c *check.C) { c.Check(out, check.DeepEquals, t.expectedOutput, check.Commentf("[%v] unexpected output. Err output: %v", t.user, errOutput)) } } - -func setupPrimaryContainer(c *check.C, dClient *Client) (string, error) { - hostname, err := os.Hostname() - if err != nil { - return "", fmt.Errorf("error getting hostname: %v", err) - } - - err = dClient.api.PullImage(docker.PullImageOptions{Repository: primaryImage}, docker.AuthConfiguration{}) - if err != nil { - return "", fmt.Errorf("error pulling image %v: %v", primaryImage, err) - } - - pCont, err := dClient.api.CreateContainer(docker.CreateContainerOptions{ - Config: &docker.Config{ - Image: primaryImage, - Cmd: []string{"/bin/sh", "-lc", "while true; do sleep 10; done"}, - Labels: map[string]string{ - "io.kubernetes.container.name": hostname, - "io.kubernetes.pod.name": hostname, - }, - }, - }) - - if err != nil { - return "", fmt.Errorf("error creating primary container: %v", err) - } - - err = dClient.api.StartContainer(pCont.ID, nil) - if err != nil { - return pCont.ID, fmt.Errorf("error starting primary container: %v", err) - } - - timeout := time.After(time.Second * 15) - for { - c, err := dClient.api.InspectContainer(pCont.ID) - if err != nil { - return pCont.ID, fmt.Errorf("error inspecting primary container: %v", err) - } - if c.State.Running { - break - } - select { - case <-time.After(time.Second): - case <-timeout: - return pCont.ID, fmt.Errorf("timeout waiting for primary container to run") - } - } - - return pCont.ID, nil -} diff --git a/internal/docker/testing/sidecar.go b/internal/docker/testing/sidecar.go new file mode 100644 index 0000000..5e2c9c2 --- /dev/null +++ b/internal/docker/testing/sidecar.go @@ -0,0 +1,70 @@ +package testing + +import ( + "fmt" + "os" + "time" + + "github.com/fsouza/go-dockerclient" + "gopkg.in/check.v1" +) + +const primaryImage = "tsuru/base-platform" + +func SetupPrimaryContainer(c *check.C) (string, func(), error) { + dClient, err := docker.NewClient("unix:///var/run/docker.sock") + if err != nil { + return "", nil, err + } + hostname, err := os.Hostname() + if err != nil { + return "", nil, fmt.Errorf("error getting hostname: %v", err) + } + + err = dClient.PullImage(docker.PullImageOptions{Repository: primaryImage}, docker.AuthConfiguration{}) + if err != nil { + return "", nil, fmt.Errorf("error pulling image %v: %v", primaryImage, err) + } + + pCont, err := dClient.CreateContainer(docker.CreateContainerOptions{ + Config: &docker.Config{ + Image: primaryImage, + Cmd: []string{"/bin/sh", "-lc", "while true; do sleep 10; done"}, + Labels: map[string]string{ + "io.kubernetes.container.name": hostname, + "io.kubernetes.pod.name": hostname, + }, + }, + }) + + if err != nil { + return "", nil, fmt.Errorf("error creating primary container: %v", err) + } + + cleanup := func() { + dClient.RemoveContainer(docker.RemoveContainerOptions{ID: pCont.ID, Force: true}) + } + + err = dClient.StartContainer(pCont.ID, nil) + if err != nil { + return pCont.ID, cleanup, fmt.Errorf("error starting primary container: %v", err) + } + + timeout := time.After(time.Second * 15) + for { + c, err := dClient.InspectContainer(pCont.ID) + if err != nil { + return pCont.ID, cleanup, fmt.Errorf("error inspecting primary container: %v", err) + } + if c.State.Running { + break + } + select { + case <-time.After(time.Second): + case <-timeout: + return pCont.ID, cleanup, fmt.Errorf("timeout waiting for primary container to run") + } + } + + return pCont.ID, cleanup, nil +} diff --git a/main.go b/main.go index 30ac757..04ba346 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ type Config struct { DockerHost string `envconfig:"DOCKER_HOST"` RunAsSidecar bool `split_words:"true"` DestinationImages []string `split_words:"true"` + SourceImage string `split_words:"true"` InputFile string `split_words:"true"` RegistryPushRetries int `split_words:"true" default:"3"` RegistryAuthEmail string `split_words:"true"` @@ -46,37 +47,48 @@ func main() { var config Config err := envconfig.Process("deployagent", &config) if err != nil { - fatal("error processing environment variables: %v", err) + fatalf("error processing environment variables: %v", err) } - c := tsuru.Client{ - URL: os.Args[1], - Token: os.Args[2], - Version: version, - } - appName := os.Args[3] - command := os.Args[4:] - var filesystem Filesystem = &localFS{Fs: fs.OsFs{}} var executor exec.Executor = &exec.OsExecutor{} if config.RunAsSidecar { dockerClient, err := docker.NewClient(config.DockerHost) if err != nil { - fatal("failed to create docker client: %v", err) + fatalf("failed to create docker client: %v", err) } sideCar, err := setupSidecar(dockerClient, config) if err != nil { - fatal("failed to create sidecar: %v", err) + fatalf("failed to create sidecar: %v", err) } executor = sideCar filesystem = &executorFS{executor: sideCar} + // we defer the call to pushSidecar so the normal build/deploy steps are executed // by the sidecar executor. This will only be executed if those steps finish without - // any error the call to fatal() exits. + // any error since the call to fatal() exits. defer pushSidecar(dockerClient, sideCar, config, os.Stdout) + + if config.SourceImage != "" { + // build/deploy/deploy-only is not required since this is an image deploy + // all we need to do is return the inspected files and image and push the + // destination images based on the sidecar container. + if err := inspect(dockerClient, config.SourceImage, filesystem, os.Stdout, os.Stderr); err != nil { + fatalf("error inspecting sidecar: %v", err) + } + return + } } + c := tsuru.Client{ + URL: os.Args[1], + Token: os.Args[2], + Version: version, + } + appName := os.Args[3] + command := os.Args[4:] + switch command[len(command)-1] { case "build": err = build(c, appName, command[:len(command)-1], filesystem, executor) @@ -94,11 +106,11 @@ func main() { err = deploy(c, appName, filesystem, executor) } if err != nil { - fatal("[deploy-agent] error: %v", err) + fatalf("[deploy-agent] error: %v", err) } } -func fatal(format string, v ...interface{}) { +func fatalf(format string, v ...interface{}) { file, err := os.OpenFile("/dev/termination-log", os.O_WRONLY|os.O_CREATE, 0666) if err == nil { fmt.Fprintf(file, format, v...)