diff --git a/.gitignore b/.gitignore index b93c3d9..d0f7f3b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ # coverage reports *.coverprofile + +# dummy config.yaml for development and manual testing +/config.yaml diff --git a/api/registry/registry.go b/api/registry/registry.go index 43610ea..13c4083 100644 --- a/api/registry/registry.go +++ b/api/registry/registry.go @@ -1,11 +1,15 @@ package registry import ( + "bufio" "crypto/rand" "fmt" + "io" "net/http" "time" + log "github.com/sirupsen/logrus" + dockerclient "github.com/ivanilves/lstags/docker/client" dockerconfig "github.com/ivanilves/lstags/docker/config" "github.com/ivanilves/lstags/repository" @@ -136,9 +140,32 @@ func (c *Container) SeedWithImages(refs ...string) ([]string, error) { pushRefs[i] = pushRef - done <- c.dockerClient.RePush(src, dst) + pullResp, err := c.dockerClient.Pull(src) + if err != nil { + done <- err + return + } + logDebugData(pullResp) + + c.dockerClient.Tag(src, dst) + + pushResp, err := c.dockerClient.Push(dst) + if err != nil { + done <- err + return + } + logDebugData(pushResp) + + done <- err }(i, ref) } return pushRefs, wait.Until(done) } + +func logDebugData(data io.Reader) { + scanner := bufio.NewScanner(data) + for scanner.Scan() { + log.Debug(scanner.Text()) + } +} diff --git a/api/registry/registry_test.go b/api/registry/registry_test.go index e1091ba..e98e4c2 100644 --- a/api/registry/registry_test.go +++ b/api/registry/registry_test.go @@ -157,11 +157,14 @@ func TestSeedContainerWithImages(t *testing.T) { for _, ref := range refs { go func(ref string) { - if err := dockerClient.Pull(ref); err != nil { + resp, err := dockerClient.Pull(ref) + if err != nil { done <- err return } + logDebugData(resp) + done <- nil }(ref) } diff --git a/api/v1/v1.go b/api/v1/v1.go index af719e5..b3858e4 100644 --- a/api/v1/v1.go +++ b/api/v1/v1.go @@ -1,8 +1,10 @@ package v1 import ( + "bufio" "fmt" log "github.com/sirupsen/logrus" + "io" "runtime" "strings" "time" @@ -285,7 +287,11 @@ func (api *API) PullTags(cn *collection.Collection) error { log.Infof("PULLING %s", ref) - done <- api.dockerClient.Pull(ref) + resp, err := api.dockerClient.Pull(ref) + + logDebugData(resp) + + done <- err } }(repo, tags, done) } @@ -326,7 +332,23 @@ func (api *API) PushTags(cn *collection.Collection, push PushConfig) error { log.Infof("[PULL/PUSH] PUSHING %s => %s", srcRef, dstRef) - done <- api.dockerClient.RePush(srcRef, dstRef) + pullResp, err := api.dockerClient.Pull(srcRef) + if err != nil { + done <- err + return + } + logDebugData(pullResp) + + api.dockerClient.Tag(srcRef, dstRef) + + pushResp, err := api.dockerClient.Push(dstRef) + if err != nil { + done <- err + return + } + logDebugData(pushResp) + + done <- err } }(repo, tags, done) } @@ -334,6 +356,13 @@ func (api *API) PushTags(cn *collection.Collection, push PushConfig) error { return wait.Until(done) } +func logDebugData(data io.Reader) { + scanner := bufio.NewScanner(data) + for scanner.Scan() { + log.Debug(scanner.Text()) + } +} + // New creates new instance of application API func New(config Config) (*API, error) { if config.VerboseLogging { @@ -349,9 +378,6 @@ func New(config Config) (*API, error) { remote.RetryRequests = config.RetryRequests remote.RetryDelay = config.RetryDelay - dockerclient.RetryPulls = config.RetryRequests - dockerclient.RetryDelay = config.RetryDelay - if config.InsecureRegistryEx != "" { repository.InsecureRegistryEx = config.InsecureRegistryEx } diff --git a/docker/client/client.go b/docker/client/client.go index 14968d7..087cafd 100644 --- a/docker/client/client.go +++ b/docker/client/client.go @@ -3,7 +3,6 @@ package client import ( "io" "io/ioutil" - "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -20,12 +19,6 @@ import ( // DockerSocket is a socket we use to connect to the Docker daemon var DockerSocket = "/var/run/docker.sock" -// RetryPulls is a number of retries we do in case of Docker pull failure -var RetryPulls = 0 - -// RetryDelay is a delay between retries of the failed Docker pulls -var RetryDelay = 5 * time.Second - // DockerClient is a raw Docker client convenience wrapper type DockerClient struct { cli *client.Client @@ -79,7 +72,7 @@ func buildImageListOptions(repo string) (types.ImageListOptions, error) { } // Pull pulls Docker image specified -func (dc *DockerClient) Pull(ref string) error { +func (dc *DockerClient) Pull(ref string) (io.ReadCloser, error) { registryAuth := dc.cnf.GetRegistryAuth( repository.GetRegistry(ref), ) @@ -89,38 +82,11 @@ func (dc *DockerClient) Pull(ref string) error { pullOptions = types.ImagePullOptions{} } - tries := 1 - - if RetryPulls > 0 { - tries = tries + RetryPulls - } - - var resp io.ReadCloser - var err error - - for try := 1; try <= tries; try++ { - resp, err = dc.cli.ImagePull(context.Background(), ref, pullOptions) - - if err == nil { - break - } - - time.Sleep(RetryDelay) - - RetryDelay += RetryDelay - } - - if err != nil { - return err - } - - _, err = ioutil.ReadAll(resp) - - return err + return dc.cli.ImagePull(context.Background(), ref, pullOptions) } // Push pushes Docker image specified -func (dc *DockerClient) Push(ref string) error { +func (dc *DockerClient) Push(ref string) (io.ReadCloser, error) { registryAuth := dc.cnf.GetRegistryAuth( repository.GetRegistry(ref), ) @@ -130,14 +96,7 @@ func (dc *DockerClient) Push(ref string) error { pushOptions = types.ImagePushOptions{RegistryAuth: "IA=="} } - resp, err := dc.cli.ImagePush(context.Background(), ref, pushOptions) - if err != nil { - return err - } - - _, err = ioutil.ReadAll(resp) - - return err + return dc.cli.ImagePush(context.Background(), ref, pushOptions) } // Tag puts a "dst" tag on "src" Docker image @@ -145,19 +104,6 @@ func (dc *DockerClient) Tag(src, dst string) error { return dc.cli.ImageTag(context.Background(), src, dst) } -// RePush pulls, tags and re-pushes given image references -func (dc *DockerClient) RePush(src, dst string) error { - if err := dc.Pull(src); err != nil { - return err - } - - if err := dc.Tag(src, dst); err != nil { - return err - } - - return dc.Push(dst) -} - // Run runs Docker container from the image specified (like "docker run") func (dc *DockerClient) Run(ref, name string, portSpecs []string) (string, error) { exposedPorts, portBindings, err := nat.ParsePortSpecs(portSpecs) @@ -167,7 +113,21 @@ func (dc *DockerClient) Run(ref, name string, portSpecs []string) (string, error ctx := context.Background() - if err := dc.Pull(ref); err != nil { + registryAuth := dc.cnf.GetRegistryAuth( + repository.GetRegistry(ref), + ) + + pullOptions := types.ImagePullOptions{RegistryAuth: registryAuth} + if registryAuth == "" { + pullOptions = types.ImagePullOptions{} + } + + pullResp, err := dc.cli.ImagePull(ctx, ref, pullOptions) + if err != nil { + return "", err + } + _, err = ioutil.ReadAll(pullResp) + if err != nil { return "", err }