From 3ec2e8414ace188626dd8d38e31b7b111e3a1c2f Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Mon, 7 Oct 2024 11:55:50 +1300 Subject: [PATCH 1/4] Adds ability to specify tags in refreshimages --- internal/generator/helpers_generator.go | 40 ++++++++ internal/generator/helpers_generator_test.go | 96 ++++++++++++++++++++ internal/generator/services.go | 14 +-- 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/internal/generator/helpers_generator.go b/internal/generator/helpers_generator.go index 976af585..226c6eea 100644 --- a/internal/generator/helpers_generator.go +++ b/internal/generator/helpers_generator.go @@ -8,6 +8,8 @@ import ( "regexp" "strings" + "github.com/distribution/reference" + "github.com/spf13/cobra" "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" "github.com/uselagoon/build-deploy-tool/internal/lagoon" @@ -271,3 +273,41 @@ func getDBaasEnvironment( } return exists, nil } + +func determineRefreshImage(serviceName, imageName string, labels map[string]string, envVars []lagoon.EnvironmentVariable) (string, error) { + tagvalue := lagoon.CheckDockerComposeLagoonLabel(labels, "lagoon.base.image.tag") + if tagvalue != "" { + + // Regular expression to match VAR and defaulttag + re := regexp.MustCompile(`\$(\w+)(?::-(\w+))?`) + matches := re.FindStringSubmatch(tagvalue) + + if len(matches) > 0 { // we have an env var, and optionally a default + tv := "" + envVarKey := matches[1] + defaultVal := matches[2] //This could be empty + for _, v := range envVars { + if v.Name == envVarKey { + // we've found a matching env var - that becomes the tag + tv = v.Value + } + } + if tv == "" { + if defaultVal != "" { + tagvalue = defaultVal + } + } else { + tagvalue = tv + } + } + + println(matches) + + imageName = fmt.Sprintf("%v:%v", imageName, tagvalue) + } + + if !reference.ReferenceRegexp.MatchString(imageName) { + return "", fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - please ensure it conforms to the structure `[REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]`", serviceName, imageName) + } + return imageName, nil +} diff --git a/internal/generator/helpers_generator_test.go b/internal/generator/helpers_generator_test.go index 831eee67..f388beb5 100644 --- a/internal/generator/helpers_generator_test.go +++ b/internal/generator/helpers_generator_test.go @@ -81,3 +81,99 @@ func Test_checkDuplicateCronjobs(t *testing.T) { }) } } + +func Test_determineRefreshImage(t *testing.T) { + type args struct { + serviceName string + imageName string + labels map[string]string + envVars []lagoon.EnvironmentVariable + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Identity function", + args: args{ + serviceName: "testservice", + imageName: "image/name:latest", + labels: nil, + envVars: nil, + }, + want: "image/name:latest", + wantErr: false, + }, + { + name: "Adds simple tag", + args: args{ + serviceName: "testservice", + imageName: "image/name", + labels: map[string]string{ + "lagoon.base.image.tag": "sometag", + }, + envVars: nil, + }, + want: "image/name:sometag", + wantErr: false, + }, + { + name: "Fails with double tags", + args: args{ + serviceName: "testservice", + imageName: "image/name:latest", + labels: map[string]string{ + "lagoon.base.image.tag": "sometag", + }, + envVars: nil, + }, + want: "", + wantErr: true, + }, + { + name: "Tag with simple arg - fallback to default", + args: args{ + serviceName: "testservice", + imageName: "image/name", + labels: map[string]string{ + "lagoon.base.image.tag": "$ENVVAR:-sometag", + }, + envVars: nil, + }, + want: "image/name:sometag", + wantErr: false, + }, + { + name: "Tag with env var that works", + args: args{ + serviceName: "testservice", + imageName: "image/name", + labels: map[string]string{ + "lagoon.base.image.tag": "$ENVVAR:-sometag", + }, + envVars: []lagoon.EnvironmentVariable{ + { + Name: "ENVVAR", + Value: "injectedTag", + }, + }, + }, + want: "image/name:injectedTag", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := determineRefreshImage(tt.args.serviceName, tt.args.imageName, tt.args.labels, tt.args.envVars) + if (err != nil) != tt.wantErr { + t.Errorf("determineRefreshImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("determineRefreshImage() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/generator/services.go b/internal/generator/services.go index f3e8908e..cc849e4f 100644 --- a/internal/generator/services.go +++ b/internal/generator/services.go @@ -7,8 +7,6 @@ import ( "strconv" "strings" - "github.com/distribution/reference" - composetypes "github.com/compose-spec/compose-go/types" "github.com/uselagoon/build-deploy-tool/internal/helpers" "github.com/uselagoon/build-deploy-tool/internal/lagoon" @@ -286,13 +284,17 @@ func composeToServiceValues( } } + // if any `lagoon.base.image` labels are set, we note them for docker pulling + // this allows us to refresh the docker-host's cache in cases where an image + // may have an update without a change in tag (i.e. "latest" tagged images) baseimage := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.base.image") if baseimage != "" { - // First, let's ensure that the structure of the base image is valid - if !reference.ReferenceRegexp.MatchString(baseimage) { - return nil, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - please ensure it conforms to the structure `[REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]`", composeService, baseimage) + + baseImageWithTag, err := determineRefreshImage(composeService, baseimage, composeServiceValues.Labels, buildValues.EnvironmentVariables) + if err != nil { + return nil, err } - buildValues.ForcePullImages = append(buildValues.ForcePullImages, baseimage) + buildValues.ForcePullImages = append(buildValues.ForcePullImages, baseImageWithTag) } // if there are overrides defined in the lagoon API `LAGOON_SERVICE_TYPES` From a34c0b4eadc00657801c01b32fc1888e5fee16ef Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Mon, 7 Oct 2024 12:06:19 +1300 Subject: [PATCH 2/4] removes println --- internal/generator/helpers_generator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/generator/helpers_generator.go b/internal/generator/helpers_generator.go index 226c6eea..9f6d3b5a 100644 --- a/internal/generator/helpers_generator.go +++ b/internal/generator/helpers_generator.go @@ -301,8 +301,6 @@ func determineRefreshImage(serviceName, imageName string, labels map[string]stri } } - println(matches) - imageName = fmt.Sprintf("%v:%v", imageName, tagvalue) } From 6868c3ab289ac4282ddd3d31ca3703fb1d44e1be Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 8 Oct 2024 15:00:31 +1100 Subject: [PATCH 3/4] refactor: just use one tag with variable replacement --- internal/generator/helpers_generator.go | 37 ++++++++------- internal/generator/helpers_generator_test.go | 50 ++++++++------------ internal/generator/services.go | 13 +++-- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/internal/generator/helpers_generator.go b/internal/generator/helpers_generator.go index 9f6d3b5a..505f5be4 100644 --- a/internal/generator/helpers_generator.go +++ b/internal/generator/helpers_generator.go @@ -274,38 +274,43 @@ func getDBaasEnvironment( return exists, nil } -func determineRefreshImage(serviceName, imageName string, labels map[string]string, envVars []lagoon.EnvironmentVariable) (string, error) { - tagvalue := lagoon.CheckDockerComposeLagoonLabel(labels, "lagoon.base.image.tag") - if tagvalue != "" { +var exp = regexp.MustCompile(`(\\*)\$\{(.+?)(?:(\:\-)(.*?))?\}`) - // Regular expression to match VAR and defaulttag - re := regexp.MustCompile(`\$(\w+)(?::-(\w+))?`) - matches := re.FindStringSubmatch(tagvalue) - - if len(matches) > 0 { // we have an env var, and optionally a default +func determineRefreshImage(serviceName, imageName string, envVars []lagoon.EnvironmentVariable) (string, []error) { + errs := []error{} + parsed := exp.ReplaceAllStringFunc(string(imageName), func(match string) string { + tagvalue := "" + re := regexp.MustCompile(`\${?(\w+)?(?::-(\w+))?}?`) + matches := re.FindStringSubmatch(match) + if len(matches) > 0 { tv := "" envVarKey := matches[1] defaultVal := matches[2] //This could be empty for _, v := range envVars { if v.Name == envVarKey { - // we've found a matching env var - that becomes the tag tv = v.Value } } if tv == "" { if defaultVal != "" { tagvalue = defaultVal + } else { + errs = append(errs, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - no matching variable or fallback found to replace requested variable %s", serviceName, imageName, envVarKey)) } } else { tagvalue = tv } } - - imageName = fmt.Sprintf("%v:%v", imageName, tagvalue) - } - - if !reference.ReferenceRegexp.MatchString(imageName) { - return "", fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - please ensure it conforms to the structure `[REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]`", serviceName, imageName) + return tagvalue + }) + if parsed == imageName { + if !reference.ReferenceRegexp.MatchString(parsed) { + if strings.Contains(parsed, "$") { + errs = append(errs, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - variables are defined incorrectly, must contain curly brackets (example: '${VARIABLE}')", serviceName, imageName)) + } else { + errs = append(errs, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - please ensure it conforms to the structure `[REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]`", serviceName, imageName)) + } + } } - return imageName, nil + return parsed, errs } diff --git a/internal/generator/helpers_generator_test.go b/internal/generator/helpers_generator_test.go index f388beb5..366a0970 100644 --- a/internal/generator/helpers_generator_test.go +++ b/internal/generator/helpers_generator_test.go @@ -86,7 +86,6 @@ func Test_determineRefreshImage(t *testing.T) { type args struct { serviceName string imageName string - labels map[string]string envVars []lagoon.EnvironmentVariable } tests := []struct { @@ -100,34 +99,27 @@ func Test_determineRefreshImage(t *testing.T) { args: args{ serviceName: "testservice", imageName: "image/name:latest", - labels: nil, envVars: nil, }, want: "image/name:latest", wantErr: false, }, { - name: "Adds simple tag", + name: "Fails with no matching variable in envvars", args: args{ serviceName: "testservice", - imageName: "image/name", - labels: map[string]string{ - "lagoon.base.image.tag": "sometag", - }, - envVars: nil, + imageName: "image/name:${NOENVVAR}", + envVars: nil, }, - want: "image/name:sometag", - wantErr: false, + want: "", + wantErr: true, }, { - name: "Fails with double tags", + name: "Fails with variable missing curly brackets", args: args{ serviceName: "testservice", - imageName: "image/name:latest", - labels: map[string]string{ - "lagoon.base.image.tag": "sometag", - }, - envVars: nil, + imageName: "image/name:$NOENVVAR", + envVars: nil, }, want: "", wantErr: true, @@ -136,11 +128,8 @@ func Test_determineRefreshImage(t *testing.T) { name: "Tag with simple arg - fallback to default", args: args{ serviceName: "testservice", - imageName: "image/name", - labels: map[string]string{ - "lagoon.base.image.tag": "$ENVVAR:-sometag", - }, - envVars: nil, + imageName: "image/name:${ENVVAR:-sometag}", + envVars: nil, }, want: "image/name:sometag", wantErr: false, @@ -149,10 +138,7 @@ func Test_determineRefreshImage(t *testing.T) { name: "Tag with env var that works", args: args{ serviceName: "testservice", - imageName: "image/name", - labels: map[string]string{ - "lagoon.base.image.tag": "$ENVVAR:-sometag", - }, + imageName: "image/name:${ENVVAR:-sometag}", envVars: []lagoon.EnvironmentVariable{ { Name: "ENVVAR", @@ -166,12 +152,16 @@ func Test_determineRefreshImage(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := determineRefreshImage(tt.args.serviceName, tt.args.imageName, tt.args.labels, tt.args.envVars) - if (err != nil) != tt.wantErr { - t.Errorf("determineRefreshImage() error = %v, wantErr %v", err, tt.wantErr) - return + got, errs := determineRefreshImage(tt.args.serviceName, tt.args.imageName, tt.args.envVars) + if len(errs) > 0 && !tt.wantErr { + for idx, err := range errs { + t.Errorf("determineRefreshImage() error = %v, wantErr %v", err, tt.wantErr) + if idx+1 == len(errs) { + return + } + } } - if got != tt.want { + if got != tt.want && !tt.wantErr { t.Errorf("determineRefreshImage() got = %v, want %v", got, tt.want) } }) diff --git a/internal/generator/services.go b/internal/generator/services.go index cc849e4f..1732e779 100644 --- a/internal/generator/services.go +++ b/internal/generator/services.go @@ -289,10 +289,15 @@ func composeToServiceValues( // may have an update without a change in tag (i.e. "latest" tagged images) baseimage := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.base.image") if baseimage != "" { - - baseImageWithTag, err := determineRefreshImage(composeService, baseimage, composeServiceValues.Labels, buildValues.EnvironmentVariables) - if err != nil { - return nil, err + baseImageWithTag, errs := determineRefreshImage(composeService, baseimage, buildValues.EnvironmentVariables) + if len(errs) > 0 { + for idx, err := range errs { + if idx+1 == len(errs) { + return nil, err + } else { + fmt.Println(err) + } + } } buildValues.ForcePullImages = append(buildValues.ForcePullImages, baseImageWithTag) } From b02abbb11666580f98ec4ce47e88b4751dacbd8d Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 8 Oct 2024 15:59:13 +1100 Subject: [PATCH 4/4] test: add test with lagoon and docker-compose files --- cmd/identify_imagebuild_test.go | 62 ++++++++++++++++++- .../docker-compose.forcebaseimagepull-2.yml | 23 +++++++ .../basic/lagoon.forcebaseimagepull-2.yml | 10 +++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 internal/testdata/basic/docker-compose.forcebaseimagepull-2.yml create mode 100644 internal/testdata/basic/lagoon.forcebaseimagepull-2.yml diff --git a/cmd/identify_imagebuild_test.go b/cmd/identify_imagebuild_test.go index 871ffc09..4c713f5e 100644 --- a/cmd/identify_imagebuild_test.go +++ b/cmd/identify_imagebuild_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + "github.com/andreyvit/diff" "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" "github.com/uselagoon/build-deploy-tool/internal/generator" "github.com/uselagoon/build-deploy-tool/internal/helpers" @@ -752,6 +753,61 @@ func TestImageBuildConfigurationIdentification(t *testing.T) { }, }, }, + { + name: "test12 Force Pull Base Images with variable replacement", + args: testdata.GetSeedData( + testdata.TestData{ + Namespace: "example-project-main", + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/basic/lagoon.forcebaseimagepull-2.yml", + ProjectVariables: []lagoon.EnvironmentVariable{ + { + Name: "BASE_IMAGE_TAG", + Value: "my-tag", + Scope: "build", + }, + { + Name: "BASE_IMAGE_REPO", + Value: "my-repo", + Scope: "build", + }, + }, + }, true), + want: imageBuild{ + BuildKit: helpers.BoolPtr(true), + BuildArguments: map[string]string{ + "BASE_IMAGE_TAG": "my-tag", + "BASE_IMAGE_REPO": "my-repo", + "LAGOON_BUILD_NAME": "lagoon-build-abcdefg", + "LAGOON_PROJECT": "example-project", + "LAGOON_ENVIRONMENT": "main", + "LAGOON_ENVIRONMENT_TYPE": "production", + "LAGOON_BUILD_TYPE": "branch", + "LAGOON_GIT_SOURCE_REPOSITORY": "ssh://git@example.com/lagoon-demo.git", + "LAGOON_KUBERNETES": "remote-cluster1", + "LAGOON_GIT_SHA": "abcdefg123456", + "LAGOON_GIT_BRANCH": "main", + "NODE_IMAGE": "example-project-main-node", + "LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----", + }, + ForcePullImages: []string{ + "registry.com/my-repo/imagename:my-tag", + }, + Images: []imageBuilds{ + { + Name: "node", + ImageBuild: generator.ImageBuild{ + BuildImage: "harbor.example/example-project/main/node:latest", + Context: "internal/testdata/basic/docker", + DockerFile: "basic.dockerfile", + TemporaryImage: "example-project-main-node", + }, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -788,10 +844,10 @@ func TestImageBuildConfigurationIdentification(t *testing.T) { t.Errorf("%v", err) } - oJ, _ := json.Marshal(out) - wJ, _ := json.Marshal(tt.want) + oJ, _ := json.MarshalIndent(out, "", " ") + wJ, _ := json.MarshalIndent(tt.want, "", " ") if string(oJ) != string(wJ) { - t.Errorf("returned output %v doesn't match want %v", string(oJ), string(wJ)) + t.Errorf("ImageBuildConfigurationIdentification() = \n%v", diff.LineDiff(string(oJ), string(wJ))) } t.Cleanup(func() { helpers.UnsetEnvVars(tt.vars) diff --git a/internal/testdata/basic/docker-compose.forcebaseimagepull-2.yml b/internal/testdata/basic/docker-compose.forcebaseimagepull-2.yml new file mode 100644 index 00000000..ba74b5a5 --- /dev/null +++ b/internal/testdata/basic/docker-compose.forcebaseimagepull-2.yml @@ -0,0 +1,23 @@ +version: '2' +services: + node: + networks: + - amazeeio-network + - default + build: + context: internal/testdata/basic/docker + dockerfile: basic.dockerfile + labels: + lagoon.type: basic + lagoon.service.usecomposeports: true + lagoon.base.image: registry.com/${BASE_IMAGE_REPO:-namespace}/imagename:${BASE_IMAGE_TAG:-latest} + volumes: + - .:/app:delegated + ports: + - '1234' + - '8191' + - '9001/udp' + +networks: + amazeeio-network: + external: true \ No newline at end of file diff --git a/internal/testdata/basic/lagoon.forcebaseimagepull-2.yml b/internal/testdata/basic/lagoon.forcebaseimagepull-2.yml new file mode 100644 index 00000000..b9ee74ac --- /dev/null +++ b/internal/testdata/basic/lagoon.forcebaseimagepull-2.yml @@ -0,0 +1,10 @@ +docker-compose-yaml: internal/testdata/basic/docker-compose.forcebaseimagepull-2.yml + +environment_variables: + git_sha: "true" + +environments: + main: + routes: + - node: + - example.com