From ea48aaf2553882d3295a3b66feac783ade6302e2 Mon Sep 17 00:00:00 2001 From: Ben Einaudi Date: Thu, 30 Jan 2020 18:33:21 +0100 Subject: [PATCH] Allow contributors to launch integration tests against local registry This change allows user to launch integration tests with a local registry Fixes #1012 --- .travis.yml | 4 +- DEVELOPMENT.md | 68 ++++++-- integration/config.go | 34 ++++ integration/integration_test.go | 96 ++++++----- integration/migrate_gcs.go | 149 ++++++++++++++++++ .../replace-gcr-with-local-registry.sh | 43 ----- 6 files changed, 294 insertions(+), 100 deletions(-) create mode 100644 integration/config.go create mode 100644 integration/migrate_gcs.go delete mode 100755 integration/replace-gcr-with-local-registry.sh diff --git a/.travis.yml b/.travis.yml index e19669cc7f..11dcbe8fd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,7 @@ before_install: - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && sudo mv container-diff-linux-amd64 /usr/local/bin/container-diff - docker run -d -p 5000:5000 --restart always --name registry registry:2 - - ./integration/replace-gcr-with-local-registry.sh integration/dockerfiles - script: - make test - - ./integration-test.sh --uploadToGCS=false + - make integration-test - make images diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b395d2d9a2..237057beca 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -67,18 +67,39 @@ _These tests will not run correctly unless you have [checked out your fork into ### Integration tests -The integration tests live in [`integration`](./integration) and can be run with: +Currently the integration tests that live in [`integration`](./integration) can be run against your own gcloud space or a local registry. + +In either case, you will need the following tools: + +* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation) + +#### GCloud + +To run integration tests with your GCloud Storage, you will also need the following tools: + +* [`gcloud`](https://cloud.google.com/sdk/install) +* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install) +* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via + the user currently logged into `gcloud` +* An image repo which you have write access to via the user currently logged into `gcloud` + +Once this step done, you must override the project using environment variables: + +* `GCS_BUCKET` - The name of your GCS bucket +* `IMAGE_REPO` - The path to your docker image repo + +This can be done as follows: ```shell export GCS_BUCKET="gs://" export IMAGE_REPO="gcr.io/somerepo" -make integration-test ``` -If you want to run `make integration-test`, you must override the project using environment variables: +Then you can launch integration tests as follows: -* `GCS_BUCKET` - The name of your GCS bucket -* `IMAGE_REPO` - The path to your docker image repo +```shell +make integration-test +``` You can also run tests with `go test`, for example to run tests individually: @@ -86,16 +107,37 @@ You can also run tests with `go test`, for example to run tests individually: go test ./integration -v --bucket $GCS_BUCKET --repo $IMAGE_REPO -run TestLayers/test_layer_Dockerfile_test_copy_bucket ``` -Requirements: +These tests will be kicked off by [reviewers](#reviews) for submitted PRs by the kokoro task. + +#### Local repository + +Otherwise you can run integration tests locally. To do so, install a local docker registry + +```shell +docker run --rm -d -p 5000:5000 --name registry registry:2 +``` + +Then export the `IMAGE_REPO` variable with the `localhost:5000`value + +```shell +export IMAGE_REPO=localhost:5000 +``` + +And run the integration tests + +```shell +make integration-test +``` + +You can also run tests with `go test`, for example to run tests individually: + +```shell +go test ./integration -v --repo localhost:5000 -run TestLayers/test_layer_Dockerfile_test_copy_bucket +``` + +These tests will be kicked off by [reviewers](#reviews) for submitted PRs by the travis task. -* [`gcloud`](https://cloud.google.com/sdk/install) -* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install) -* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation) -* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via - the user currently logged into `gcloud` -* An image repo which you have write access to via the user currently logged into `gcloud` -These tests will be kicked off by [reviewers](#reviews) for submitted PRs. ### Benchmarking diff --git a/integration/config.go b/integration/config.go new file mode 100644 index 0000000000..dceb6d91d3 --- /dev/null +++ b/integration/config.go @@ -0,0 +1,34 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import "strings" + +type gcpConfig struct { + gcsBucket string + imageRepo string + onbuildBaseImage string + hardlinkBaseImage string + serviceAccount string + dockerMajorVersion int +} + +const gcrRepoPrefix string = "gcr.io/" + +func (config *gcpConfig) isGcrRepository() bool { + return strings.HasPrefix(config.imageRepo, gcrRepoPrefix) +} diff --git a/integration/integration_test.go b/integration/integration_test.go index c5c910d885..24d2736a74 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -20,6 +20,7 @@ import ( "encoding/json" "flag" "fmt" + "github.com/pkg/errors" "log" "math" "os" @@ -80,37 +81,67 @@ func getDockerMajorVersion() int { } return ver } +func launchTests(m *testing.M, dockerfiles [] string) (error, int) { -func TestMain(m *testing.M) { - if !meetsRequirements() { - fmt.Println("Missing required tools") - os.Exit(1) - } - config = initGCPConfig() - - if config.uploadToGCS { + if config.isGcrRepository() { contextFile, err := CreateIntegrationTarball() if err != nil { - fmt.Println("Failed to create tarball of integration files for build context", err) - os.Exit(1) + return errors.Wrap(err, "Failed to create tarball of integration files for build context"), 1 } fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile) if err != nil { - fmt.Println("Failed to upload build context", err) - os.Exit(1) + return errors.Wrap(err, "Failed to upload build context"), 1 } - err = os.Remove(contextFile) - if err != nil { - fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err) - os.Exit(1) + if err = os.Remove(contextFile); err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to remove tarball at %s", contextFile)), 1 } RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) }) defer DeleteFromBucket(fileInBucket) + } else { + var err error + var migratedFiles [] string + if err, migratedFiles = MigrateGcsRegistry(dockerfilesPath, dockerfiles, config.imageRepo); err != nil { + RollbackMigratedFiles(dockerfilesPath, migratedFiles) + return errors.Wrap(err, "Fail to migrate dockerfiles from gcs"), 1 + } + RunOnInterrupt(func() { RollbackMigratedFiles(dockerfilesPath, migratedFiles) }) + defer RollbackMigratedFiles(dockerfilesPath, migratedFiles) + } + if err := buildRequiredImages(); err != nil { + return errors.Wrap(err, "Error while building images"), 1 + } + + imageBuilder = NewDockerFileBuilder(dockerfiles) + + return nil, m.Run() +} + +func TestMain(m *testing.M) { + if !meetsRequirements() { + fmt.Println("Missing required tools") + os.Exit(1) } + if dockerfiles, err := FindDockerFiles(dockerfilesPath); err != nil { + fmt.Println("Coudn't create map of dockerfiles", err) + os.Exit(1) + } else { + config = initGCPConfig() + if err, exitCode := launchTests(m, dockerfiles); err != nil { + fmt.Println(err) + os.Exit(exitCode) + } else { + os.Exit(exitCode) + } + } + + +} + +func buildRequiredImages() error { setupCommands := []struct { name string command []string @@ -145,20 +176,10 @@ func TestMain(m *testing.M) { fmt.Println(setupCmd.name) cmd := exec.Command(setupCmd.command[0], setupCmd.command[1:]...) if out, err := RunCommandWithoutTest(cmd); err != nil { - fmt.Printf("%s failed: %s", setupCmd.name, err) - fmt.Println(string(out)) - os.Exit(1) + return errors.Wrap(err, fmt.Sprintf("%s failed: %s", setupCmd.name, string(out))) } } - - dockerfiles, err := FindDockerFiles(dockerfilesPath) - if err != nil { - fmt.Printf("Coudn't create map of dockerfiles: %s", err) - os.Exit(1) - } - imageBuilder = NewDockerFileBuilder(dockerfiles) - - os.Exit(m.Run()) + return nil } func TestRun(t *testing.T) { @@ -535,16 +556,6 @@ func logBenchmarks(benchmark string) error { return nil } -type gcpConfig struct { - gcsBucket string - imageRepo string - onbuildBaseImage string - hardlinkBaseImage string - serviceAccount string - dockerMajorVersion int - uploadToGCS bool -} - type imageDetails struct { name string numLayers int @@ -560,7 +571,6 @@ func initGCPConfig() *gcpConfig { flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.") flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo or serviceAccount must be set.") flag.StringVar(&c.serviceAccount, "serviceAccount", "", "The path to the service account push images to GCR and upload/download files to GCS.") - flag.BoolVar(&c.uploadToGCS, "uploadToGCS", true, "Upload the tar-ed contents of `integration` dir to GCS bucket. Default is true. Set this to false to prevent uploading.") flag.Parse() if len(c.serviceAccount) > 0 { @@ -575,8 +585,12 @@ func initGCPConfig() *gcpConfig { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", absPath) } - if c.gcsBucket == "" || c.imageRepo == "" { - log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo) + if c.imageRepo == "" { + log.Fatalf("You must provide a docker repo (\"%s\" was provided)", c.imageRepo) + } + + if c.isGcrRepository() && c.gcsBucket == "" { + log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) when using a gcr docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo) } if !strings.HasSuffix(c.imageRepo, "/") { c.imageRepo = c.imageRepo + "/" diff --git a/integration/migrate_gcs.go b/integration/migrate_gcs.go new file mode 100644 index 0000000000..80057a8801 --- /dev/null +++ b/integration/migrate_gcs.go @@ -0,0 +1,149 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "path" + "regexp" + "strings" +) + +func MigrateGcsRegistry(dockerfilesPath string, dockerfiles []string, targetRepo string) (error, [] string) { + var migratedFiles [] string + importedImages := map[string]interface{}{} + + for _, dockerfile := range dockerfiles { + if err, gcrBaseImages := getGcrBaseImages(dockerfilesPath, dockerfile); err != nil { + return err, migratedFiles + } else if len(gcrBaseImages) > 0 { + fmt.Printf("Migrating %s\n", dockerfile) + if err := os.Rename(path.Join(dockerfilesPath, dockerfile), path.Join(dockerfilesPath, saveName(dockerfile))); err != nil { + return err, migratedFiles + } else { + migratedFiles = append(migratedFiles, dockerfile) + if err := migrateFile(dockerfilesPath, dockerfile, targetRepo); err != nil { + return err, migratedFiles + } + for _, gcrBaseImage := range gcrBaseImages { + importedImages[gcrBaseImage] = nil + } + } + } + } + for image, _ := range importedImages { + if err := importImage(image, targetRepo); err != nil { + return err, migratedFiles + } + } + return nil, migratedFiles +} + +func RollbackMigratedFiles(dockerfilesPath string, dockerfiles [] string) [] error { + var result [] error + for _, dockerfile := range dockerfiles { + fmt.Printf("Rolling back %s\n", dockerfile) + if err := os.Rename(path.Join(dockerfilesPath, saveName(dockerfile)), path.Join(dockerfilesPath, dockerfile)); err != nil { + result = append(result, err) + } + } + return result +} + +func importImage(image string, targetRepo string) error { + fmt.Printf("Importing %s to %s\n", image, targetRepo) + targetImage := strings.ReplaceAll(image, "gcr.io/", targetRepo) + pullCmd := exec.Command("docker", "pull", image) + if out, err := RunCommandWithoutTest(pullCmd); err != nil { + return fmt.Errorf("Failed to pull image %s with docker command \"%s\": %s %s", image, pullCmd.Args, err, string(out)) + } + + tagCmd := exec.Command("docker", "tag", image, targetImage) + if out, err := RunCommandWithoutTest(tagCmd); err != nil { + return fmt.Errorf("Failed to tag image %s to %s with docker command \"%s\": %s %s", image, targetImage, tagCmd.Args, err, string(out)) + } + + pushCmd := exec.Command("docker", "push", targetImage) + if out, err := RunCommandWithoutTest(pushCmd); err != nil { + return fmt.Errorf("Failed to push image %s with docker command \"%s\": %s %s", targetImage, pushCmd.Args, err, string(out)) + } + return nil +} + +func migrateFile(dockerfilesPath string, dockerfile string, imageRepo string) error { + var input *os.File + var output *os.File + var err error + if input, err = os.Open(path.Join(dockerfilesPath, saveName(dockerfile))); err != nil { + return err + } + defer input.Close() + if output, err = os.Create(path.Join(dockerfilesPath, dockerfile)); err != nil { + return err + } + defer output.Close() + + var lines [] string + + scanner := bufio.NewScanner(input) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + line := scanner.Text() + if isFromGcrBaseImageInstruction(line) { + lines = append(lines, strings.ReplaceAll(line, gcrRepoPrefix, imageRepo)) + } else { + lines = append(lines, line) + } + } + rawOutput := []byte(strings.Join(append(lines, ""), "\n")) + if written, err := output.Write(rawOutput); err != nil { + return err + } else if written != len(rawOutput) { + return fmt.Errorf("invalid number of byte written. Got %d, expected %d", written, len(rawOutput)) + } + return nil +} + +func saveName(dockerfile string) string { + return fmt.Sprintf("%s_save_%d", dockerfile, os.Getpid()) +} + +func getGcrBaseImages(dockerfilesPath string, dockerfile string) (error, []string) { + if file, err := os.Open(path.Join(dockerfilesPath, dockerfile)); err != nil { + return err, nil + } else { + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + var result [] string + for scanner.Scan() { + line := scanner.Text() + if isFromGcrBaseImageInstruction(line) { + result = append(result, strings.Trim(strings.Split(line, " ")[1], " ")) + } + } + return nil, result + } +} + +func isFromGcrBaseImageInstruction(line string) bool { + result, _ := regexp.MatchString(fmt.Sprintf("FROM +%s", gcrRepoPrefix), line) + return result +} diff --git a/integration/replace-gcr-with-local-registry.sh b/integration/replace-gcr-with-local-registry.sh deleted file mode 100755 index e40337fdee..0000000000 --- a/integration/replace-gcr-with-local-registry.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This script is needed due to the following bug: -# https://github.com/GoogleContainerTools/kaniko/issues/966 - - -if [ "$#" -ne 1 ]; then - echo "Please specify path to dockerfiles as first argument." - echo "Usage: `basename $0` integration/dockerfiles" - exit 2 -fi - -dir_with_docker_files=$1 - -for dockerfile in $dir_with_docker_files/*; do - cat $dockerfile | grep '^FROM' | grep "gcr" | while read -r line; do - gcr_repo=$(echo "$line" | awk '{ print $2 }') - local_repo=$(echo "$gcr_repo" | sed -e "s/^.*gcr.io\(\/.*\)$/localhost:5000\1/") - remove_digest=$(echo "$local_repo" | cut -f1 -d"@") - echo "Running docker pull $gcr_repo" - docker pull "$gcr_repo" - echo "Running docker tag $gcr_repo $remove_digest" - docker tag "$gcr_repo" "$remove_digest" - echo "Running docker push $remove_digest" - docker push "$remove_digest" - echo "Updating dockerfile $dockerfile to use local repo $local_repo" - sed -i -e "s/^\(FROM \).*gcr.io\(.*\)$/\1localhost:5000\2/" $dockerfile - done -done