From d843c7d9b186bd8adedff10994af27962d6994f6 Mon Sep 17 00:00:00 2001 From: Ben Einaudi Date: Thu, 30 Jan 2020 18:33:21 +0100 Subject: [PATCH] Allow contributors to launch integration tests against local registry This change allows user to launch integration tests with a local registry Fixes #1012 --- .travis.yml | 4 +- DEVELOPMENT.md | 68 ++++++-- integration/config.go | 34 ++++ integration/images.go | 4 +- integration/integration_test.go | 102 +++++++----- integration/migrate_gcs.go | 155 ++++++++++++++++++ .../replace-gcr-with-local-registry.sh | 43 ----- 7 files changed, 305 insertions(+), 105 deletions(-) create mode 100644 integration/config.go create mode 100644 integration/migrate_gcs.go delete mode 100755 integration/replace-gcr-with-local-registry.sh diff --git a/.travis.yml b/.travis.yml index e19669cc7f..11dcbe8fd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,7 @@ before_install: - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && sudo mv container-diff-linux-amd64 /usr/local/bin/container-diff - docker run -d -p 5000:5000 --restart always --name registry registry:2 - - ./integration/replace-gcr-with-local-registry.sh integration/dockerfiles - script: - make test - - ./integration-test.sh --uploadToGCS=false + - make integration-test - make images diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b395d2d9a2..237057beca 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -67,18 +67,39 @@ _These tests will not run correctly unless you have [checked out your fork into ### Integration tests -The integration tests live in [`integration`](./integration) and can be run with: +Currently the integration tests that live in [`integration`](./integration) can be run against your own gcloud space or a local registry. + +In either case, you will need the following tools: + +* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation) + +#### GCloud + +To run integration tests with your GCloud Storage, you will also need the following tools: + +* [`gcloud`](https://cloud.google.com/sdk/install) +* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install) +* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via + the user currently logged into `gcloud` +* An image repo which you have write access to via the user currently logged into `gcloud` + +Once this step done, you must override the project using environment variables: + +* `GCS_BUCKET` - The name of your GCS bucket +* `IMAGE_REPO` - The path to your docker image repo + +This can be done as follows: ```shell export GCS_BUCKET="gs://" export IMAGE_REPO="gcr.io/somerepo" -make integration-test ``` -If you want to run `make integration-test`, you must override the project using environment variables: +Then you can launch integration tests as follows: -* `GCS_BUCKET` - The name of your GCS bucket -* `IMAGE_REPO` - The path to your docker image repo +```shell +make integration-test +``` You can also run tests with `go test`, for example to run tests individually: @@ -86,16 +107,37 @@ You can also run tests with `go test`, for example to run tests individually: go test ./integration -v --bucket $GCS_BUCKET --repo $IMAGE_REPO -run TestLayers/test_layer_Dockerfile_test_copy_bucket ``` -Requirements: +These tests will be kicked off by [reviewers](#reviews) for submitted PRs by the kokoro task. + +#### Local repository + +Otherwise you can run integration tests locally. To do so, install a local docker registry + +```shell +docker run --rm -d -p 5000:5000 --name registry registry:2 +``` + +Then export the `IMAGE_REPO` variable with the `localhost:5000`value + +```shell +export IMAGE_REPO=localhost:5000 +``` + +And run the integration tests + +```shell +make integration-test +``` + +You can also run tests with `go test`, for example to run tests individually: + +```shell +go test ./integration -v --repo localhost:5000 -run TestLayers/test_layer_Dockerfile_test_copy_bucket +``` + +These tests will be kicked off by [reviewers](#reviews) for submitted PRs by the travis task. -* [`gcloud`](https://cloud.google.com/sdk/install) -* [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install) -* [`container-diff`](https://github.com/GoogleContainerTools/container-diff#installation) -* A bucket in [GCS](https://cloud.google.com/storage/) which you have write access to via - the user currently logged into `gcloud` -* An image repo which you have write access to via the user currently logged into `gcloud` -These tests will be kicked off by [reviewers](#reviews) for submitted PRs. ### Benchmarking diff --git a/integration/config.go b/integration/config.go new file mode 100644 index 0000000000..2a6df72a21 --- /dev/null +++ b/integration/config.go @@ -0,0 +1,34 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import "strings" + +type integrationTestConfig struct { + gcsBucket string + imageRepo string + onbuildBaseImage string + hardlinkBaseImage string + serviceAccount string + dockerMajorVersion int +} + +const gcrRepoPrefix string = "gcr.io/" + +func (config *integrationTestConfig) isGcrRepository() bool { + return strings.HasPrefix(config.imageRepo, gcrRepoPrefix) +} diff --git a/integration/images.go b/integration/images.go index f4df7af494..b3da624069 100644 --- a/integration/images.go +++ b/integration/images.go @@ -154,7 +154,7 @@ func addServiceAccountFlags(flags []string, serviceAccount string) []string { // BuildImage will build dockerfile (located at dockerfilesPath) using both kaniko and docker. // The resulting image will be tagged with imageRepo. If the dockerfile will be built with // context (i.e. it is in `buildContextTests`) the context will be pulled from gcsBucket. -func (d *DockerFileBuilder) BuildImage(config *gcpConfig, dockerfilesPath, dockerfile string) error { +func (d *DockerFileBuilder) BuildImage(config *integrationTestConfig, dockerfilesPath, dockerfile string) error { gcsBucket, serviceAccount, imageRepo := config.gcsBucket, config.serviceAccount, config.imageRepo _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) @@ -269,7 +269,7 @@ func populateVolumeCache() error { } // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built -func (d *DockerFileBuilder) buildCachedImages(config *gcpConfig, cacheRepo, dockerfilesPath string, version int) error { +func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int) error { imageRepo, serviceAccount := config.imageRepo, config.serviceAccount _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) diff --git a/integration/integration_test.go b/integration/integration_test.go index c5c910d885..cc7913110d 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -33,12 +33,14 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/daemon" + "github.com/pkg/errors" + "github.com/GoogleContainerTools/kaniko/pkg/timing" "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/GoogleContainerTools/kaniko/testutil" ) -var config *gcpConfig +var config *integrationTestConfig var imageBuilder *DockerFileBuilder const ( @@ -80,37 +82,66 @@ func getDockerMajorVersion() int { } return ver } +func launchTests(m *testing.M, dockerfiles []string) (int, error) { -func TestMain(m *testing.M) { - if !meetsRequirements() { - fmt.Println("Missing required tools") - os.Exit(1) - } - config = initGCPConfig() - - if config.uploadToGCS { + if config.isGcrRepository() { contextFile, err := CreateIntegrationTarball() if err != nil { - fmt.Println("Failed to create tarball of integration files for build context", err) - os.Exit(1) + return 1, errors.Wrap(err, "Failed to create tarball of integration files for build context") } fileInBucket, err := UploadFileToBucket(config.gcsBucket, contextFile, contextFile) if err != nil { - fmt.Println("Failed to upload build context", err) - os.Exit(1) + return 1, errors.Wrap(err, "Failed to upload build context") } - err = os.Remove(contextFile) - if err != nil { - fmt.Printf("Failed to remove tarball at %s: %s\n", contextFile, err) - os.Exit(1) + if err = os.Remove(contextFile); err != nil { + return 1, errors.Wrap(err, fmt.Sprintf("Failed to remove tarball at %s", contextFile)) } RunOnInterrupt(func() { DeleteFromBucket(fileInBucket) }) defer DeleteFromBucket(fileInBucket) + } else { + var err error + var migratedFiles []string + if migratedFiles, err = MigrateGcsRegistry(dockerfilesPath, dockerfiles, config.imageRepo); err != nil { + RollbackMigratedFiles(dockerfilesPath, migratedFiles) + return 1, errors.Wrap(err, "Fail to migrate dockerfiles from gcs") + } + RunOnInterrupt(func() { RollbackMigratedFiles(dockerfilesPath, migratedFiles) }) + defer RollbackMigratedFiles(dockerfilesPath, migratedFiles) + } + if err := buildRequiredImages(); err != nil { + return 1, errors.Wrap(err, "Error while building images") + } + + imageBuilder = NewDockerFileBuilder(dockerfiles) + + return m.Run(), nil +} + +func TestMain(m *testing.M) { + if !meetsRequirements() { + fmt.Println("Missing required tools") + os.Exit(1) + } + + if dockerfiles, err := FindDockerFiles(dockerfilesPath); err != nil { + fmt.Println("Coudn't create map of dockerfiles", err) + os.Exit(1) + } else { + config = initIntegrationTestConfig() + if exitCode, err := launchTests(m, dockerfiles); err != nil { + fmt.Println(err) + os.Exit(exitCode) + } else { + os.Exit(exitCode) + } } +} + +func buildRequiredImages() error { setupCommands := []struct { name string command []string @@ -145,20 +176,10 @@ func TestMain(m *testing.M) { fmt.Println(setupCmd.name) cmd := exec.Command(setupCmd.command[0], setupCmd.command[1:]...) if out, err := RunCommandWithoutTest(cmd); err != nil { - fmt.Printf("%s failed: %s", setupCmd.name, err) - fmt.Println(string(out)) - os.Exit(1) + return errors.Wrap(err, fmt.Sprintf("%s failed: %s", setupCmd.name, string(out))) } } - - dockerfiles, err := FindDockerFiles(dockerfilesPath) - if err != nil { - fmt.Printf("Coudn't create map of dockerfiles: %s", err) - os.Exit(1) - } - imageBuilder = NewDockerFileBuilder(dockerfiles) - - os.Exit(m.Run()) + return nil } func TestRun(t *testing.T) { @@ -535,16 +556,6 @@ func logBenchmarks(benchmark string) error { return nil } -type gcpConfig struct { - gcsBucket string - imageRepo string - onbuildBaseImage string - hardlinkBaseImage string - serviceAccount string - dockerMajorVersion int - uploadToGCS bool -} - type imageDetails struct { name string numLayers int @@ -555,12 +566,11 @@ func (i imageDetails) String() string { return fmt.Sprintf("Image: [%s] Digest: [%s] Number of Layers: [%d]", i.name, i.digest, i.numLayers) } -func initGCPConfig() *gcpConfig { - var c gcpConfig +func initIntegrationTestConfig() *integrationTestConfig { + var c integrationTestConfig flag.StringVar(&c.gcsBucket, "bucket", "gs://kaniko-test-bucket", "The gcs bucket argument to uploaded the tar-ed contents of the `integration` dir to.") flag.StringVar(&c.imageRepo, "repo", "gcr.io/kaniko-test", "The (docker) image repo to build and push images to during the test. `gcloud` must be authenticated with this repo or serviceAccount must be set.") flag.StringVar(&c.serviceAccount, "serviceAccount", "", "The path to the service account push images to GCR and upload/download files to GCS.") - flag.BoolVar(&c.uploadToGCS, "uploadToGCS", true, "Upload the tar-ed contents of `integration` dir to GCS bucket. Default is true. Set this to false to prevent uploading.") flag.Parse() if len(c.serviceAccount) > 0 { @@ -575,8 +585,12 @@ func initGCPConfig() *gcpConfig { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", absPath) } - if c.gcsBucket == "" || c.imageRepo == "" { - log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) and a docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo) + if c.imageRepo == "" { + log.Fatalf("You must provide a docker repo (\"%s\" was provided)", c.imageRepo) + } + + if c.isGcrRepository() && c.gcsBucket == "" { + log.Fatalf("You must provide a gcs bucket (\"%s\" was provided) when using a gcr docker repo (\"%s\" was provided)", c.gcsBucket, c.imageRepo) } if !strings.HasSuffix(c.imageRepo, "/") { c.imageRepo = c.imageRepo + "/" diff --git a/integration/migrate_gcs.go b/integration/migrate_gcs.go new file mode 100644 index 0000000000..d26f777993 --- /dev/null +++ b/integration/migrate_gcs.go @@ -0,0 +1,155 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "path" + "regexp" + "strings" +) + +// This function crawl through dockerfiles and replace all reference to "gcr.io/" in FROM instruction and replace it targetRepo +// The result is the array containing all modified file +// Each of this file is saved +func MigrateGcsRegistry(dockerfilesPath string, dockerfiles []string, targetRepo string) ([]string, error) { + var savedFiles []string + importedImages := map[string]interface{}{} + + for _, dockerfile := range dockerfiles { + if referencedImages, savedFile, err := migrateFile(dockerfilesPath, dockerfile, targetRepo); err != nil { + if savedFile { + savedFiles = append(savedFiles, dockerfile) + } + return savedFiles, err + } else if savedFile { + savedFiles = append(savedFiles, dockerfile) + for _, referencedImage := range referencedImages { + importedImages[referencedImage] = nil + } + } + } + for image := range importedImages { + if err := importImage(image, targetRepo); err != nil { + return savedFiles, err + } + } + return savedFiles, nil +} + +// This function rollback all previously modified files +func RollbackMigratedFiles(dockerfilesPath string, dockerfiles []string) []error { + var result []error + for _, dockerfile := range dockerfiles { + fmt.Printf("Rolling back %s\n", dockerfile) + if err := recoverDockerfile(dockerfilesPath, dockerfile); err != nil { + result = append(result, err) + } + } + return result +} + +// Import the gcr.io image such as gcr.io/my-image to targetRepo +func importImage(image string, targetRepo string) error { + fmt.Printf("Importing %s to %s\n", image, targetRepo) + targetImage := strings.ReplaceAll(image, "gcr.io/", targetRepo) + pullCmd := exec.Command("docker", "pull", image) + if out, err := RunCommandWithoutTest(pullCmd); err != nil { + return fmt.Errorf("Failed to pull image %s with docker command \"%s\": %s %s", image, pullCmd.Args, err, string(out)) + } + + tagCmd := exec.Command("docker", "tag", image, targetImage) + if out, err := RunCommandWithoutTest(tagCmd); err != nil { + return fmt.Errorf("Failed to tag image %s to %s with docker command \"%s\": %s %s", image, targetImage, tagCmd.Args, err, string(out)) + } + + pushCmd := exec.Command("docker", "push", targetImage) + if out, err := RunCommandWithoutTest(pushCmd); err != nil { + return fmt.Errorf("Failed to push image %s with docker command \"%s\": %s %s", targetImage, pushCmd.Args, err, string(out)) + } + return nil +} + +// takes a dockerfile and replace each gcr.io/ occurrence in FROM instruction and replace it with imageRepo +// return true if the file was saved +// if so, the array is non nil and contains each gcr image name +func migrateFile(dockerfilesPath string, dockerfile string, imageRepo string) ([]string, bool, error) { + var input *os.File + var output *os.File + var err error + var referencedImages []string + + if input, err = os.Open(path.Join(dockerfilesPath, dockerfile)); err != nil { + return nil, false, err + } + defer input.Close() + + var lines []string + scanner := bufio.NewScanner(input) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + line := scanner.Text() + if isFromGcrBaseImageInstruction(line) { + referencedImages = append(referencedImages, strings.Trim(strings.Split(line, " ")[1], " ")) + lines = append(lines, strings.ReplaceAll(line, gcrRepoPrefix, imageRepo)) + } else { + lines = append(lines, line) + } + } + rawOutput := []byte(strings.Join(append(lines, ""), "\n")) + + if len(referencedImages) == 0 { + return nil, false, nil + } + + if err = saveDockerfile(dockerfilesPath, dockerfile); err != nil { + return nil, false, err + } + + if output, err = os.Create(path.Join(dockerfilesPath, dockerfile)); err != nil { + return nil, true, err + } + defer output.Close() + + if written, err := output.Write(rawOutput); err != nil { + return nil, true, err + } else if written != len(rawOutput) { + return nil, true, fmt.Errorf("invalid number of byte written. Got %d, expected %d", written, len(rawOutput)) + } + return referencedImages, true, nil + +} + +func isFromGcrBaseImageInstruction(line string) bool { + result, _ := regexp.MatchString(fmt.Sprintf("FROM +%s", gcrRepoPrefix), line) + return result +} + +func saveDockerfile(dockerfilesPath string, dockerfile string) error { + return os.Rename(path.Join(dockerfilesPath, dockerfile), path.Join(dockerfilesPath, saveName(dockerfile))) +} + +func recoverDockerfile(dockerfilesPath string, dockerfile string) error { + return os.Rename(path.Join(dockerfilesPath, saveName(dockerfile)), path.Join(dockerfilesPath, dockerfile)) +} + +func saveName(dockerfile string) string { + return fmt.Sprintf("%s_save_%d", dockerfile, os.Getpid()) +} diff --git a/integration/replace-gcr-with-local-registry.sh b/integration/replace-gcr-with-local-registry.sh deleted file mode 100755 index e40337fdee..0000000000 --- a/integration/replace-gcr-with-local-registry.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This script is needed due to the following bug: -# https://github.com/GoogleContainerTools/kaniko/issues/966 - - -if [ "$#" -ne 1 ]; then - echo "Please specify path to dockerfiles as first argument." - echo "Usage: `basename $0` integration/dockerfiles" - exit 2 -fi - -dir_with_docker_files=$1 - -for dockerfile in $dir_with_docker_files/*; do - cat $dockerfile | grep '^FROM' | grep "gcr" | while read -r line; do - gcr_repo=$(echo "$line" | awk '{ print $2 }') - local_repo=$(echo "$gcr_repo" | sed -e "s/^.*gcr.io\(\/.*\)$/localhost:5000\1/") - remove_digest=$(echo "$local_repo" | cut -f1 -d"@") - echo "Running docker pull $gcr_repo" - docker pull "$gcr_repo" - echo "Running docker tag $gcr_repo $remove_digest" - docker tag "$gcr_repo" "$remove_digest" - echo "Running docker push $remove_digest" - docker push "$remove_digest" - echo "Updating dockerfile $dockerfile to use local repo $local_repo" - sed -i -e "s/^\(FROM \).*gcr.io\(.*\)$/\1localhost:5000\2/" $dockerfile - done -done