refactor(containers): container builds use GCS for cache storage (#14)

* refactor(containers): container builds use GCS for cache storage

This makes a few changes:
1. Before the compilation, we pull down a $repo-$branch.tgz file and
   uncompress it (e.g. igor-master.tgz or rosco-release-1.17.x.tgz) to
   the /workspace volume
2. The Dockerfile.compile image is now expected to be a build step. This
   means we can run the container and it will compile files on the
   /workspace volume. Currently, we copy all the files into the
   container, compile them there, and then have to copy them back out.
   This new method is more along the lines with how GCB is supposed to
   work, but it does mean every Dockerfile.compile has to be
   changed to use CMD instead of RUN.
3. After the compile, we tar up .gradle/caches and .gradle/wrapper to
   $repo-$branch.tgz and send it to gs://spinnaker-build-cache. This
   storage bucket is set to auto-delete files that haven't been
   modified in 14 days.

Removing our dependence on the `gradle_cache` image means that the
various microservices can define their own compile image with different
build tool configurations. I'd really like to have this for the Java 11
migration.

It also means we could move the Debian builds into Cloud Build, since
we aren't depending on them to produce that `gradle_cache` artifact.

* refactor(containers): make the cache bucket configurable

* fix(containers): use {save,restore}_cache from the build project

dev/buildtool/cloudbuild/README.md

@@ -0,0 +1,16 @@
+## Google Cloud Build files
+
+The `containers-build-java8` and `containers-tag-java8` files are
+[Google Cloud Build build configurations](https://cloud.google.com/cloud-build/docs/build-config)
+for building the Spinnaker microservices.
+
+In order to use them, there must be a `save_cache` and `restore_cache` image in
+the Google Container Registry of the project in which the configurations are
+executed. You can create those images with the `build-steps.yml` file:
+
+```
+gcloud builds submit --config=build-steps.yml --project=spinnaker-community .
+```
+
+The source for these images is the
+[cloud-builders-community repository](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/cache).

dev/buildtool/cloudbuild/build-steps.yml

@@ -0,0 +1,12 @@
+# This file publishes the images that are used by containers-*.yml. It should be
+# run manually to create these images.
+steps:
+  - id: fetchCachingRepo
+    waitFor: ["-"]
+    name: gcr.io/cloud-builders/git
+    args: ["clone", "https://github.com/GoogleCloudPlatform/cloud-builders-community"]
+  # The caching repo contains its own cloud build file which will publish 'save_cache' and 'restore_cache' images.
+  - id: buildCachingRepo
+    waitFor: ["fetchCachingRepo"]
+    name: gcr.io/cloud-builders/gcloud
+    args: ["builds", "submit", "--config", "cloud-builders-community/cache/cloudbuild.yaml", "cloud-builders-community/cache"]

dev/buildtool/cloudbuild-build-java8.yml → dev/buildtool/cloudbuild/containers-build-java8.yml

@@ -1,15 +1,26 @@
 steps:
+  - id: restoreCache
+    name: gcr.io/$PROJECT_ID/restore_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
   - id: buildCompileImage
-    waitFor: ["-"]
+    waitFor: ["restoreCache"]
     name: gcr.io/cloud-builders/docker
-    args: [
-            "build",
-            "-t", "compile",
-            "-f", "Dockerfile.compile",
-            ".",
-          ]
-  - id: buildSlimImage
+    args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."]
+  - id: compile
     waitFor: ["buildCompileImage"]
+    name: compile
+  - id: saveCache
+    waitFor: ["compile"]
+    name: gcr.io/$PROJECT_ID/save_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
+      - "--path=.gradle/caches"
+      - "--path=.gradle/wrapper"
+  - id: buildSlimImage
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -19,7 +30,7 @@ steps:
             ".",
           ]
   - id: buildUbuntuImage
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -28,7 +39,7 @@ steps:
             ".",
           ]
   - id: buildJava8Image
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -37,7 +48,7 @@ steps:
             ".",
           ]
   - id: buildUbuntuJava8Image
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -54,3 +65,5 @@ images:
 timeout: 3600s
 options:
   machineType: N1_HIGHCPU_8
+substitutions:
+  _COMPILE_CACHE_BUCKET: spinnaker-build-cache

dev/buildtool/cloudbuild-tag-java8.yml → dev/buildtool/cloudbuild/containers-tag-java8.yml

@@ -1,15 +1,26 @@
 steps:
+  - id: restoreCache
+    name: gcr.io/$PROJECT_ID/restore_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
   - id: buildCompileImage
-    waitFor: ["-"]
+    waitFor: ["restoreCache"]
     name: gcr.io/cloud-builders/docker
-    args: [
-            "build",
-            "-t", "compile",
-            "-f", "Dockerfile.compile",
-            ".",
-          ]
-  - id: buildSlimImage
+    args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."]
+  - id: compile
     waitFor: ["buildCompileImage"]
+    name: compile
+  - id: saveCache
+    waitFor: ["compile"]
+    name: gcr.io/$PROJECT_ID/save_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
+      - "--path=.gradle/caches"
+      - "--path=.gradle/wrapper"
+  - id: buildSlimImage
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -20,7 +31,7 @@ steps:
             ".",
           ]
   - id: buildUbuntuImage
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -38,3 +49,5 @@ images:
 timeout: 3600s
 options:
   machineType: N1_HIGHCPU_8
+substitutions:
+  _COMPILE_CACHE_BUCKET: spinnaker-build-cache

dev/buildtool/container_commands.py

@@ -93,22 +93,25 @@ def __build_with_gcb(self, repository, build_version):
     if os.path.isdir(gradle_cache):
       shutil.rmtree(gradle_cache)
 
-    cloudbuild_file_name = 'cloudbuild-tag-java8.yml'
+    cloudbuild_file_name = 'containers-tag-java8.yml'
     if os.path.exists(os.path.join(git_dir, 'Dockerfile.java8')):
-      cloudbuild_file_name = 'cloudbuild-build-java8.yml'
+      cloudbuild_file_name = 'containers-build-java8.yml'
 
-    cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), cloudbuild_file_name)
+    cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                     'cloudbuild',
+                                     cloudbuild_file_name)
     service_name = self.scm.repository_name_to_service_name(repository.name)
     # Note this command assumes a cwd of git_dir
     command = ('gcloud builds submit '
                ' --account={account} --project={project}'
-               ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry}'
+               ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry},_BRANCH_NAME={branch_name},'
                ' --config={cloudbuild_config} .'
                .format(account=options.gcb_service_account,
                        project=options.gcb_project,
                        docker_registry=options.docker_registry,
                        tag_name=build_version,
                        image_name=service_name,
+                       branch_name=options.git_branch,
                        cloudbuild_config=cloudbuild_config))
 
     logfile = self.get_logfile_path(name + '-gcb-build')
@@ -135,6 +138,7 @@ def init_argparser(self, parser, defaults):
     super(BuildContainerFactory, self).init_argparser(parser, defaults)
 
     self.add_bom_parser_args(parser, defaults)
+    BranchSourceCodeManager.add_parser_args(parser, defaults)
     self.add_argument(
         parser, 'gcb_project', defaults, None,
         help='The GCP project ID that builds the containers when'

dev/buildtool/flow_build.sh

@@ -43,7 +43,7 @@ function run_build_flow() {
   wait_for_commands_or_die "Bom"
 
   start_command_unless NO_CONTAINERS "build_bom_containers" \
-      $EXTRA_BOM_COMMAND_ARGS
+      $EXTRA_BOM_COMMAND_ARGS --git_branch $BOM_BRANCH
   start_command_unless NO_DEBIANS "build_debians" \
       $EXTRA_BOM_COMMAND_ARGS \
       "--max_local_builds=6"