refactor(containers): container builds use GCS for cache storage

dev/buildtool/cloudbuild/README.md

@@ -0,0 +1,16 @@
+## Google Cloud Build files
+
+The `containers-build-java8` and `containers-tag-java8` files are
+[Google Cloud Build build configurations](https://cloud.google.com/cloud-build/docs/build-config)
+for building the Spinnaker microservices.
+
+In order to use them, there must be a `save_cache` and `restore_cache` image in
+the Google Container Registry of the project in which the configurations are
+executed. You can create those images with the `build-steps.yml` file:
+
+```
+gcloud builds submit --config=build-steps.yml --project=spinnaker-community .
+```
+
+The source for these images is the
+[cloud-builders-community repository](https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/cache).

dev/buildtool/cloudbuild/build-steps.yml

@@ -0,0 +1,12 @@
+# This file publishes the images that are used by containers-*.yml. It should be
+# run manually to create these images.
+steps:
+  - id: fetchCachingRepo
+    waitFor: ["-"]
+    name: gcr.io/cloud-builders/git
+    args: ["clone", "https://github.com/GoogleCloudPlatform/cloud-builders-community"]
+  # The caching repo contains its own cloud build file which will publish 'save_cache' and 'restore_cache' images.
+  - id: buildCachingRepo
+    waitFor: ["fetchCachingRepo"]
+    name: gcr.io/cloud-builders/gcloud
+    args: ["builds", "submit", "--config", "cloud-builders-community/cache/cloudbuild.yaml", "cloud-builders-community/cache"]

dev/buildtool/cloudbuild-build-java8.yml → dev/buildtool/cloudbuild/containers-build-java8.yml

@@ -1,15 +1,26 @@
 steps:
+  - id: restoreCache
+    name: gcr.io/$PROJECT_ID/restore_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
   - id: buildCompileImage
-    waitFor: ["-"]
+    waitFor: ["restoreCache"]
     name: gcr.io/cloud-builders/docker
-    args: [
-            "build",
-            "-t", "compile",
-            "-f", "Dockerfile.compile",
-            ".",
-          ]
-  - id: buildSlimImage
+    args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."]
+  - id: compile
     waitFor: ["buildCompileImage"]
+    name: compile
+  - id: saveCache
+    waitFor: ["compile"]
+    name: gcr.io/$PROJECT_ID/save_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
+      - "--path=.gradle/caches"
+      - "--path=.gradle/wrapper"
+  - id: buildSlimImage
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -19,7 +30,7 @@ steps:
             ".",
           ]
   - id: buildUbuntuImage
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -28,7 +39,7 @@ steps:
             ".",
           ]
   - id: buildJava8Image
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -37,7 +48,7 @@ steps:
             ".",
           ]
   - id: buildUbuntuJava8Image
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -54,3 +65,5 @@ images:
 timeout: 3600s
 options:
   machineType: N1_HIGHCPU_8
+substitutions:
+  _COMPILE_CACHE_BUCKET: spinnaker-build-cache

dev/buildtool/cloudbuild-tag-java8.yml → dev/buildtool/cloudbuild/containers-tag-java8.yml

@@ -1,15 +1,26 @@
 steps:
+  - id: restoreCache
+    name: gcr.io/$PROJECT_ID/restore_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
   - id: buildCompileImage
-    waitFor: ["-"]
+    waitFor: ["restoreCache"]
     name: gcr.io/cloud-builders/docker
-    args: [
-            "build",
-            "-t", "compile",
-            "-f", "Dockerfile.compile",
-            ".",
-          ]
-  - id: buildSlimImage
+    args: ["build", "-t", "compile", "-f", "Dockerfile.compile", "."]
+  - id: compile
     waitFor: ["buildCompileImage"]
+    name: compile
+  - id: saveCache
+    waitFor: ["compile"]
+    name: gcr.io/$PROJECT_ID/save_cache:latest
+    args:
+      - "--bucket=gs://$_COMPILE_CACHE_BUCKET"
+      - "--key=$_IMAGE_NAME-$_BRANCH_NAME"
+      - "--path=.gradle/caches"
+      - "--path=.gradle/wrapper"
+  - id: buildSlimImage
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -20,7 +31,7 @@ steps:
             ".",
           ]
   - id: buildUbuntuImage
-    waitFor: ["buildCompileImage"]
+    waitFor: ["compile"]
     name: gcr.io/cloud-builders/docker
     args: [
             "build",
@@ -38,3 +49,5 @@ images:
 timeout: 3600s
 options:
   machineType: N1_HIGHCPU_8
+substitutions:
+  _COMPILE_CACHE_BUCKET: spinnaker-build-cache

dev/buildtool/container_commands.py

@@ -93,22 +93,25 @@ def __build_with_gcb(self, repository, build_version):
     if os.path.isdir(gradle_cache):
       shutil.rmtree(gradle_cache)
 
-    cloudbuild_file_name = 'cloudbuild-tag-java8.yml'
+    cloudbuild_file_name = 'containers-tag-java8.yml'
     if os.path.exists(os.path.join(git_dir, 'Dockerfile.java8')):
-      cloudbuild_file_name = 'cloudbuild-build-java8.yml'
+      cloudbuild_file_name = 'containers-build-java8.yml'
 
-    cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), cloudbuild_file_name)
+    cloudbuild_config = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                     'cloudbuild',
+                                     cloudbuild_file_name)
     service_name = self.scm.repository_name_to_service_name(repository.name)
     # Note this command assumes a cwd of git_dir
     command = ('gcloud builds submit '
                ' --account={account} --project={project}'
-               ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry}'
+               ' --substitutions=TAG_NAME={tag_name},_IMAGE_NAME={image_name},_DOCKER_REGISTRY={docker_registry},_BRANCH_NAME={branch_name},'
                ' --config={cloudbuild_config} .'
                .format(account=options.gcb_service_account,
                        project=options.gcb_project,
                        docker_registry=options.docker_registry,
                        tag_name=build_version,
                        image_name=service_name,
+                       branch_name=options.git_branch,
                        cloudbuild_config=cloudbuild_config))
 
     logfile = self.get_logfile_path(name + '-gcb-build')
@@ -135,6 +138,7 @@ def init_argparser(self, parser, defaults):
     super(BuildContainerFactory, self).init_argparser(parser, defaults)
 
     self.add_bom_parser_args(parser, defaults)
+    BranchSourceCodeManager.add_parser_args(parser, defaults)
     self.add_argument(
         parser, 'gcb_project', defaults, None,
         help='The GCP project ID that builds the containers when'

dev/buildtool/flow_build.sh

@@ -43,7 +43,7 @@ function run_build_flow() {
   wait_for_commands_or_die "Bom"
 
   start_command_unless NO_CONTAINERS "build_bom_containers" \
-      $EXTRA_BOM_COMMAND_ARGS
+      $EXTRA_BOM_COMMAND_ARGS --git_branch $BOM_BRANCH
   start_command_unless NO_DEBIANS "build_debians" \
       $EXTRA_BOM_COMMAND_ARGS \
       "--max_local_builds=6"