Add GCP image uploads to the pipeline

HACKING.md

@@ -201,6 +201,29 @@ EOF
 oc create secret generic aws-fcos-kola-bot-config --from-file=config=/path/to/kola-secret
 ```
 
+### [OPTIONAL] Creating GCP credentials configs
+
+If you are in production where we upload images to GCP OR you want to
+test uploading to GCP as part of your pipeline development, you need to
+create a upload credentials for a service account as a secret within OpenShift.
+For more information on creating a service account see
+[the Google Cloud Docs](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating).
+
+Once you have the json file that represents the credentials for your service account
+from GCP, create the secret in OpenShift:
+
+```
+oc create secret generic gcp-image-upload-config --from-file=config=/path/to/upload-secret
+```
+
+We also have a second GCP config that can be used for running kola tests. If you have a
+single account that you'd like to use for both image uploading and tests you can do that
+assuming they have enough permissions.
+
+```
+oc create secret generic gcp-kola-tests-config --from-file=config=/path/to/kola-secret
+```
+
 ### [OPTIONAL] Allocating S3 storage
 
 If you want to store builds persistently, now is a good time to allocate

Jenkinsfile

@@ -23,6 +23,7 @@ node {
     src_config_ref = utils.get_pipeline_annotation('source-config-ref')
     s3_bucket = utils.get_pipeline_annotation('s3-bucket')
     kvm_selector = utils.get_pipeline_annotation('kvm-selector')
+    gcp_gs_bucket = utils.get_pipeline_annotation('gcp-gs-bucket')
 
     // sanity check that a valid prefix is provided if in devel mode and drop
     // the trailing '-' in the devel prefix
@@ -383,7 +384,7 @@ lock(resource: "build-${params.STREAM}") {
                 """)
             }
 
-            stage('Build GCE') {
+            stage('Build GCP') {
                 utils.shwrap("""
                 cosa buildextend-gcp
                 """)
@@ -416,6 +417,23 @@ lock(resource: "build-${params.STREAM}") {
                     """)
                 }
             }
+
+            // If there is a config for GCP then we'll upload our image to GCP
+            if (utils.path_exists("\${GCP_IMAGE_UPLOAD_CONFIG}")) {
+                stage('Upload GCP') {
+                    utils.shwrap("""
+                    # pick up the project to use from the config
+                    gcp_project=\$(jq -r .project_id \${GCP_IMAGE_UPLOAD_CONFIG})
+                    cosa buildextend-gcp \
+                        --build=${newBuildID} \
+                        --upload \
+                        --family fedora-coreos-${params.STREAM} \
+                        --project=\${gcp_project} \
+                        --bucket gs://${gcp_gs_bucket}/image-import \
+                        --json \${GCP_IMAGE_UPLOAD_CONFIG}
+                    """)
+                }
+            }
         }
 
         stage('Archive') {

deploy

@@ -100,6 +100,8 @@ def parse_args():
                         help="Repo and ref to use for FCOS config")
     parser.add_argument("--bucket", metavar='BUCKET',
                         help="AWS S3 bucket to use")
+    parser.add_argument("--gcp-gs-bucket", metavar='GCP_GS_BUCKET',
+                        help="GCP GS bucket to use for image uploads during import")
     parser.add_argument("--kvm-selector", help="KVM selector",
                         choices=['kvm-device-plugin', 'legacy-oci-kvm-hook'])
     parser.add_argument("--cosa-img", metavar='FQIN',
@@ -148,6 +150,8 @@ def process_template(args):
         params['PVC_SIZE'] = args.pvc_size
     if args.kvm_selector:
         params['KVM_SELECTOR'] = args.kvm_selector
+    if args.gcp_gs_bucket:
+        params['GCP_GS_BUCKET'] = args.gcp_gs_bucket
 
     print("Parameters:")
     for k, v in params.items():

manifests/pipeline.yaml

@@ -44,6 +44,9 @@ parameters:
   - description: Whether to use KVM device plugin or legacy OCI KVM hook
     name: KVM_SELECTOR
     value: legacy-oci-kvm-hook
+  - description: GCP GS bucket to use for image uploads (or blank for none)
+    name: GCP_GS_BUCKET
+    value: fedora-coreos-cloud-image-uploads
 
 objects:
 
@@ -103,6 +106,7 @@ objects:
         coreos.com/developer-prefix: ${DEVELOPER_PREFIX}
         coreos.com/s3-bucket: ${S3_BUCKET}
         coreos.com/kvm-selector: ${KVM_SELECTOR}
+        coreos.com/gcp-gs-bucket: ${GCP_GS_BUCKET}
     spec:
       # note no triggers: the base pipeline is only ever triggered manually, or
       # by one of the stream-specific pipelines

manifests/pod.yaml

@@ -24,6 +24,10 @@ spec:
          value: /.aws-fcos-builds-bot-config/config
        - name: AWS_FCOS_KOLA_BOT_CONFIG
          value: /.aws-fcos-kola-bot-config/config
+       - name: GCP_IMAGE_UPLOAD_CONFIG
+         value: /.gcp-image-upload-config/config
+       - name: GCP_KOLA_TESTS_CONFIG
+         value: /.gcp-kola-tests-config/config
      volumeMounts:
      - name: cache
        mountPath: /srv/
@@ -33,6 +37,12 @@ spec:
      - name: aws-fcos-kola-bot-config
        mountPath: /.aws-fcos-kola-bot-config/
        readOnly: true
+     - name: gcp-image-upload-config
+       mountPath: /.gcp-image-upload-config/
+       readOnly: true
+     - name: gcp-kola-tests-config
+       mountPath: /.gcp-kola-tests-config/
+       readOnly: true
      - name: github-token
        mountPath: /.github
        readOnly: true
@@ -65,6 +75,16 @@ spec:
     secret:
       secretName: aws-fcos-kola-bot-config
       optional: true
+  # This secret is used for uploading to GCP
+  - name: gcp-image-upload-config
+    secret:
+      secretName: gcp-image-upload-config
+      optional: true
+  # This secret is used for running GCP kola tests
+  - name: gcp-kola-tests-config
+    secret:
+      secretName: gcp-kola-tests-config
+      optional: true
   - name: github-token
     secret:
       secretName: coreosbot-github-token