CI Framework Addition

.ci/docker/Dockerfile_dswx_hls

@@ -54,9 +54,10 @@ USER 0:0
 RUN set -ex \
     && cd ${PGE_DEST_DIR} \
     && mkdir -p /opt/conda/bin \
-    && cp ${PGE_DEST_DIR}/opera/scripts/pge_docker_entrypoint.sh /opt/conda/bin \
-    && chmod +x /opt/conda/bin/pge_docker_entrypoint.sh
-
+    && cp ${PGE_DEST_DIR}/opera/scripts/*_entrypoint.sh /opt/conda/bin \
+    && chmod +x /opt/conda/bin/*_entrypoint.sh \
+    && source /opt/conda/bin/activate root \
+    && conda install --yes --channel conda-forge --file ${PGE_DEST_DIR}/opera/requirements.txt
 
 # Set the Docker entrypoint and clear the default command
 ENTRYPOINT ["/opt/conda/bin/pge_docker_entrypoint.sh"]

.ci/jenkins/Jenkinsfile

@@ -0,0 +1,143 @@
+#!/usr/bin/env groovy
+
+// Copyright 2021, by the California Institute of Technology.
+// ALL RIGHTS RESERVED.
+// United States Government sponsorship acknowledged.
+// Any commercial use must be negotiated with the Office of Technology Transfer
+// at the California Institute of Technology.
+// This software may be subject to U.S. export control laws and regulations.
+// By accepting this document, the user agrees to comply with all applicable
+// U.S. export laws and regulations. User has the responsibility to obtain
+// export licenses, or other export authority as may be required, before
+// exporting such information to foreign countries or providing access to
+// foreign persons.
+
+pipeline {
+    agent any
+    parameters {
+        string(name: 'ART_URL', defaultValue: 'https://artifactory-fn.jpl.nasa.gov/artifactory',
+               description: 'Artifactory-FN URL.')
+        string(name: 'ART_TAR_PATH', defaultValue: 'general/gov/nasa/jpl/opera/sds/pge/',
+               description: 'Artifactory path to publish PGE docker image tar files to.')
+        string(name: 'ART_DOCKER_PATH', defaultValue: '/gov/nasa/jpl/opera/sds/pge/',
+               description: 'Artifactory path to push Docker images.')
+        string(name: 'ART_DOCKER_REGISTRY', defaultValue: 'artifactory-fn.jpl.nasa.gov:16001',
+               description: 'Address of Artifactory-FN Docker registry for uploading Docker images.')
+        credentials (name: 'ART_CREDENTIALS',
+                     defaultValue: 'collinss-artifactory-fn-credentials',
+                     credentialType: 'com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl',
+                     description: 'Artifactory-FN credentials for account collinss. Used to push/pull images from Artifactory during build.',
+                     required: true)
+    }
+    environment {
+        DOCKER_IMAGE_PREFIX = 'opera_pge'
+        DOCKER_IMAGE_SUFFIXES = 'dswx_hls'
+        DOCKER_TAG = """${sh(
+                     returnStdout: true,
+                     script: 'echo ${GIT_BRANCH##*/}'
+                     ).trim()}"""
+    }
+    stages {
+        stage('Build OPERA PGE Docker image(s)') {
+            steps {
+                script {
+                    docker.withRegistry ('https://' + params.ART_DOCKER_REGISTRY, params.ART_CREDENTIALS) {
+                        echo "Building ${DOCKER_IMAGE_PREFIX} docker images with tag ${DOCKER_TAG}"
+                        sh label: 'Build all OPERA Docker images',
+                           script: ".ci/scripts/build_all_images.sh $DOCKER_TAG"
+                    }
+                }
+            }
+        }
+        stage('Test OPERA PGE Docker image(s)') {
+            steps {
+                echo "Testing ${DOCKER_IMAGE_PREFIX} docker images with tag ${DOCKER_TAG}"
+                sh label: 'Test all OPERA Docker images',
+                   script: ".ci/scripts/test_all_images.sh $DOCKER_TAG"
+                junit 'test_results/**/*.xml'
+                archiveArtifacts artifacts: 'test_results/**/*.log', fingerprint: true
+                // TODO add HTML publishing of test coverage reports once plugin is installed
+            }
+        }
+        stage('Upload OPERA PGE Docker image tar files to Artifactory-FN') {
+            steps {
+                script {
+                    rtServer (
+                        id: 'ARTIFACTORY_FN_SERVER',
+                        url: params.ART_URL,
+                        credentialsId: params.ART_CREDENTIALS,
+                        timeout: 300
+                    )
+
+                    DOCKER_IMAGE_SUFFIXES.tokenize(',').each { DOCKER_IMAGE_SUFFIX ->
+                        DOCKER_IMAGE = "${DOCKER_IMAGE_PREFIX}/${DOCKER_IMAGE_SUFFIX}:${DOCKER_TAG}"
+                        TAR_FILE_NAME = "${DOCKER_IMAGE_PREFIX}-${DOCKER_IMAGE_SUFFIX}-${DOCKER_TAG}.tar"
+                        echo "Saving Docker image ${DOCKER_IMAGE} to tar file ${TAR_FILE_NAME}"
+                        sh "docker save -o ${TAR_FILE_NAME} ${DOCKER_IMAGE}"
+                        sh "gzip -f ${TAR_FILE_NAME}"
+                        TAR_GZ_FILE_NAME = "${TAR_FILE_NAME}.gz"
+
+                        echo "Uploading Docker image tar file to Artifactory-FN"
+
+                        rtUpload(
+                            serverId: "ARTIFACTORY_FN_SERVER",
+                            spec:
+                              """{
+                                  "files": [
+                                      {
+                                          "pattern": "${TAR_GZ_FILE_NAME}",
+                                          "target": "${params.ART_TAR_PATH}"
+                                      }
+                                  ]
+                              }"""
+                        )
+
+                        sh "rm -f ${TAR_FILE_NAME} ${TAR_GZ_FILE_NAME}"
+                    }
+                }
+            }
+        }
+        stage('Upload Docker images to Artifactory FN Docker Registry') {
+            steps {
+                script {
+                    rtServer (
+                        id: 'ARTIFACTORY_FN_DOCKER_REGISTRY',
+                        url: params.ART_DOCKER_REGISTRY,
+                        credentialsId: params.ART_CREDENTIALS,
+                        timeout: 300
+                    )
+
+                    DOCKER_IMAGE_SUFFIXES.tokenize(',').each { DOCKER_IMAGE_SUFFIX ->
+                        DOCKER_IMAGE = "${DOCKER_IMAGE_PREFIX}/${DOCKER_IMAGE_SUFFIX}:${DOCKER_TAG}"
+                        sh "docker tag ${DOCKER_IMAGE} ${ART_DOCKER_REGISTRY}${ART_DOCKER_PATH}${DOCKER_IMAGE}"
+                        echo "Publishing Docker Image ${DOCKER_IMAGE} to Artifactory FN Docker Registry"
+                        rtDockerPush(
+                            serverId: "ARTIFACTORY_FN_DOCKER_REGISTRY",
+                            image: "${ART_DOCKER_REGISTRY}${ART_DOCKER_PATH}${DOCKER_IMAGE}",
+                            targetRepo: "docker-develop-local"
+                        )
+                    }
+                }
+            }
+        }
+    }
+    post {
+        always {
+            echo "Cleaning up Docker images from local host"
+            sh ".ci/scripts/cleanup.sh ${DOCKER_TAG}"
+            deleteDir()
+        }
+        success {
+            echo 'Succeeded!'
+        }
+        unstable {
+            echo 'Unstable :/'
+        }
+        failure {
+            echo 'Failed :('
+        }
+        changed {
+            echo 'Things were different before...'
+        }
+    }
+}

.ci/scripts/build_all_images.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Copyright 2021, by the California Institute of Technology.
+# ALL RIGHTS RESERVED.
+# United States Government sponsorship acknowledged.
+# Any commercial use must be negotiated with the Office of Technology Transfer
+# at the California Institute of Technology.
+# This software may be subject to U.S. export control laws and regulations.
+# By accepting this document, the user agrees to comply with all applicable
+# U.S. export laws and regulations. User has the responsibility to obtain
+# export licenses, or other export authority as may be required, before
+# exporting such information to foreign countries or providing access to
+# foreign persons.
+
+#
+# Script to build all OPERA PGE Docker images
+#
+
+echo '
+=======================================
+
+Building all OPERA PGE docker images...
+
+=======================================
+'
+
+TAG=$1
+
+# defaults
+[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..)
+[ -z "${TAG}" ] && TAG="${USER}-dev"
+
+echo "WORKSPACE: $WORKSPACE"
+echo "TAG: $TAG"
+
+# check .ci scripts directory exists
+if [ ! -d "${WORKSPACE}/.ci" ]; then
+  echo "Error: the .ci directory doesn't exist at ${WORKSPACE}/.ci"
+  exit 1
+fi
+
+# Build all of the Docker images
+BUILD_SCRIPTS_DIR=${WORKSPACE}/.ci/scripts
+${BUILD_SCRIPTS_DIR}/build_dswx_hls.sh ${TAG}
+
+echo 'Build Complete'
+
+exit 0

.ci/scripts/build_dswx_hls.sh

@@ -23,8 +23,6 @@ Building DSWx-HLS PGE docker image...
 =====================================
 '
 
-set -e
-
 IMAGE='opera_pge/dswx_hls'
 TAG=$1
 WORKSPACE=$2
@@ -65,6 +63,15 @@ cp -r ${WORKSPACE}/src/opera \
 cp ${WORKSPACE}/COPYING \
    ${STAGING_DIR}/opera
 
+cp ${WORKSPACE}/requirements.txt \
+   ${STAGING_DIR}/opera
+
+cp ${WORKSPACE}/.flake8 \
+   ${STAGING_DIR}/opera
+
+cp ${WORKSPACE}/.pylintrc \
+   ${STAGING_DIR}/opera
+
 # Create a VERSION file in the staging area to track version and build time
 printf "pge_version: ${TAG}\npge_build_datetime: ${BUILD_DATE_TIME}\n" \
     > ${STAGING_DIR}/opera/VERSION \
@@ -82,4 +89,6 @@ docker build --rm --force-rm -t ${IMAGE}:${TAG} \
     --build-arg PGE_SOURCE_DIR=$(basename ${STAGING_DIR}) \
     --file ${WORKSPACE}/.ci/docker/Dockerfile_dswx_hls ${WORKSPACE}
 
-exit $?
+echo "DSWx-HLS PGE Docker image build complete"
+
+exit 0

.ci/scripts/cleanup.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2021, by the California Institute of Technology.
+# ALL RIGHTS RESERVED.
+# United States Government sponsorship acknowledged.
+# Any commercial use must be negotiated with the Office of Technology Transfer
+# at the California Institute of Technology.
+# This software may be subject to U.S. export control laws and regulations.
+# By accepting this document, the user agrees to comply with all applicable
+# U.S. export laws and regulations. User has the responsibility to obtain
+# export licenses, or other export authority as may be required, before
+# exporting such information to foreign countries or providing access to
+# foreign persons.
+
+# Script to clean up Docker images from the CI pipeline instance
+
+# command line args
+TAG=$1
+
+# defaults
+[ -z "${TAG}" ] && TAG="${USER}-dev"
+
+# Remove docker images with the specified tag
+echo "Removing all Docker images with tag ${TAG}..."
+for i in {1..2}
+do
+  docker images | grep ${TAG} | awk '{print $3}' | xargs -r docker rmi -f
+  docker images -q --filter "label=org.label-schema.version=${TAG}" | xargs -r docker rmi -f
+done

.ci/scripts/test_all_images.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Copyright 2021, by the California Institute of Technology.
+# ALL RIGHTS RESERVED.
+# United States Government sponsorship acknowledged.
+# Any commercial use must be negotiated with the Office of Technology Transfer
+# at the California Institute of Technology.
+# This software may be subject to U.S. export control laws and regulations.
+# By accepting this document, the user agrees to comply with all applicable
+# U.S. export laws and regulations. User has the responsibility to obtain
+# export licenses, or other export authority as may be required, before
+# exporting such information to foreign countries or providing access to
+# foreign persons.
+
+#
+# Script to test all OPERA PGE Docker images
+#
+
+echo '
+=======================================
+
+Testing all OPERA PGE docker images...
+
+=======================================
+'
+
+TAG=$1
+
+# defaults
+[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..)
+[ -z "${TAG}" ] && TAG="${USER}-dev"
+
+echo "WORKSPACE: $WORKSPACE"
+echo "TAG: $TAG"
+
+# check .ci scripts directory exists
+if [ ! -d "${WORKSPACE}/.ci" ]; then
+  echo "Error: the .ci directory doesn't exist at ${WORKSPACE}/.ci"
+  exit 1
+fi
+
+# Build all of the Docker images
+BUILD_SCRIPTS_DIR=${WORKSPACE}/.ci/scripts
+${BUILD_SCRIPTS_DIR}/test_dswx_hls.sh ${TAG}
+
+echo 'Build Complete'
+
+exit 0

.ci/scripts/test_dswx_hls.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+# Copyright 2021, by the California Institute of Technology.
+# ALL RIGHTS RESERVED.
+# United States Government sponsorship acknowledged.
+# Any commercial use must be negotiated with the Office of Technology Transfer
+# at the California Institute of Technology.
+# This software may be subject to U.S. export control laws and regulations.
+# By accepting this document, the user agrees to comply with all applicable
+# U.S. export laws and regulations. User has the responsibility to obtain
+# export licenses, or other export authority as may be required, before
+# exporting such information to foreign countries or providing access to
+# foreign persons.
+#
+# Script to execute tests on OPERA DSWx-HLS PGE Docker image
+#
+
+echo '
+=====================================
+
+Testing DSWx-HLS PGE Docker image...
+
+=====================================
+'
+
+IMAGE="opera_pge/dswx_hls"
+TAG=$1
+WORKSPACE=$2
+TEST_RESULTS_REL_DIR="test_results/dswx_hls"
+
+# defaults
+[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..)
+[ -z "${TAG}" ] && TAG="${USER}-dev"
+
+echo "Test results output directory:  ${WORKSPACE}/${TEST_RESULTS_REL_DIR}"
+mkdir --mode=777 --parents ${WORKSPACE}/${TEST_RESULTS_REL_DIR}
+
+# Use the environment of the docker image to run linting, tests, etc...
+DOCKER_RUN="docker run --rm \
+    -v ${WORKSPACE}:/workspace \
+    -u conda:conda \
+    --entrypoint /opt/conda/bin/pge_tests_entrypoint.sh \
+    ${IMAGE}:${TAG}"
+
+# linting and pep8 style check (configured by .flake8 and .pylintrc)
+${DOCKER_RUN} flake8 \
+    --config /home/conda/opera/.flake8 \
+    --jobs auto \
+    --application-import-names opera \
+    --output-file /workspace/${TEST_RESULTS_REL_DIR}/flake8.log \
+    /home/conda/opera
+
+${DOCKER_RUN} pylint \
+    --rcfile=/home/conda/opera/.pylintrc \
+    --jobs 0 \
+    --output=/workspace/${TEST_RESULTS_REL_DIR}/pylint.log \
+    --enable-all-extensions \
+    /home/conda/opera
+
+# pytest (including code coverage)
+${DOCKER_RUN} bash -c "pytest \
+    --junit-xml=/workspace/${TEST_RESULTS_REL_DIR}/pytest-junit.xml \
+    --cov=/home/conda/opera/pge \
+    --cov=/home/conda/opera/scripts \
+    --cov=/home/conda/opera/util \
+    --cov-report=term \
+    --cov-report=html:/workspace/${TEST_RESULTS_REL_DIR}/coverage_html \
+    /home/conda/opera > /workspace/${TEST_RESULTS_REL_DIR}/pytest.log 2>&1"
+
+# Open up permissions on all output reports so the CI system can delete them
+# after they're archived within Jenkins
+${DOCKER_RUN} bash -c "chmod \
+    -R 777 /workspace/${TEST_RESULTS_REL_DIR}/"
+
+echo "DSWx-HLS PGE Docker image test complete"
+
+exit 0