From e599477c9d82137a6beb37fb9a2dd2e56c8ce374 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:25:02 -0600 Subject: [PATCH] Move Docker scripts to be consistent with other METplus components (#1907) --- .github/jobs/create_output_data_volumes.sh | 2 +- .../jobs}/docker_data_output/Dockerfile | 0 .github/jobs/docker_setup.sh | 4 +- .github/jobs/docker_update_data_volumes.py | 138 --------- .../continuous_integration.rst | 6 +- docs/Users_Guide/installation.rst | 11 +- .../scripts}/docker/Dockerfile | 0 .../scripts}/docker/hooks/build | 0 .../scripts}/docker/hooks/get_met_version | 2 +- .../scripts}/docker_env/Dockerfile | 0 .../scripts}/docker_env/Dockerfile.gempak_env | 0 .../docker_env/Dockerfile.gfdl-tracker | 0 .../docker_env/Dockerfile.metplus_base | 0 .../docker_env/Dockerfile.py_embed_base | 0 .../scripts}/docker_env/README.md | 2 +- .../scripts}/docker_env/scripts/cfgrib_env.sh | 0 .../docker_env/scripts/cycloneplotter_env.sh | 0 .../scripts}/docker_env/scripts/diff_env.sh | 0 .../scripts}/docker_env/scripts/gempak_env.sh | 0 .../scripts}/docker_env/scripts/h5py_env.sh | 0 .../docker_env/scripts/icecover_env.sh | 0 .../docker_env/scripts/metdataio_env.sh | 0 .../docker_env/scripts/metplotpy_env.sh | 0 .../docker_env/scripts/metplus_base_env.sh | 0 .../docker_env/scripts/netcdf4_env.sh | 0 .../docker_env/scripts/py_embed_base_env.sh | 0 .../scripts}/docker_env/scripts/pygrib_env.sh | 0 .../scripts}/docker_env/scripts/pytest_env.sh | 0 .../docker_env/scripts/spacetime_env.sh | 0 .../docker_env/scripts/weatherregime_env.sh | 0 .../scripts}/docker_env/scripts/xesmf_env.sh | 0 scripts/docker/docker_data/Dockerfile | 33 --- .../docker/docker_data/build_docker_images.sh | 271 ------------------ scripts/docker/docker_data/hooks/build | 3 - 34 files changed, 10 insertions(+), 462 deletions(-) rename {scripts/docker => .github/jobs}/docker_data_output/Dockerfile (100%) delete mode 100755 .github/jobs/docker_update_data_volumes.py rename {scripts => internal/scripts}/docker/Dockerfile (100%) rename {scripts => internal/scripts}/docker/hooks/build (100%) rename {scripts => internal/scripts}/docker/hooks/get_met_version (81%) rename {scripts/docker => internal/scripts}/docker_env/Dockerfile (100%) rename {scripts/docker => internal/scripts}/docker_env/Dockerfile.gempak_env (100%) rename {scripts/docker => internal/scripts}/docker_env/Dockerfile.gfdl-tracker (100%) rename {scripts/docker => internal/scripts}/docker_env/Dockerfile.metplus_base (100%) rename {scripts/docker => internal/scripts}/docker_env/Dockerfile.py_embed_base (100%) rename {scripts/docker => internal/scripts}/docker_env/README.md (99%) rename {scripts/docker => internal/scripts}/docker_env/scripts/cfgrib_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/cycloneplotter_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/diff_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/gempak_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/h5py_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/icecover_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/metdataio_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/metplotpy_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/metplus_base_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/netcdf4_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/py_embed_base_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/pygrib_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/pytest_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/spacetime_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/weatherregime_env.sh (100%) rename {scripts/docker => internal/scripts}/docker_env/scripts/xesmf_env.sh (100%) delete mode 100644 scripts/docker/docker_data/Dockerfile delete mode 100755 scripts/docker/docker_data/build_docker_images.sh delete mode 100644 scripts/docker/docker_data/hooks/build diff --git a/.github/jobs/create_output_data_volumes.sh b/.github/jobs/create_output_data_volumes.sh index 38f55b7dea..abf3e7e8a7 100755 --- a/.github/jobs/create_output_data_volumes.sh +++ b/.github/jobs/create_output_data_volumes.sh @@ -28,7 +28,7 @@ if [ $? != 0 ]; then exit 1 fi -docker_data_output_dir=scripts/docker/docker_data_output +docker_data_output_dir=.github/jobs/docker_data_output success=1 for vol_name in use_cases_*; do diff --git a/scripts/docker/docker_data_output/Dockerfile b/.github/jobs/docker_data_output/Dockerfile similarity index 100% rename from scripts/docker/docker_data_output/Dockerfile rename to .github/jobs/docker_data_output/Dockerfile diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 31b996c9a4..ad20f64ecc 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -29,9 +29,9 @@ duration=$(( SECONDS - start_seconds )) echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS)" # set DOCKERFILE_PATH that is used by docker hook script get_met_version -export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/scripts/docker/Dockerfile +export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile -MET_TAG=`${GITHUB_WORKSPACE}/scripts/docker/hooks/get_met_version` +MET_TAG=`${GITHUB_WORKSPACE}/internal/scripts/docker/hooks/get_met_version` MET_DOCKER_REPO=met-dev if [ "${MET_TAG}" != "develop" ]; then diff --git a/.github/jobs/docker_update_data_volumes.py b/.github/jobs/docker_update_data_volumes.py deleted file mode 100755 index d19d731e4c..0000000000 --- a/.github/jobs/docker_update_data_volumes.py +++ /dev/null @@ -1,138 +0,0 @@ -#! /usr/bin/env python3 - -# Run by GitHub Actions (in .github/workflows/testing.yml) check DTCenter web -# server for any input data tarfiles that have been updated and need to be -# regenerated as Docker data volumes to be used in use case tests. -# Push new/updated data volumes up to DockerHub - -import sys -import os -import shlex -import requests -from bs4 import BeautifulSoup -import dateutil.parser -from urllib.parse import urljoin -import subprocess - -from docker_utils import docker_get_volumes_last_updated, get_data_repo -from docker_utils import get_branch_name - -# URL containing METplus sample data tarfiles -WEB_DATA_DIR = 'https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data/' - -# path to script that builds docker data volumes -BUILD_DOCKER_IMAGES = os.path.join(os.environ.get('GITHUB_WORKSPACE', ''), - 'scripts', - 'docker', - 'docker_data', - 'build_docker_images.sh') - -def get_tarfile_last_modified(search_dir): - # get list of tarfiles from website - soup = BeautifulSoup(requests.get(search_dir).content, - 'html.parser') - tarfiles = [a_tag.get_text() for a_tag in soup.find_all('a') - if a_tag.get_text().startswith('sample_data')] - - # get last modified time of each tarfile - tarfile_last_modified = {} - for tarfile in tarfiles: - tarfile_url = urljoin(search_dir+'/', tarfile) - last_modified = requests.head(tarfile_url).headers['last-modified'] - tarfile_last_modified[tarfile] = last_modified - - return tarfile_last_modified - -def create_data_volumes(branch_name, volumes): - if not volumes: - print("No volumes to build") - return True - - data_repo = get_data_repo(branch_name) - # log into docker using encrypted credentials and - # call build_docker_images.sh script - cmd = (f'{BUILD_DOCKER_IMAGES} -pull {branch_name} ' - f'-data {",".join(volumes)} -push {data_repo}') - print(f'Running command: {cmd}') - ret = subprocess.run(shlex.split(cmd), check=True) - - if ret.returncode: - print(f'ERROR: Command failed: {cmd}') - return False - - return True - -def main(): - - if not os.environ.get('DOCKER_USERNAME'): - print("DockerHub credentials are not stored. " - "Skipping data volume handling.") - sys.exit(0) - - # check if tarfile directory exists on web - branch_name = get_branch_name() - if not branch_name: - print("Could not get current branch. Exiting.") - sys.exit(1) - - if branch_name.endswith('-ref'): - branch_name = branch_name[0:-4] - - # search dir should be develop, feature_NNN, or vX.Y - if branch_name.startswith('main_v'): - web_subdir = branch_name[5:] - else: - web_subdir = branch_name - - if not os.environ.get('GITHUB_WORKSPACE'): - print("GITHUB_WORKSPACE not set. Exiting.") - sys.exit(1) - - search_dir = f"{urljoin(WEB_DATA_DIR, web_subdir)}/" - print(f"Looking for tarfiles in {search_dir}") - - dir_request = requests.get(search_dir) - # if it does not exist, exit script - if dir_request.status_code != 200: - print(f'URL does not exist: {search_dir}') - sys.exit(0) - - # get last modified time of each tarfile - tarfile_last_modified = get_tarfile_last_modified(search_dir) - - volumes_last_updated = docker_get_volumes_last_updated(branch_name) - - # check status of each tarfile and add them to the list of volumes to create if needed - volumes_to_create = [] - for tarfile, last_modified in tarfile_last_modified.items(): - category = os.path.splitext(tarfile)[0].split('-')[1] - print(f"Checking tarfile: {category}") - - volume_name = f'{branch_name}-{category}' - # if the data volume does not exist, create it and push it to DockerHub - if not volume_name in volumes_last_updated.keys(): - print(f'{volume_name} data volume does not exist. Creating data volume.') - volumes_to_create.append(category) - continue - - # if data volume does exist, get last updated time of volume and compare to - # tarfile last modified. if any tarfile was modified after creation of - # corresponding volume, recreate those data volumes - volume_dt = dateutil.parser.parse(volumes_last_updated[volume_name]) - tarfile_dt = dateutil.parser.parse(last_modified) - - print(f"Volume time: {volume_dt.strftime('%Y%m%d %H:%M:%S')}") - print(f"Tarfile time: {tarfile_dt.strftime('%Y%m%d %H:%M:%S')}") - - # if the tarfile has been modified more recently than the data volume was created, - # recreate the data volume - if volume_dt < tarfile_dt: - print(f'{tarfile} has changed since {volume_name} was created. ' - 'Regenerating data volume.') - volumes_to_create.append(category) - - if not create_data_volumes(branch_name, volumes_to_create): - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/docs/Contributors_Guide/continuous_integration.rst b/docs/Contributors_Guide/continuous_integration.rst index bed495e39c..fad214a532 100644 --- a/docs/Contributors_Guide/continuous_integration.rst +++ b/docs/Contributors_Guide/continuous_integration.rst @@ -685,7 +685,7 @@ images using Conda that can be used to run use cases. These images are stored on DockerHub in *dtcenter/metplus-envs* and are named with a tag that corresponds to the keyword without the **_env** suffix. The environments were created using Docker commands via scripts that are found -in *scripts/docker/docker_env*. +in *internal/scripts/docker_env*. Existing keywords that set up Conda environments used for use cases are: * cfgrib_env @@ -793,14 +793,14 @@ packages take a very long time to install in Docker. The new approach involves creating Docker images that use Conda to create a Python environment that can run the use case. To see what is available in each of the existing Python environments, refer to the comments in the scripts found in -*scripts/docker/docker_env/scripts*. +*internal/scripts/docker_env/scripts*. New environments must be added by a METplus developer, so please create a discussion on the `METplus GitHub Discussions `_ forum if none of these environments contain the package requirements needed to run a new use case. -A **README.md** file can be found in *scripts/docker/docker_env* that +A **README.md** file can be found in *internal/scripts/docker_env* that provides commands that can be run to recreate a Docker image if the conda environment needs to be updated. Please note that Docker must be installed on the workstation used to create new Docker images and diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index dcde8cefa1..18a248e4f9 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -221,16 +221,11 @@ The METplus Wrappers source code contains the following directory structure:: METplus/ build_components/ docs/ - environment.yml internal/ manage_exernals/ metplus/ parm/ produtil/ - README.md - requirements.txt - scripts/ - setup.py ush/ The top-level METplus Wrappers directory consists of a README.md file @@ -249,7 +244,8 @@ The Doxygen documentation is useful to contributors and is not necessary for METplus end-users. The **internal/** directory contains scripts that are only -relevant to METplus developers and contributors. +relevant to METplus developers and contributors, such as tests and files +used with Docker. The **manage_externals/** directory contains scripts used to facilitate the downloading and management @@ -262,9 +258,6 @@ METplus Wrappers. The **produtil/** directory contains part of the external utility produtil. -The **scripts/** directory contains scripts that are used for creating -Docker images. - The **ush/** directory contains the run_metplus.py script that is executed to run use cases. diff --git a/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile similarity index 100% rename from scripts/docker/Dockerfile rename to internal/scripts/docker/Dockerfile diff --git a/scripts/docker/hooks/build b/internal/scripts/docker/hooks/build similarity index 100% rename from scripts/docker/hooks/build rename to internal/scripts/docker/hooks/build diff --git a/scripts/docker/hooks/get_met_version b/internal/scripts/docker/hooks/get_met_version similarity index 81% rename from scripts/docker/hooks/get_met_version rename to internal/scripts/docker/hooks/get_met_version index 6cb53c9ad8..f580b55cb9 100755 --- a/scripts/docker/hooks/get_met_version +++ b/internal/scripts/docker/hooks/get_met_version @@ -1,7 +1,7 @@ #!/bin/bash # get version, use develop or X+6.Y for MET_BRANCH -version_file=$(dirname $DOCKERFILE_PATH)/../../metplus/VERSION +version_file=$(dirname $DOCKERFILE_PATH)/../../../metplus/VERSION if cat $version_file | egrep -q '^[0-9.]+$'; then let major=$(cut -d '.' -f1 $version_file)+6 diff --git a/scripts/docker/docker_env/Dockerfile b/internal/scripts/docker_env/Dockerfile similarity index 100% rename from scripts/docker/docker_env/Dockerfile rename to internal/scripts/docker_env/Dockerfile diff --git a/scripts/docker/docker_env/Dockerfile.gempak_env b/internal/scripts/docker_env/Dockerfile.gempak_env similarity index 100% rename from scripts/docker/docker_env/Dockerfile.gempak_env rename to internal/scripts/docker_env/Dockerfile.gempak_env diff --git a/scripts/docker/docker_env/Dockerfile.gfdl-tracker b/internal/scripts/docker_env/Dockerfile.gfdl-tracker similarity index 100% rename from scripts/docker/docker_env/Dockerfile.gfdl-tracker rename to internal/scripts/docker_env/Dockerfile.gfdl-tracker diff --git a/scripts/docker/docker_env/Dockerfile.metplus_base b/internal/scripts/docker_env/Dockerfile.metplus_base similarity index 100% rename from scripts/docker/docker_env/Dockerfile.metplus_base rename to internal/scripts/docker_env/Dockerfile.metplus_base diff --git a/scripts/docker/docker_env/Dockerfile.py_embed_base b/internal/scripts/docker_env/Dockerfile.py_embed_base similarity index 100% rename from scripts/docker/docker_env/Dockerfile.py_embed_base rename to internal/scripts/docker_env/Dockerfile.py_embed_base diff --git a/scripts/docker/docker_env/README.md b/internal/scripts/docker_env/README.md similarity index 99% rename from scripts/docker/docker_env/README.md rename to internal/scripts/docker_env/README.md index 521d071846..516a430613 100644 --- a/scripts/docker/docker_env/README.md +++ b/internal/scripts/docker_env/README.md @@ -1,6 +1,6 @@ # Docker Conda Environments -Run the commands from this directory (scripts/docker/docker_env). +Run the commands from this directory (internal/scripts/docker_env). Instructions include how to create Docker images in dtcenter/metplus-envs so environments are available for the automated tests. Instructions to create these Conda environments on a local machine are also provided. diff --git a/scripts/docker/docker_env/scripts/cfgrib_env.sh b/internal/scripts/docker_env/scripts/cfgrib_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/cfgrib_env.sh rename to internal/scripts/docker_env/scripts/cfgrib_env.sh diff --git a/scripts/docker/docker_env/scripts/cycloneplotter_env.sh b/internal/scripts/docker_env/scripts/cycloneplotter_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/cycloneplotter_env.sh rename to internal/scripts/docker_env/scripts/cycloneplotter_env.sh diff --git a/scripts/docker/docker_env/scripts/diff_env.sh b/internal/scripts/docker_env/scripts/diff_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/diff_env.sh rename to internal/scripts/docker_env/scripts/diff_env.sh diff --git a/scripts/docker/docker_env/scripts/gempak_env.sh b/internal/scripts/docker_env/scripts/gempak_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/gempak_env.sh rename to internal/scripts/docker_env/scripts/gempak_env.sh diff --git a/scripts/docker/docker_env/scripts/h5py_env.sh b/internal/scripts/docker_env/scripts/h5py_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/h5py_env.sh rename to internal/scripts/docker_env/scripts/h5py_env.sh diff --git a/scripts/docker/docker_env/scripts/icecover_env.sh b/internal/scripts/docker_env/scripts/icecover_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/icecover_env.sh rename to internal/scripts/docker_env/scripts/icecover_env.sh diff --git a/scripts/docker/docker_env/scripts/metdataio_env.sh b/internal/scripts/docker_env/scripts/metdataio_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/metdataio_env.sh rename to internal/scripts/docker_env/scripts/metdataio_env.sh diff --git a/scripts/docker/docker_env/scripts/metplotpy_env.sh b/internal/scripts/docker_env/scripts/metplotpy_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/metplotpy_env.sh rename to internal/scripts/docker_env/scripts/metplotpy_env.sh diff --git a/scripts/docker/docker_env/scripts/metplus_base_env.sh b/internal/scripts/docker_env/scripts/metplus_base_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/metplus_base_env.sh rename to internal/scripts/docker_env/scripts/metplus_base_env.sh diff --git a/scripts/docker/docker_env/scripts/netcdf4_env.sh b/internal/scripts/docker_env/scripts/netcdf4_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/netcdf4_env.sh rename to internal/scripts/docker_env/scripts/netcdf4_env.sh diff --git a/scripts/docker/docker_env/scripts/py_embed_base_env.sh b/internal/scripts/docker_env/scripts/py_embed_base_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/py_embed_base_env.sh rename to internal/scripts/docker_env/scripts/py_embed_base_env.sh diff --git a/scripts/docker/docker_env/scripts/pygrib_env.sh b/internal/scripts/docker_env/scripts/pygrib_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/pygrib_env.sh rename to internal/scripts/docker_env/scripts/pygrib_env.sh diff --git a/scripts/docker/docker_env/scripts/pytest_env.sh b/internal/scripts/docker_env/scripts/pytest_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/pytest_env.sh rename to internal/scripts/docker_env/scripts/pytest_env.sh diff --git a/scripts/docker/docker_env/scripts/spacetime_env.sh b/internal/scripts/docker_env/scripts/spacetime_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/spacetime_env.sh rename to internal/scripts/docker_env/scripts/spacetime_env.sh diff --git a/scripts/docker/docker_env/scripts/weatherregime_env.sh b/internal/scripts/docker_env/scripts/weatherregime_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/weatherregime_env.sh rename to internal/scripts/docker_env/scripts/weatherregime_env.sh diff --git a/scripts/docker/docker_env/scripts/xesmf_env.sh b/internal/scripts/docker_env/scripts/xesmf_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/xesmf_env.sh rename to internal/scripts/docker_env/scripts/xesmf_env.sh diff --git a/scripts/docker/docker_data/Dockerfile b/scripts/docker/docker_data/Dockerfile deleted file mode 100644 index b5889ec1fd..0000000000 --- a/scripts/docker/docker_data/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM centos:7 -MAINTAINER George McCabe - -# Required arguments -ARG TARFILE_URL -ARG MOUNTPT - -# Check that TARFILE_URL is defined. -RUN if [ "x${TARFILE_URL}" == "x" ]; then \ - echo "ERROR: TARFILE_URL undefined! Rebuild with \"--build-arg TARFILE_URL={One or more URL's}\""; \ - exit 1; \ - fi - -# Check that MOUNTPT is defined. -RUN if [ "x${MOUNTPT}" == "x" ]; then \ - echo "ERROR: MOUNTPT undefined! Rebuild with \"--build-arg MOUNTPT={VOLUME mount directory}\""; \ - exit 1; \ - fi - -ARG DATA_DIR=/data/input/METplus_Data -ENV CASE_DIR=${DATA_DIR} -RUN mkdir -p ${CASE_DIR} - -RUN for URL in `echo ${TARFILE_URL} | tr "," " "`; do \ - echo "Downloading: ${URL}"; \ - curl -SL ${URL} | tar -xzC ${CASE_DIR}; \ - done - -# Define the volume mount point -VOLUME ${MOUNTPT} - -USER root -CMD ["true"] diff --git a/scripts/docker/docker_data/build_docker_images.sh b/scripts/docker/docker_data/build_docker_images.sh deleted file mode 100755 index 5f751e7961..0000000000 --- a/scripts/docker/docker_data/build_docker_images.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/bin/bash -# -# Build METplus-Data Docker images for sample data tarballs -#======================================================================= -# -# This script pulls sample data tarfiles from: -# https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data/ -# -# Where is specified using the required "-pull" command line -# option. It searches for tarfiles in that directory named -# "sample_data-.tgz". When the "-data" option is used, -# it only processes the specified list of datasets. Otherwise, it -# processes all datasets in that directory. For each dataset, it builds -# a Docker data volume. -# -# Each directory must contain a file named -# "volume_mount_directories". Each line of that file is formatted as: -# : -# For example, "s2s:model_applications/s2s" indicates the directory -# that should be mounted for the s2s dataset. -# -# When "-union" is used, it also builds a Docker data volume for all -# datasets in that directory. When "-push" is used, it pushes the -# resulting images to the specified DockerHub repository. -# -# See Usage statement below. -# -#======================================================================= - -# Constants -SCRIPT_DIR=$(dirname $0) -PULL_URL="https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data" -MOUNTPT_FILE="volume_mount_directories" -MOUNTPT_BASE="/data/input/METplus_Data" - -# -# Usage statement for this script -# -function usage { - cat << EOF - -Usage: build_docker_images.sh - -pull version - [-data list] - [-union] - [-all] - [-push repo] - [-help] - - where: - "-pull version" defines the version of the datasets to be pulled (required). - "-data list" overrides the use of all datasets for this version with a comma-separated list (optional). - "-union" also creates one data volume with all datasets for this version (optional). - "-all" create data volumes from all available datasets for this version (optional). - "-push repo" pushes the images to the specified DockerHub repository (optional). - "-help" prints the usage statement. - - e.g. Pull from ${PULL_URL}/ - Push to DockerHub :- - -EOF - - exit 1 -} - -# -# Command runner utility function -# -function run_command { - echo "RUNNING: $*" - $* - error=$? - if [ ${error} -ne 0 ]; then - echo "ERROR:" - echo "ERROR: '$*' exited with status = ${error}" - echo "ERROR:" - exit ${error} - fi -} - -# Defaults for command line options -DO_UNION=0 -DO_PUSH=0 -DO_ALL=0 - -# Default for checking if using tagged version -TAGGED_VERSION=0 - -# Parse command line options -while true; do - case "$1" in - - pull | -pull | --pull ) - VERSION=$2 - echo "Will pull data from ${PULL_URL}/${VERSION}" - shift 2;; - - data | -data | --data ) - if [ -z ${PULL_DATA+x} ]; then - PULL_DATA=$2 - else - PULL_DATA="${PULL_DATA},$2" - fi - shift 2;; - - union | -union | --union ) - DO_UNION=1 - echo "Will create a data volume containing all input datasets." - shift;; - - all | -all | --all ) - DO_ALL=1 - echo "Will create a data volume for each available input dataset." - shift;; - - push | -push | --push ) - DO_PUSH=1 - PUSH_REPO=$2 - if [ -z ${PUSH_REPO} ]; then - echo "ERROR: Must provide push repository after -push" - usage - fi - echo "Will push images to DockerHub ${PUSH_REPO}." - shift 2;; - - help | -help | --help ) - usage - shift;; - - -* ) - echo "ERROR:" - echo "ERROR: Unsupported option: $1" - echo "ERROR:" - usage;; - - * ) - break;; - - esac -done - -# Check that the version has been specified -if [ -z ${VERSION+x} ]; then - echo "ERROR:" - echo "ERROR: The '-pull' option is required!" - echo "ERROR:" - usage -fi - -# use VERSION in the Docker image tag unless using a tagged version -DOCKER_VERSION=${VERSION} - -# check if using a tagged version (e.g v4.0) -# remove v from version if tagged version -if [[ ${VERSION} =~ ^v[0-9.]+$ ]]; then - TAGGED_VERSION=1 - DOCKER_VERSION=${VERSION:1} -fi - - -# Define the target repository if necessary -if [ -z ${PUSH_REPO+x} ]; then - - # Push tagged versions (e.g. v4.0) to metplus-data - # and all others to metplus-data-dev - if [ ${TAGGED_VERSION} == 1 ]; then - PUSH_REPO="dtcenter/metplus-data" - else - PUSH_REPO="dtcenter/metplus-data-dev" - fi -fi - -# Print the datasets to be processed -if [ -z ${PULL_DATA+x} ]; then - echo "Will process all available datasets." -else - echo "Will process the following datasets: ${PULL_DATA}" -fi - -# Get list of available tarfiles -TARFILE_LIST=`curl -s ${PULL_URL}/${VERSION}/ | tr "<>" "\n" | egrep sample_data | egrep -v href` - -if [[ ${TARFILE_LIST} == "" ]]; then - echo "ERROR:" - echo "ERROR: No tarfiles found in ${PULL_URL}/${VERSION}" - echo "ERROR:" - exit 1 -fi - -# Build separate image for each tarfile -for TARFILE in $TARFILE_LIST; do - - # Build a list of all URL's - CUR_URL="${PULL_URL}/${VERSION}/${TARFILE}" - - if [ -z ${URL_LIST+x} ]; then - URL_LIST=${CUR_URL} - else - URL_LIST="${URL_LIST},${CUR_URL}" - fi - - # Parse the current dataset name - CUR_DATA=`echo $TARFILE | cut -d'-' -f2 | sed 's/.tgz//g'` - - if [ -z ${PULL_DATA+x} ] || [ `echo ${PULL_DATA} | grep ${CUR_DATA}` ] || [ ${DO_ALL} == 1 ]; then - echo "Processing \"${TARFILE}\" ..." - else - echo "Skipping \"${TARFILE}\" since \"${CUR_DATA}\" was not requested in \"-data\"." - continue - fi - - # Define the docker image name - IMGNAME="${PUSH_REPO}:${DOCKER_VERSION}-${CUR_DATA}" - - # Determine the mount point - MOUNTPT_URL="${PULL_URL}/${VERSION}/${MOUNTPT_FILE}" - MOUNTPT=${MOUNTPT_BASE}/`curl -s ${MOUNTPT_URL} | grep "${CUR_DATA}:" | cut -d':' -f2` - - if [[ ${MOUNTPT} == "" ]]; then - echo "ERROR:" - echo "ERROR: No entry found for \"${CUR_DATA}\" found in ${MOUNTPT_URL}!" - echo "ERROR:" - exit 1 - fi - - echo - echo "Building image ... ${IMGNAME}" - echo - - run_command docker build -t ${IMGNAME} ${SCRIPT_DIR} \ - --build-arg TARFILE_URL=${CUR_URL} \ - --build-arg MOUNTPT=${MOUNTPT} - - if [ ${DO_PUSH} == 1 ]; then - echo - echo "Pushing image ... ${IMGNAME}" - echo - # if DOCKER_USERNAME is set, then run docker login - if [ ! -z ${DOCKER_USERNAME+x} ]; then - echo "Logging into Docker ..." - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - fi - run_command docker push ${IMGNAME} - - fi - -done - -# -# Build one image for all tarfiles -# - -if [ ${DO_UNION} == 1 ]; then - - IMGNAME="${PUSH_REPO}:${DOCKER_VERSION}" - MOUNTPT="${MOUNTPT_BASE}" - - run_command docker build -t ${IMGNAME} ${SCRIPT_DIR} \ - --build-arg TARFILE_URL=${URL_LIST} \ - --build-arg MOUNTPT=${MOUNTPT} - - if [ ${DO_PUSH} == 1 ]; then - echo - echo "Pushing image ... ${IMGNAME}" - echo - - run_command docker push ${IMGNAME} - - fi -fi - diff --git a/scripts/docker/docker_data/hooks/build b/scripts/docker/docker_data/hooks/build deleted file mode 100644 index 5adac21119..0000000000 --- a/scripts/docker/docker_data/hooks/build +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./build_docker_images.sh -pull v${DOCKER_TAG} -union -push dtcenter/metplus-data