diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh index 67bb67146a..eb8656297f 100755 --- a/.github/jobs/build_docker_image.sh +++ b/.github/jobs/build_docker_image.sh @@ -10,6 +10,7 @@ CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build.log time_command docker build -t ${DOCKERHUB_TAG} \ --build-arg SOURCE_BRANCH \ + --build-arg MET_BASE_REPO \ --build-arg MET_BASE_IMAGE \ -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE} if [ $? != 0 ]; then diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index 54aeca8507..6e0445131a 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -5,7 +5,8 @@ run_push=false run_unit_tests=false run_diff=false run_update_truth=false -met_base_image=minimum +met_base_repo=met-base +met_base_image=v1.0 input_data_version=develop truth_data_version=develop @@ -102,7 +103,7 @@ fi # if running unit tests, use unit_test MET base image and push image if [ "$run_unit_tests" == "true" ]; then - met_base_image=unit_test + met_base_repo=met-base-unit-test run_push=true fi @@ -112,6 +113,7 @@ echo ::set-output name=run_push::$run_push echo ::set-output name=run_unit_tests::$run_unit_tests echo ::set-output name=run_diff::$run_diff echo ::set-output name=run_update_truth::$run_update_truth +echo ::set-output name=met_base_repo::$met_base_repo echo ::set-output name=met_base_image::$met_base_image echo ::set-output name=input_data_version::$input_data_version echo ::set-output name=truth_data_version::$truth_data_version @@ -121,6 +123,7 @@ echo run_push: $run_push echo run_unit_tests: $run_unit_tests echo run_diff: $run_diff echo run_update_truth: $run_update_truth +echo met_base_repo: $met_base_repo echo met_base_image: $met_base_image echo input_data_version: $input_data_version echo truth_data_version: $truth_data_version diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index 63c306f7e1..1c91ace1cd 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -26,7 +26,8 @@ jobs: run: .github/jobs/build_docker_image.sh env: SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite - MET_BASE_IMAGE: minimum + MET_BASE_REPO: met-base + MET_BASE_IMAGE: v1.0 - name: Push Docker Image run: .github/jobs/push_docker_image.sh diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 550add3594..8e1152ebda 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -54,6 +54,7 @@ jobs: run_unit_tests: ${{ steps.job_status.outputs.run_unit_tests }} run_diff: ${{ steps.job_status.outputs.run_diff }} run_update_truth: ${{ steps.job_status.outputs.run_update_truth }} + met_base_repo: ${{ steps.job_status.outputs.met_base_repo }} met_base_image: ${{ steps.job_status.outputs.met_base_image }} branch_name: ${{ steps.job_status.outputs.branch_name }} truth_data_version: ${{ steps.job_status.outputs.truth_data_version }} @@ -74,6 +75,7 @@ jobs: run: .github/jobs/build_docker_image.sh env: SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + MET_BASE_REPO: ${{ needs.job_control.outputs.met_base_repo }} MET_BASE_IMAGE: ${{ needs.job_control.outputs.met_base_image }} - name: Copy Docker build log into logs directory diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index d324cbb6d4..61b1d3a1bf 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -1,6 +1,7 @@ -ARG MET_BASE_IMAGE=minimum +ARG MET_BASE_REPO=met-base +ARG MET_BASE_IMAGE=v1.0 -FROM dtcenter/met-base:${MET_BASE_IMAGE} +FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_IMAGE} MAINTAINER John Halley Gotway # diff --git a/internal/scripts/docker/Dockerfile.copy b/internal/scripts/docker/Dockerfile.copy index 18d5d3ac35..4e2e2f023e 100644 --- a/internal/scripts/docker/Dockerfile.copy +++ b/internal/scripts/docker/Dockerfile.copy @@ -1,6 +1,7 @@ -ARG MET_BASE_IMAGE=minimum +ARG MET_BASE_REPO=met-base +ARG MET_BASE_IMAGE=v1.0 -FROM dtcenter/met-base:${MET_BASE_IMAGE} +FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_IMAGE} MAINTAINER John Halley Gotway # diff --git a/internal/scripts/docker/Dockerfile.minimum b/internal/scripts/docker/Dockerfile.minimum deleted file mode 100644 index 028871196b..0000000000 --- a/internal/scripts/docker/Dockerfile.minimum +++ /dev/null @@ -1,138 +0,0 @@ -FROM centos:7 -MAINTAINER John Halley Gotway - -# -# Define the compilers. -# -ENV CC /usr/bin/gcc -ENV CXX /usr/bin/g++ -ENV FC /usr/bin/gfortran -ENV F77 /usr/bin/gfortran - -# -# Define package URL's. -# -ENV HDF4_URL http://www.hdfgroup.org/ftp/HDF/releases/HDF4.2r3/src/HDF4.2r3.tar.gz -ENV HDFEOS_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/HDF-EOS2.16v1.00.tar.Z - -ENV NETCDF4C_URL https://github.com/Unidata/netcdf-c/archive/v4.4.1.1.zip -ENV NETCDF4CXX_URL https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz - -ENV BUFRLIB_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/BUFRLIB_v10-2-3.tar -ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz - -# -# Install the required packages. -# -RUN yum -y update \ - && yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 \ - libpng-devel jasper jasper-devel zlib zlib-devel \ - cairo-devel freetype-devel epel-release \ - hostname m4 make tar tcsh ksh time wget which \ - flex flex-devel bison bison-devel unzip \ - && yum -y install git g2clib-devel hdf5-devel.x86_64 gsl-devel \ - && yum -y install gv ncview wgrib wgrib2 ImageMagick ps2pdf \ - && yum -y install python3 python3-devel python3-pip \ - && pip3 install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - -# -# Set the working directory. -# -WORKDIR /met - -# -# Setup the environment for interactive bash/csh container shells. -# -RUN echo export MET_BASE=/usr/local/share/met >> /etc/bashrc \ - && echo setenv MET_BASE /usr/local/share/met >> /etc/csh.cshrc \ - && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /etc/bashrc \ - && echo setenv MET_FONT_DIR /usr/local/share/met/fonts >> /etc/csh.cshrc \ - && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /etc/bashrc \ - && echo setenv RSCRIPTS_BASE /usr/local/share/met/Rscripts >> /etc/csh.cshrc \ - && echo export LD_LIBRARY_PATH=/usr/local/lib >> /etc/bashrc \ - && echo setenv LD_LIBRARY_PATH /usr/local/lib >> /etc/csh.cshrc -ENV LD_LIBRARY_PATH /usr/local/lib -ENV MET_FONT_DIR /usr/local/share/met/fonts - -# -# Download and install BUFRLIB. -# -RUN mkdir -p /met/logs \ - && mkdir -p /met/external_libs/BUFRLIB \ - && cd /met/external_libs/BUFRLIB \ - && echo "Downloading BUFRLIB from ${BUFRLIB_URL}" \ - && curl -SL ${BUFRLIB_URL} | tar xC /met/external_libs/BUFRLIB \ - && cat preproc.sh | sed 's/cpp /cpp -traditional-cpp /g' > preproc_patch.sh \ - && chmod +x preproc_patch.sh \ - && LOG_FILE=/met/logs/BUFRLIB_build.log \ - && echo "Compiling BUFRLIB and writing log file ${LOG_FILE}" \ - && ./preproc_patch.sh *.F > ${LOG_FILE} \ - && ${CC} -c -DUNDERSCORE *.c >> ${LOG_FILE} \ - && ${FC} -c -fno-second-underscore *.f >> ${LOG_FILE} \ - && ar crv libbufr.a *.o >> ${LOG_FILE} \ - && rm -f /usr/local/lib/libbufr.a \ - && cp *.a /usr/local/lib \ - && cd /met/external_libs \ - && rm -rf BUFRLIB - -# -# Download and install NetCDF4 (C and C++). -# -RUN mkdir -p /met/external_libs/netcdf \ - && cd /met/external_libs/netcdf \ - && echo "Downloading netcdf-c-4.4.1.1 from ${NETCDF4C_URL}" \ - && wget ${NETCDF4C_URL} \ - && unzip v4.4.1.1.zip \ - && cd netcdf-c-4.4.1.1 \ - && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_configure.log \ - && echo "Configuring netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_make_install.log \ - && echo "Compiling netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading from ${NETCDF4CXX_URL}" \ - && cd /met/external_libs/netcdf \ - && wget ${NETCDF4CXX_URL} \ - && tar -xzf v4.3.0.tar.gz \ - && cd netcdf-cxx4-4.3.0 \ - && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_configure.log \ - && echo "Configuring netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_make_install.log \ - && echo "Compiling netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && cd /met/external_libs \ - && rm -rf netcdf - -# -# Download and install HDF4 and HDFEOS. -# -RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ - && curl -SL ${HDF4_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/HDF4.2r3 \ - && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ - && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ - && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ - && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ - && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ - && echo "Compiling HDF4.2r3 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading hdfeos from ${HDFEOS_URL}" \ - && curl -SL ${HDFEOS_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/hdfeos \ - && LOG_FILE=/met/logs/hdfeos_configure.log \ - && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ - && LOG_FILE=/met/logs/hdfeos_make_install.log \ - && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && mkdir /usr/local/hdfeos/include \ - && cp include/*.h /usr/local/hdfeos/include/. \ - && cd /met/external_libs \ - && rm -rf HDF4.2r3 hdfeos - -RUN echo "Downloading GhostScript fonts from ${GSFONT_URL} into /usr/local/share/met" \ - && mkdir -p /usr/local/share/met \ - && curl -SL ${GSFONT_URL} | tar zxC /usr/local/share/met diff --git a/internal/scripts/docker/Dockerfile.test b/internal/scripts/docker/Dockerfile.test deleted file mode 100644 index bc7963cb91..0000000000 --- a/internal/scripts/docker/Dockerfile.test +++ /dev/null @@ -1,26 +0,0 @@ -ARG MET_BASE_IMAGE=minimum - -FROM dtcenter/met-base:${MET_BASE_IMAGE} -MAINTAINER John Halley Gotway - -# -# Set the working directory. -# -WORKDIR /met - -# -# Download and install MET and GhostScript fonts. -# Delete the MET source code for tagged releases matching "v"*. -# -RUN echo "Installing tools needed for running MET unit tests..." \ - && echo "Installing Perl XML Parser..." \ - && yum makecache \ - && yum -y install perl-XML-Parser \ - && echo "Installing R..." \ - && yum -y install R \ - && echo "Installing R ncdf4 1.19..." \ - && wget https://cran.r-project.org/src/contrib/ncdf4_1.19.tar.gz \ - && R CMD INSTALL ncdf4_1.19.tar.gz \ - && echo "Installing NCO (for ncdiff)..." \ - && yum -y install nco \ - && echo "Finished installing unit test tools" diff --git a/internal/scripts/docker/README.md b/internal/scripts/docker/README.md index 87635ceb84..6198dcca86 100644 --- a/internal/scripts/docker/README.md +++ b/internal/scripts/docker/README.md @@ -2,20 +2,12 @@ Run all of the Docker commands from the top-level directory of the MET repository -## Build image with minimum requirements needed to build MET - -```docker build -t dtcenter/met-base:minimum -f internal/scripts/docker/Dockerfile.minimum . -docker push dtcenter/met-base:minimum``` - -## Build image with requirements to build MET and run MET unit tests - -```docker build -t dtcenter/met-base:unit_test -f internal/scripts/docker/Dockerfile.test . -docker push dtcenter/met-base:unit_test``` - ## Build MET from clone -```docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} internal/scripts/docker -docker push dtcenter/met:${TAG_NAME}``` +``` +docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} internal/scripts/docker +docker push dtcenter/met:${TAG_NAME} +``` where: * TAG_NAME is the name of the DockerHub tag to create @@ -23,8 +15,10 @@ where: ## Build MET from local source code with minimum requirements -```docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} -f internal/scripts/docker/Dockerfile.copy . -docker push dtcenter/met:${TAG_NAME}``` +``` +docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} -f internal/scripts/docker/Dockerfile.copy . +docker push dtcenter/met:${TAG_NAME} +``` where: * TAG_NAME is the name of the DockerHub tag to create @@ -32,9 +26,12 @@ where: ## Build MET from local source code with unit test requirements -```docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} --build-arg MET_BASE_IMAGE=unit_test -f internal/scripts/docker/Dockerfile.copy . -docker push dtcenter/met:${TAG_NAME}``` +``` +docker build -t dtcenter/met:${TAG_NAME} --build-arg SOURCE_BRANCH=${BRANCH_NAME} --build-arg MET_BASE_IMAGE=unit_test_${MET_BASE_IMAGE_VERSION} -f internal/scripts/docker/Dockerfile.copy . +docker push dtcenter/met:${TAG_NAME} +``` where: * TAG_NAME is the name of the DockerHub tag to create * BRANCH_NAME is the identifier to use for $MET_GIT_NAME inside image +* MET_BASE_IMAGE_VERSION is the METbaseimage tag version number \ No newline at end of file