diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index a40d6b2e56..edf0b00167 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -39,7 +39,7 @@ def get_dockerhub_url(branch_name): def docker_get_volumes_last_updated(current_branch): import requests dockerhub_url = get_dockerhub_url(current_branch) - dockerhub_request = requests.get(dockerhub_url) + dockerhub_request = requests.get(dockerhub_url, timeout=60) if dockerhub_request.status_code != 200: print(f"Could not find DockerHub URL: {dockerhub_url}") return None @@ -61,7 +61,7 @@ def docker_get_volumes_last_updated(current_branch): volumes_last_updated[repo_name] = repo['last_updated'] if not page['next']: break - page = requests.get(page['next']).json() + page = requests.get(page['next'], timeout=60).json() attempts += 1 return volumes_last_updated diff --git a/.github/jobs/get_data_volumes.py b/.github/jobs/get_data_volumes.py index 5c568577a1..e492711710 100755 --- a/.github/jobs/get_data_volumes.py +++ b/.github/jobs/get_data_volumes.py @@ -40,7 +40,7 @@ def main(args): branch_name = get_branch_name() if not branch_name: print("Could not get current branch. Exiting.") - sys.exit(1) + return None # remove -ref from end of branch name if found if branch_name.endswith('-ref'): @@ -88,7 +88,7 @@ def main(args): # use it, otherwise use develop version of data volume elif (metplus_version == 'develop' and f'{branch_name}-{model_app_name}' in available_volumes): - volume_name = f'{branch_name}-{model_app_name}' + volume_name = f'{branch_name}-{model_app_name}' else: volume_name = f'{metplus_version}-{model_app_name}' @@ -97,13 +97,15 @@ def main(args): cmd = (f'docker create --name {model_app_name} ' f'{full_volume_name}') if not run_commands(cmd): - continue + print(f'ERROR: Could not create data volume for {full_volume_name}') + return None # add name to volumes from list to pass to docker build volume_list.append(f'--volumes-from {model_app_name}') return ' '.join(volume_list) + if __name__ == "__main__": # split up command line args that have commas before passing into main args = [] @@ -111,4 +113,7 @@ def main(args): for arg in sys.argv[1:]: args.extend(arg.split(',')) out = main(args) + if out is None: + print("ERROR: Something went wrong") + sys.exit(1) print(out) diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index 1a4c908b14..59c5be37ed 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -43,6 +43,9 @@ output_category = f"output-{output_data_branch}-{artifact_name}" VOLUMES_FROM = get_data_volumes.main([output_category]) +if VOLUMES_FROM is None: + print("ERROR: Could not get truth data to run diff") + sys.exit(1) print(f"Output Volumes: {VOLUMES_FROM}") diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 931a79d76d..d65cf247c5 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -49,6 +49,10 @@ def main(): ) # get input data volumes volumes_from = get_data_volumes.main(categories_list) + if volumes_from is None: + print('ERROR: Could not get input data to run use cases') + sys.exit(1) + print(f"Input Volumes: {volumes_from}") # build Docker image with conda environment and METplus branch image diff --git a/internal/scripts/docker_env/Dockerfile.conda b/internal/scripts/docker_env/Dockerfile.conda index f1a62fa780..e2679c05a9 100644 --- a/internal/scripts/docker_env/Dockerfile.conda +++ b/internal/scripts/docker_env/Dockerfile.conda @@ -5,6 +5,6 @@ ARG DEBIAN_VERSION=12 FROM debian:${DEBIAN_VERSION}-slim RUN apt update && apt install -y curl \ - && curl https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh > /miniconda.sh \ - && bash /miniconda.sh -b -p /usr/local/conda \ + && curl -L https://github.com/conda-forge/miniforge/releases/download/23.3.1-1/Mambaforge-23.3.1-1-$(uname)-$(uname -m).sh -o /miniforge.sh \ + && bash /miniforge.sh -b -p /usr/local/conda \ && /usr/local/conda/bin/conda update -y -n base -c conda-forge conda