From 40ded3892d0ec9bcf9ceb4f3f7f5cdb2b316c753 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 17 Nov 2023 18:15:01 -0600 Subject: [PATCH 01/24] Add script to test nightly environments are solvable and using recent nightlies. --- ci/check_conda_nightly_env.py | 98 +++++++++++++++++++++++++++++++++++ ci/release/update-version.sh | 1 + ci/test_conda_nightly_env.sh | 29 +++++++++++ 3 files changed, 128 insertions(+) create mode 100644 ci/check_conda_nightly_env.py create mode 100755 ci/test_conda_nightly_env.sh diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py new file mode 100644 index 00000000..c887af15 --- /dev/null +++ b/ci/check_conda_nightly_env.py @@ -0,0 +1,98 @@ +import json +import re +import subprocess +import sys +from datetime import datetime, timedelta + + +OLD_PACKAGE_THRESHOLD_DAYS = 3 + + +def is_rapids_nightly_package(package_info): + return package_info["channel"] == "rapidsai-nightly" + + +def get_package_date(package): + date_re = re.compile(r"(2\d{5})") + + # 2\d{5} matches 6 digits starting with "2" + date_re = r"_(2\d{5})_" + + # Use regex to find the date string in the input + match = re.search(date_re, package["build_string"]) + + if match: + # Convert the date string to a datetime object + date_string = match.group(1) + date_object = datetime.strptime(date_string, "%y%m%d") + return date_object + + print( + f"Date string not found for {package['name']} " + f"in the build string '{package['build_string']}'." + ) + return None + + +def check_env(json_path): + """Validate rapids conda environments. + + Parses JSON output of `conda create` and check the dates on the RAPIDS + packages to ensure nightlies are relatively new. + """ + + with open(json_path) as f: + try: + json_data = json.load(f) + except ValueError as e: + print("Error: JSON data file from conda failed to load:") + print(e) + sys.exit(1) + + if "error" in json_data: + print("Error: conda failed:") + print() + print(json_data["error"]) + sys.exit(1) + + package_data = json_data["actions"]["LINK"] + + rapids_package_data = list(filter(is_rapids_nightly_package, package_data)) + + # Dictionary to store the packages and their dates + rapids_package_dates = { + package["name"]: get_package_date(package) + for package in rapids_package_data + } + + old_threshold = datetime.now() - timedelta(days=OLD_PACKAGE_THRESHOLD_DAYS) + old_packages = { + package: date + for package, date in rapids_package_dates.items() + if date is not None and date < old_threshold + } + + # If there are old packages, raise an error + if old_packages: + print() + print( + "Error: The following nightly packages are more than " + f"{OLD_PACKAGE_THRESHOLD_DAYS} days old:" + ) + for package, date in old_packages.items(): + date_string = date.strftime("%Y-%m-%d") + print(f" - {package}: {date_string}") + sys.exit(1) + + print(f"All packages are less than {OLD_PACKAGE_THRESHOLD_DAYS} days old.") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print( + "Provide only one argument, the filepath to a JSON output from " + "conda." + ) + sys.exit(1) + + check_env(sys.argv[1]) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index aa7bfe8c..f6f45f2a 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -41,6 +41,7 @@ function sed_runner() { } sed_runner "/RAPIDS_VER=/ s/[0-9][0-9].[0-9][0-9]/${NEXT_SHORT_TAG}/" ci/conda-pack.sh +sed_runner "/RAPIDS_VERSION=/ s/[0-9][0-9].[0-9][0-9]/${NEXT_SHORT_TAG}/" ci/test_conda_nightly_env.sh for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" diff --git a/ci/test_conda_nightly_env.sh b/ci/test_conda_nightly_env.sh new file mode 100755 index 00000000..c6218052 --- /dev/null +++ b/ci/test_conda_nightly_env.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +RAPIDS_VERSION="23.12" +CUDA_VERSION=${RAPIDS_CUDA_VERSION%.*} + +JSON_FILENAME="rapids_cuda${CUDA_VERSION}_py${RAPIDS_PY_VERSION}.json" + +echo "Creating conda environment with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" +#rapids-logger "Creating conda environment with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" + +#rapids-conda-retry \ +conda \ + create \ + --solver=libmamba \ + -n rapids-${RAPIDS_VERSION} \ + -c rapidsai-nightly \ + -c conda-forge \ + -c nvidia \ + rapids=${RAPIDS_VERSION} \ + python=${RAPIDS_PY_VERSION} \ + cuda-version=${CUDA_VERSION} \ + --dry-run \ + --json \ + | tee "${JSON_FILENAME}" + +python ci/check_conda_nightly_env.py "${JSON_FILENAME}" From 551169fa307bbc0ca1cf219adac3923b8e3bb470 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 17 Nov 2023 18:19:47 -0600 Subject: [PATCH 02/24] Clean up script. --- ci/check_conda_nightly_env.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index c887af15..9030a8a8 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -13,9 +13,7 @@ def is_rapids_nightly_package(package_info): def get_package_date(package): - date_re = re.compile(r"(2\d{5})") - - # 2\d{5} matches 6 digits starting with "2" + # Matches 6 digits starting with "2", which should be YYMMDD date_re = r"_(2\d{5})_" # Use regex to find the date string in the input @@ -65,7 +63,8 @@ def check_env(json_path): for package in rapids_package_data } - old_threshold = datetime.now() - timedelta(days=OLD_PACKAGE_THRESHOLD_DAYS) + today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + old_threshold = today - timedelta(days=OLD_PACKAGE_THRESHOLD_DAYS) old_packages = { package: date for package, date in rapids_package_dates.items() From df6074e79963bcc903b2bc4a919086face384bd1 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 12:38:56 -0500 Subject: [PATCH 03/24] Add test-conda-nightly-env to pr.yaml for testing. --- .github/workflows/pr.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 356955cf..1367946b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -10,13 +10,20 @@ concurrency: cancel-in-progress: true jobs: + pr-builder: + needs: + - build + - test-conda-nightly-env + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 build: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request - pr-builder: - needs: - - build + test-conda-nightly-env: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/test_conda_nightly_env.yaml" From 98383f6a2717a4975dc3d6faf9dfd732c330c0e0 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 12:39:38 -0500 Subject: [PATCH 04/24] Skip build jobs for testing. --- .github/workflows/pr.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1367946b..0fd716e5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,15 +12,15 @@ concurrency: jobs: pr-builder: needs: - - build + #- build - test-conda-nightly-env secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 - build: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 - with: - build_type: pull-request +# build: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 +# with: +# build_type: pull-request test-conda-nightly-env: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 From d487411aff8de1e5a4fd33fc364445395ff00761 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 12:40:08 -0500 Subject: [PATCH 05/24] Update RAPIDS_VERSION. --- ci/test_conda_nightly_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_conda_nightly_env.sh b/ci/test_conda_nightly_env.sh index c6218052..5176fbf2 100755 --- a/ci/test_conda_nightly_env.sh +++ b/ci/test_conda_nightly_env.sh @@ -3,7 +3,7 @@ set -euo pipefail -RAPIDS_VERSION="23.12" +RAPIDS_VERSION="24.10" CUDA_VERSION=${RAPIDS_CUDA_VERSION%.*} JSON_FILENAME="rapids_cuda${CUDA_VERSION}_py${RAPIDS_PY_VERSION}.json" From 55da82bfb07e4914eb03d26b6bc47092b58bec09 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 12:42:49 -0500 Subject: [PATCH 06/24] Fix extension. --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0fd716e5..284e30c5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,4 +26,4 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request - script: "ci/test_conda_nightly_env.yaml" + script: "ci/test_conda_nightly_env.sh" From 9d3a8b0216e0af157a2e995bca819a978d063ec4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 13:22:01 -0500 Subject: [PATCH 07/24] Exclude some packages, and collapse conda dry-run output. --- ci/check_conda_nightly_env.py | 16 ++++++++++++++++ ci/test_conda_nightly_env.sh | 12 ++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index 9030a8a8..ddc5dc91 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -7,12 +7,28 @@ OLD_PACKAGE_THRESHOLD_DAYS = 3 +EXCLUDED_PACKAGES = { + # These packages do not have date strings: + "cubinlinker", + "rapids-dask-dependency", + "libxgboost", + "py-xgboost", + "xgboost", + # TODO: Remove nx-cugraph after https://github.com/rapidsai/cugraph/pull/4639 + "nx-cugraph", + # TODO: Do we want ucx-proc on rapidsai or from conda-forge? + "ucx-proc", +} + def is_rapids_nightly_package(package_info): return package_info["channel"] == "rapidsai-nightly" def get_package_date(package): + if package in EXCLUDED_PACKAGES: + return None + # Matches 6 digits starting with "2", which should be YYMMDD date_re = r"_(2\d{5})_" diff --git a/ci/test_conda_nightly_env.sh b/ci/test_conda_nightly_env.sh index 5176fbf2..7fe46cee 100755 --- a/ci/test_conda_nightly_env.sh +++ b/ci/test_conda_nightly_env.sh @@ -8,11 +8,11 @@ CUDA_VERSION=${RAPIDS_CUDA_VERSION%.*} JSON_FILENAME="rapids_cuda${CUDA_VERSION}_py${RAPIDS_PY_VERSION}.json" -echo "Creating conda environment with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" -#rapids-logger "Creating conda environment with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" +rapids-logger "Creating conda environment with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" -#rapids-conda-retry \ -conda \ +echo "::group::conda dry-run output" + +rapids-conda-retry \ create \ --solver=libmamba \ -n rapids-${RAPIDS_VERSION} \ @@ -26,4 +26,8 @@ conda \ --json \ | tee "${JSON_FILENAME}" +echo "::endgroup::" + +rapids-logger "Parsing results from conda dry-run with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" + python ci/check_conda_nightly_env.py "${JSON_FILENAME}" From b323bfab3d15410f87b7cfbce495b3f4128a9c7d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 13:26:49 -0500 Subject: [PATCH 08/24] Use package name. --- ci/check_conda_nightly_env.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index ddc5dc91..a4aaaee3 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -8,6 +8,9 @@ OLD_PACKAGE_THRESHOLD_DAYS = 3 EXCLUDED_PACKAGES = { + # These packages are not built every night: + "rapids", + "rapids-xgboost", # These packages do not have date strings: "cubinlinker", "rapids-dask-dependency", @@ -26,7 +29,7 @@ def is_rapids_nightly_package(package_info): def get_package_date(package): - if package in EXCLUDED_PACKAGES: + if package["name"] in EXCLUDED_PACKAGES: return None # Matches 6 digits starting with "2", which should be YYMMDD From c245fc6cb1623d425e39d3bc9aaff04e7f5cbf36 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 13:37:37 -0500 Subject: [PATCH 09/24] Clean up. --- ci/test_conda_nightly_env.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ci/test_conda_nightly_env.sh b/ci/test_conda_nightly_env.sh index 7fe46cee..1cbd9e83 100755 --- a/ci/test_conda_nightly_env.sh +++ b/ci/test_conda_nightly_env.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. set -euo pipefail @@ -10,11 +10,8 @@ JSON_FILENAME="rapids_cuda${CUDA_VERSION}_py${RAPIDS_PY_VERSION}.json" rapids-logger "Creating conda environment with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" -echo "::group::conda dry-run output" - rapids-conda-retry \ create \ - --solver=libmamba \ -n rapids-${RAPIDS_VERSION} \ -c rapidsai-nightly \ -c conda-forge \ @@ -26,8 +23,6 @@ rapids-conda-retry \ --json \ | tee "${JSON_FILENAME}" -echo "::endgroup::" - rapids-logger "Parsing results from conda dry-run with rapids=${RAPIDS_VERSION}, python=${RAPIDS_PY_VERSION}, cuda-version=${CUDA_VERSION}" python ci/check_conda_nightly_env.py "${JSON_FILENAME}" From 4af2f571a3c2ed3a9fbfa27369269097f0ba5a02 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 13:38:42 -0500 Subject: [PATCH 10/24] Skip codecov. --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 284e30c5..8e02872b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,3 +27,4 @@ jobs: with: build_type: pull-request script: "ci/test_conda_nightly_env.sh" + run_codecov: false From 0f4741fb77a0c9e37e46c9c416f237bd340b60d7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 13:55:12 -0500 Subject: [PATCH 11/24] Exclude pynvjitlink, fail on missing date strings, show package dates. --- ci/check_conda_nightly_env.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index a4aaaee3..578417ab 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -13,6 +13,7 @@ "rapids-xgboost", # These packages do not have date strings: "cubinlinker", + "pynvjitlink", "rapids-dask-dependency", "libxgboost", "py-xgboost", @@ -48,7 +49,7 @@ def get_package_date(package): f"Date string not found for {package['name']} " f"in the build string '{package['build_string']}'." ) - return None + sys.exit(1) def check_env(json_path): @@ -102,7 +103,8 @@ def check_env(json_path): print(f" - {package}: {date_string}") sys.exit(1) - print(f"All packages are less than {OLD_PACKAGE_THRESHOLD_DAYS} days old.") + print(f"All packages are less than {OLD_PACKAGE_THRESHOLD_DAYS} days old:") + print(rapids_package_dates) if __name__ == "__main__": From d9478b47613f9341794a747a421097549edced66 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 14:21:28 -0500 Subject: [PATCH 12/24] Pretty-print dates. --- ci/check_conda_nightly_env.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index 578417ab..53c7d41f 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -98,13 +98,14 @@ def check_env(json_path): "Error: The following nightly packages are more than " f"{OLD_PACKAGE_THRESHOLD_DAYS} days old:" ) - for package, date in old_packages.items(): + for package, date in sorted(old_packages.items()): date_string = date.strftime("%Y-%m-%d") - print(f" - {package}: {date_string}") sys.exit(1) print(f"All packages are less than {OLD_PACKAGE_THRESHOLD_DAYS} days old:") - print(rapids_package_dates) + for package, date in sorted(rapids_package_dates.items()): + date_string = date.strftime("%Y-%m-%d") + print(f" - {package}: {date_string}") if __name__ == "__main__": From df76342bc9342d88082c7b222af1cbea88ab2aa9 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 14:25:21 -0500 Subject: [PATCH 13/24] Use build workflow to get CPU runners and high matrix coverage. --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8e02872b..487c21d1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,8 +23,8 @@ jobs: # build_type: pull-request test-conda-nightly-env: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + # We use a build workflow so that we get CPU jobs and high matrix coverage + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/test_conda_nightly_env.sh" - run_codecov: false From 26e41433a7037b10fefcadc6d9b031a6e5b0d747 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 14:57:02 -0500 Subject: [PATCH 14/24] Improve formatting. --- ci/check_conda_nightly_env.py | 62 ++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index 53c7d41f..fcfe2a59 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -24,6 +24,12 @@ "ucx-proc", } +# ANSI color codes used to highlight lines +FAIL = "\033[91m" +WARNING = "\033[93m" +OKGREEN = "\033[92m" +ENDC = "\033[0m" + def is_rapids_nightly_package(package_info): return package_info["channel"] == "rapidsai-nightly" @@ -46,10 +52,9 @@ def get_package_date(package): return date_object print( - f"Date string not found for {package['name']} " - f"in the build string '{package['build_string']}'." + f"{WARNING}Date string not found for {package['name']} " + f"in the build string '{package['build_string']}'.{ENDC}" ) - sys.exit(1) def check_env(json_path): @@ -57,21 +62,25 @@ def check_env(json_path): Parses JSON output of `conda create` and check the dates on the RAPIDS packages to ensure nightlies are relatively new. + + Returns an exit code value. """ + exit_code = 0 + with open(json_path) as f: try: json_data = json.load(f) except ValueError as e: print("Error: JSON data file from conda failed to load:") print(e) - sys.exit(1) + return 1 if "error" in json_data: print("Error: conda failed:") print() print(json_data["error"]) - sys.exit(1) + return 1 package_data = json_data["actions"]["LINK"] @@ -79,10 +88,10 @@ def check_env(json_path): # Dictionary to store the packages and their dates rapids_package_dates = { - package["name"]: get_package_date(package) - for package in rapids_package_data + package["name"]: get_package_date(package) for package in rapids_package_data } + # If there are old packages, show an error today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) old_threshold = today - timedelta(days=OLD_PACKAGE_THRESHOLD_DAYS) old_packages = { @@ -90,30 +99,45 @@ def check_env(json_path): for package, date in rapids_package_dates.items() if date is not None and date < old_threshold } - - # If there are old packages, raise an error if old_packages: + exit_code = 1 print() print( - "Error: The following nightly packages are more than " - f"{OLD_PACKAGE_THRESHOLD_DAYS} days old:" + f"{FAIL}Error: The following packages are more than " + f"{OLD_PACKAGE_THRESHOLD_DAYS} days old:{ENDC}" ) for package, date in sorted(old_packages.items()): date_string = date.strftime("%Y-%m-%d") - sys.exit(1) + print(f"{FAIL} - {package}: {date_string}{ENDC}") + + # If there are undated packages, show an error + undated_packages = { + package: date for package, date in rapids_package_dates.items() if date is None + } + if undated_packages: + exit_code = 1 + print() + print( + f"{FAIL}Error: The following packages are missing dates in their build strings:{ENDC}" + ) + for package, date in sorted(undated_packages.items()): + print(f"{FAIL} - {package}{ENDC}") - print(f"All packages are less than {OLD_PACKAGE_THRESHOLD_DAYS} days old:") + print() + print( + f"The following packages are less than {OLD_PACKAGE_THRESHOLD_DAYS} days old:" + ) for package, date in sorted(rapids_package_dates.items()): date_string = date.strftime("%Y-%m-%d") - print(f" - {package}: {date_string}") + status = WARNING if date < today - timedelta(days=1) else OKGREEN + print(f"{status} - {package}: {date_string}{ENDC}") + + return exit_code if __name__ == "__main__": if len(sys.argv) != 2: - print( - "Provide only one argument, the filepath to a JSON output from " - "conda." - ) + print("Provide only one argument, the filepath to a JSON output from " "conda.") sys.exit(1) - check_env(sys.argv[1]) + sys.exit(check_env(sys.argv[1])) From 52c5308e13aa986549b4972491eaf3fb6e2a1233 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 15:00:20 -0500 Subject: [PATCH 15/24] Fix exclusions. --- ci/check_conda_nightly_env.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index fcfe2a59..03ed5220 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -97,7 +97,9 @@ def check_env(json_path): old_packages = { package: date for package, date in rapids_package_dates.items() - if date is not None and date < old_threshold + if package not in EXCLUDED_PACKAGES + and date is not None + and date < old_threshold } if old_packages: exit_code = 1 From 171644a60cf5d9be847e2d696bad7a0e9be031c7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 15:01:10 -0500 Subject: [PATCH 16/24] Fix warning for day-old packages. --- ci/check_conda_nightly_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index 03ed5220..3880fd25 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -131,7 +131,7 @@ def check_env(json_path): ) for package, date in sorted(rapids_package_dates.items()): date_string = date.strftime("%Y-%m-%d") - status = WARNING if date < today - timedelta(days=1) else OKGREEN + status = WARNING if date < today else OKGREEN print(f"{status} - {package}: {date_string}{ENDC}") return exit_code From a27c3fc46233fb9680503d50a0de668512e2a0f8 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 15:04:52 -0500 Subject: [PATCH 17/24] Fix exclusion logic. --- ci/check_conda_nightly_env.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index 3880fd25..afd962e6 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -97,9 +97,7 @@ def check_env(json_path): old_packages = { package: date for package, date in rapids_package_dates.items() - if package not in EXCLUDED_PACKAGES - and date is not None - and date < old_threshold + if date is not None and date < old_threshold } if old_packages: exit_code = 1 @@ -114,7 +112,9 @@ def check_env(json_path): # If there are undated packages, show an error undated_packages = { - package: date for package, date in rapids_package_dates.items() if date is None + package: date + for package, date in rapids_package_dates.items() + if package not in EXCLUDED_PACKAGES and date is None } if undated_packages: exit_code = 1 From 2a5c29c6ace53d681e69178c54349797e48b7a52 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 15:41:26 -0500 Subject: [PATCH 18/24] Skip packages without dates. --- ci/check_conda_nightly_env.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index afd962e6..9a094aaa 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -130,6 +130,8 @@ def check_env(json_path): f"The following packages are less than {OLD_PACKAGE_THRESHOLD_DAYS} days old:" ) for package, date in sorted(rapids_package_dates.items()): + if date is None: + continue date_string = date.strftime("%Y-%m-%d") status = WARNING if date < today else OKGREEN print(f"{status} - {package}: {date_string}{ENDC}") From 4aeb2f5e9504d7a3b93ef412c58d68479df06b74 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 15:44:57 -0500 Subject: [PATCH 19/24] Update colors. --- ci/check_conda_nightly_env.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index 9a094aaa..e0f6c8fb 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -25,9 +25,9 @@ } # ANSI color codes used to highlight lines -FAIL = "\033[91m" -WARNING = "\033[93m" -OKGREEN = "\033[92m" +FAIL = "\033[31m" +WARNING = "\033[33m" +OKGREEN = "\033[32m" ENDC = "\033[0m" From 15c2c9dcd3486e58151f6821d7d2245573f50da3 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 15:49:01 -0500 Subject: [PATCH 20/24] Improve formatting. --- ci/check_conda_nightly_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index e0f6c8fb..0e86c2f3 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -108,7 +108,7 @@ def check_env(json_path): ) for package, date in sorted(old_packages.items()): date_string = date.strftime("%Y-%m-%d") - print(f"{FAIL} - {package}: {date_string}{ENDC}") + print(f"{FAIL} - {(package + ':'):<24}\t{date_string}{ENDC}") # If there are undated packages, show an error undated_packages = { @@ -134,7 +134,7 @@ def check_env(json_path): continue date_string = date.strftime("%Y-%m-%d") status = WARNING if date < today else OKGREEN - print(f"{status} - {package}: {date_string}{ENDC}") + print(f"{status} - {(package + ':'):<24}\t{date_string}{ENDC}") return exit_code From 9b720867abbf46788349645099c766161d9a9325 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 15:50:36 -0500 Subject: [PATCH 21/24] Remove nx-cugraph. --- ci/check_conda_nightly_env.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/check_conda_nightly_env.py b/ci/check_conda_nightly_env.py index 0e86c2f3..32bf4fae 100644 --- a/ci/check_conda_nightly_env.py +++ b/ci/check_conda_nightly_env.py @@ -18,8 +18,6 @@ "libxgboost", "py-xgboost", "xgboost", - # TODO: Remove nx-cugraph after https://github.com/rapidsai/cugraph/pull/4639 - "nx-cugraph", # TODO: Do we want ucx-proc on rapidsai or from conda-forge? "ucx-proc", } From 925dd5897873649b5ad9affda357f63fe03bf6e5 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 16:02:49 -0500 Subject: [PATCH 22/24] Re-enable PR builds of integration packages. --- .github/workflows/pr.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 487c21d1..41862216 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,11 +16,11 @@ jobs: - test-conda-nightly-env secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 -# build: -# secrets: inherit -# uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 -# with: -# build_type: pull-request + build: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + with: + build_type: pull-request test-conda-nightly-env: secrets: inherit # We use a build workflow so that we get CPU jobs and high matrix coverage From ede8cf2a1e4cb9a269a0fcc2c35833155079ce04 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Aug 2024 16:02:57 -0500 Subject: [PATCH 23/24] Add test workflow. --- .github/workflows/test.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..bb57d711 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,23 @@ +name: test + +on: + workflow_dispatch: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + +jobs: + test-conda-nightly-env: + secrets: inherit + # We use a build workflow so that we get CPU jobs and high matrix coverage + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/test_conda_nightly_env.sh" From f76b29a44e2bbcec945d94ce146600db0aa335fb Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 6 Sep 2024 16:45:19 -0500 Subject: [PATCH 24/24] Re-enable build job. --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 41862216..727e5180 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,7 +12,7 @@ concurrency: jobs: pr-builder: needs: - #- build + - build - test-conda-nightly-env secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10