diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bdfff9d22..9ddded445 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,6 +6,24 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--settings-file=./python/setup.cfg"] + files: ^python/ + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + args: ["--config=./python/setup.cfg"] + files: ^python/ + - repo: https://github.com/google/yapf + rev: v0.40.2 + hooks: + - id: yapf + args: ["--style", "./python/setup.cfg"] + files: ^python/ default_language_version: python: python3 diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml deleted file mode 100644 index ecd4003b6..000000000 --- a/ci/conda/environments/dev_env.yml +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Dependencies needed for development environment. Runtime deps are in meta.yml -name: mrc -channels: - - rapidsai - - nvidia/label/cuda-11.8.0 - - nvidia - - rapidsai-nightly - - conda-forge -dependencies: - - autoconf>=2.69 - - bash-completion - - benchmark=1.6.0 - - boost-cpp=1.82 - - ccache - - cmake=3.25 - - cuda-toolkit # Version comes from the channel above - - cxx-compiler # Sets up the distro versions of our compilers - - doxygen=1.9.2 - - flake8 - - flatbuffers=2.0 - - gcovr=5.0 - - gdb - - gflags=2.2 - - git>=2.35.3 # Needed for wildcards on safe.directory - - glog=0.6 - - gmock=1.13 - - graphviz=3.0 - - libgrpc=1.54.0 - - gtest=1.13 - - gxx=11.2 # Specifies which versions of GXX and GCC to use - - isort - - jinja2=3.0 - - lcov=1.15 - - libhwloc=2.9.2 - - libprotobuf=3.21 - - librmm=23.06 - - libtool - - ninja=1.10 - - nlohmann_json=3.9 - - numactl-libs-cos7-x86_64 - - numpy>=1.21 - - pip - - pkg-config=0.29 - - pybind11-stubgen=0.10 - - pytest - - pytest-timeout - - pytest-asyncio - - python=3.10 - - scikit-build>=0.17 - - sysroot_linux-64=2.17 - - ucx=1.14 - - yapf - - # Remove once `mamba repoquery whoneeds cudatoolkit` is empty. For now, we need to specify a version - - cudatoolkit=11.8 diff --git a/ci/githooks/pre-commit b/ci/githooks/pre-commit index e74e35fb3..7fa4b83a1 100755 --- a/ci/githooks/pre-commit +++ b/ci/githooks/pre-commit @@ -41,6 +41,5 @@ export CHANGED_FILES=$(GIT_DIFF_ARGS="--cached --name-only" get_modified_files) if [[ "${CHANGED_FILES}" != "" ]]; then run_and_check "python3 ci/scripts/copyright.py --git-diff-staged --update-current-year --verify-apache-v2 --git-add" - run_and_check "ci/scripts/python_checks.sh" SKIP_CLANG_TIDY=1 SKIP_IWYU=1 run_and_check "ci/scripts/cpp_checks.sh" fi diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index e64b36183..56eb72b95 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -36,8 +36,5 @@ ${MRC_ROOT}/ci/scripts/version_checks.sh rapids-logger "Running C++ style checks" ${MRC_ROOT}/ci/scripts/cpp_checks.sh -rapids-logger "Runing Python style checks" -${MRC_ROOT}/ci/scripts/python_checks.sh - rapids-logger "Checking copyright headers" python ${MRC_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} diff --git a/ci/scripts/python_checks.sh b/ci/scripts/python_checks.sh deleted file mode 100755 index fb6015735..000000000 --- a/ci/scripts/python_checks.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Based on style.sh from Morpheus - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -source ${SCRIPT_DIR}/common.sh - -# Ignore errors and set path -set +e -LC_ALL=C.UTF-8 -LANG=C.UTF-8 - -# Pre-populate the return values in case they are skipped -ISORT_RETVAL=0 -FLAKE_RETVAL=0 -YAPF_RETVAL=0 - -get_modified_files ${PYTHON_FILE_REGEX} MRC_MODIFIED_FILES - -# When invoked by the git pre-commit hook CHANGED_FILES will already be defined -if [[ -n "${MRC_MODIFIED_FILES}" ]]; then - echo -e "Running Python checks on ${#MRC_MODIFIED_FILES[@]} files:" - - for f in "${MRC_MODIFIED_FILES[@]}"; do - echo " $f" - done - - if [[ "${SKIP_ISORT}" == "" ]]; then - ISORT_OUTPUT=`python3 -m isort --settings-file ${PY_CFG} --filter-files --check-only ${MRC_MODIFIED_FILES[@]} 2>&1` - ISORT_RETVAL=$? - fi - - if [[ "${SKIP_FLAKE}" == "" ]]; then - FLAKE_OUTPUT=`python3 -m flake8 --config ${PY_CFG} ${MRC_MODIFIED_FILES[@]} 2>&1` - FLAKE_RETVAL=$? - fi - - if [[ "${SKIP_YAPF}" == "" ]]; then - # Run yapf. Will return 1 if there are any diffs - YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MRC_MODIFIED_FILES[@]} 2>&1` - YAPF_RETVAL=$? - fi -else - echo "No modified Python files to check" -fi - -# Output results if failure otherwise show pass -if [[ "${SKIP_ISORT}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: isort check\n\n" -elif [ "${ISORT_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n" - echo -e "${ISORT_OUTPUT}" - echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: isort style check\n\n" -fi - -if [[ "${SKIP_FLAKE}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: flake8 check\n\n" -elif [ "${FLAKE_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" - echo -e "${FLAKE_OUTPUT}" - echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: flake8 style check\n\n" -fi - -if [[ "${SKIP_YAPF}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: yapf check\n\n" -elif [ "${YAPF_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: yapf style check; begin output\n\n" - echo -e "Incorrectly formatted files:" - YAPF_OUTPUT=`echo "${YAPF_OUTPUT}" | sed -nr 's/^\+\+\+ ([^ ]*) *\(reformatted\)$/\1/p'` - echo -e "${YAPF_OUTPUT}" - echo -e "\n\n>>>> FAILED: yapf style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: yapf style check\n\n" -fi - -RETVALS=(${ISORT_RETVAL} ${FLAKE_RETVAL} ${YAPF_RETVAL}) -IFS=$'\n' -RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1` - -exit $RETVAL diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 313a30ff4..47efa30b9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -42,6 +42,7 @@ dependencies: - llvmdev=16 - ninja=1.10 - nlohmann_json=3.9 +- numactl-libs-cos7-x86_64 - numpy>=1.21 - pkg-config=0.29 - pre-commit diff --git a/conda/environments/ci_cuda-118_arch-x86_64.yaml b/conda/environments/ci_cuda-118_arch-x86_64.yaml index 30288a878..4935bb82e 100644 --- a/conda/environments/ci_cuda-118_arch-x86_64.yaml +++ b/conda/environments/ci_cuda-118_arch-x86_64.yaml @@ -21,7 +21,6 @@ dependencies: - cuda-version=11.8 - cxx-compiler - doxygen=1.9.2 -- flake8 - gcovr=5.0 - glog=0.6 - graphviz=3.0 @@ -33,6 +32,7 @@ dependencies: - librmm=23.06 - ninja=1.10 - nlohmann_json=3.9 +- numactl-libs-cos7-x86_64 - pkg-config=0.29 - pre-commit - pybind11-stubgen=0.10 @@ -42,5 +42,4 @@ dependencies: - python=3.10 - scikit-build>=0.17 - ucx=1.14 -- yapf name: ci_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 8d6f0d3b2..cc46ca5fb 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -63,6 +63,7 @@ dependencies: - librmm=23.06 - ninja=1.10 - nlohmann_json=3.9 + - numactl-libs-cos7-x86_64 - pkg-config=0.29 - pybind11-stubgen=0.10 - python=3.10 @@ -84,17 +85,17 @@ dependencies: - clang=16 - clangdev=16 - clangxx=16 + - flake8 - gdb - libclang-cpp=16 - libclang=16 - llvmdev=16 + - yapf code_style: common: - output_types: [conda] packages: - - flake8 - - yapf - include-what-you-use=0.20 testing: diff --git a/docs/quickstart/python/mrc_qs_python/_version.py b/docs/quickstart/python/mrc_qs_python/_version.py index 1ca6b055c..7fb4694cb 100644 --- a/docs/quickstart/python/mrc_qs_python/_version.py +++ b/docs/quickstart/python/mrc_qs_python/_version.py @@ -29,8 +29,7 @@ import re import subprocess import sys -from typing import Callable -from typing import Dict +from typing import Callable, Dict def get_keywords(): diff --git a/docs/quickstart/python/mrc_qs_python/ex02_reactive_operators/run.py b/docs/quickstart/python/mrc_qs_python/ex02_reactive_operators/run.py index e181ad053..0ed479f86 100644 --- a/docs/quickstart/python/mrc_qs_python/ex02_reactive_operators/run.py +++ b/docs/quickstart/python/mrc_qs_python/ex02_reactive_operators/run.py @@ -15,9 +15,10 @@ import dataclasses -import mrc from mrc.core import operators as ops +import mrc + @dataclasses.dataclass class MyCustomClass: diff --git a/docs/quickstart/python/versioneer.py b/docs/quickstart/python/versioneer.py index 5e21cd07d..350aa2069 100644 --- a/docs/quickstart/python/versioneer.py +++ b/docs/quickstart/python/versioneer.py @@ -286,8 +286,7 @@ import re import subprocess import sys -from typing import Callable -from typing import Dict +from typing import Callable, Dict class VersioneerConfig: