diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index d09661121b..707a87bc1e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,7 +1,7 @@ name: Bug Report description: File a bug report title: "[Bug]: " -labels: ["bug"] +labels: ["bug :bug:"] body: - type: markdown @@ -44,18 +44,19 @@ body: label: Python version description: What Python version? options: + - 3.7 - 3.8 - 3.9 - - 3.10 - - 3.7 + - "3.10" - type: dropdown id: pytorch-version attributes: label: PyTorch version description: What PyTorch version? options: + - 1.12 - 1.11 - - 1.10 + - "1.10" - 1.9 - 1.8 - 1.7 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 267787f455..5185637281 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -50,6 +50,3 @@ my be illegible. It may be easiest to save the output of each to a file. #### Does this change modify the behaviour of other functions? If so, which? yes / no - - -skip ci diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 1791f1dd8f..1d823f452e 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -29,6 +29,10 @@ categories: labels: - 'io' - 'communication' + - title: 'Google Summer of Code 2022' + label: 'GSoC22' + - title: 'Array API' + label: 'array API' change-template: '- #$NUMBER $TITLE (by @$AUTHOR)' categorie-template: '### $TITLE' exclude-labels: diff --git a/.github/workflows/mirrorci.yml b/.github/workflows/ci_cpu.yml similarity index 58% rename from .github/workflows/mirrorci.yml rename to .github/workflows/ci_cpu.yml index 54d0f4a273..a8b3f2efa7 100644 --- a/.github/workflows/mirrorci.yml +++ b/.github/workflows/ci_cpu.yml @@ -10,11 +10,11 @@ jobs: - name: Mirror + trigger CI uses: SvanBoxel/gitlab-mirror-and-ci-action@master with: - args: "https://gitlab.jsc.fz-juelich.de/haf/heat" + args: "https://gitlab.hzdr.de/haf/heat" env: - FORCE_PUSH: "false" - GITLAB_HOSTNAME: "gitlab.jsc.fz-juelich.de" + FORCE_PUSH: "true" + GITLAB_HOSTNAME: "gitlab.hzdr.de" GITLAB_USERNAME: "" - GITLAB_PASSWORD: ${{ secrets.GITLAB_TOKEN }} - GITLAB_PROJECT_ID: "4935" + GITLAB_PASSWORD: ${{ secrets.GITLAB_TOKEN_1 }} + GITLAB_PROJECT_ID: "845" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10a3b4f889..822a501a9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,19 +1,29 @@ test: - image: ubuntu:20.04 + image: nvidia/cuda:11.6.2-runtime-ubuntu20.04 tags: - - heat + - cuda + - x86_64 script: - - apt update - - apt -y install build-essential python3-pip curl - - DEBIAN_FRONTEND=noninteractive apt -y install libopenmpi-dev openmpi-bin openmpi-doc - - apt -y install libhdf5-openmpi-dev libpnetcdf-dev - - pip install pytest coverage - - pip install .[hdf5,netcdf] - - COVERAGE_FILE=report/cov/coverage1 mpirun --allow-run-as-root -n 1 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report1.xml heat/ - - COVERAGE_FILE=report/cov/coverage2 mpirun --allow-run-as-root -n 3 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report2.xml heat/ - - COVERAGE_FILE=report/cov/coverage5 mpirun --allow-run-as-root -n 5 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report5.xml heat/ - - COVERAGE_FILE=report/cov/coverage8 mpirun --allow-run-as-root -n 8 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report8.xml heat/ - - coverage combine report/cov/* - - coverage report - - coverage xml - - curl -s https://codecov.io/bash | bash -s -- -c -F unit -f coverage.xml -t $CODECOV_TOKEN || echo "Codecov failed to upload" + - apt update + - apt -y install build-essential python3-pip curl git + - DEBIAN_FRONTEND=noninteractive apt -y install libopenmpi-dev openmpi-bin openmpi-doc + - apt -y install libhdf5-openmpi-dev libpnetcdf-dev + - pip install pytest coverage + - pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116 + - pip install .[hdf5,netcdf] + - COVERAGE_FILE=report/cov/coverage1 HEAT_TEST_USE_DEVICE=cpu mpirun --allow-run-as-root -n 1 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report1.xml heat/ + - COVERAGE_FILE=report/cov/coverage2 HEAT_TEST_USE_DEVICE=gpu mpirun --allow-run-as-root -n 3 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report3.xml heat/ + - COVERAGE_FILE=report/cov/coverage5 HEAT_TEST_USE_DEVICE=cpu mpirun --allow-run-as-root -n 5 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report5.xml heat/ + - COVERAGE_FILE=report/cov/coverage8 HEAT_TEST_USE_DEVICE=gpu mpirun --allow-run-as-root -n 6 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report6.xml heat/ + - coverage combine report/cov/* + - coverage report + - coverage xml + - curl -Os https://uploader.codecov.io/latest/linux/codecov + - chmod +x codecov + - ./codecov -F unit -f ./coverage.xml -t $CODECOV_TOKEN -Z + artifacts: + when: always + paths: + - report/test/report*.xml + reports: + junit: report/test/report*.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bff221de53..20132efc45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: check-added-large-files - id: flake8 - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 22.10.0 hooks: - id: black - repo: https://github.com/pycqa/pydocstyle diff --git a/README.md b/README.md index 7d9510a677..fc2b53886e 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,7 @@ Heat is a distributed tensor framework for high performance data analytics. Project Status -------------- - -[![Jenkins](https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fheat-ci.fz-juelich.de%2Fjob%2Fheat%2Fjob%2Fheat%2Fjob%2Fmain%2F&label=CPU)](https://heat-ci.fz-juelich.de/blue/organizations/jenkins/heat%2Fheat/activity?branch=main) -[![Jenkins](https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fheat-ci.fz-juelich.de%2Fjob%2FGPU%2520Cluster%2Fjob%2Fmain%2F&label=GPU)](https://heat-ci.fz-juelich.de/blue/organizations/jenkins/GPU%20Cluster%2Fmain/activity) +[![Mirror and run GitLab CI](https://github.com/helmholtz-analytics/heat/actions/workflows/ci_cpu.yml/badge.svg)](https://github.com/helmholtz-analytics/heat/actions/workflows/ci_cpu.yml) [![Documentation Status](https://readthedocs.org/projects/heat/badge/?version=latest)](https://heat.readthedocs.io/en/latest/?badge=latest) [![codecov](https://codecov.io/gh/helmholtz-analytics/heat/branch/main/graph/badge.svg)](https://codecov.io/gh/helmholtz-analytics/heat) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) diff --git a/codecov.yml b/codecov.yml index 169384c39e..95cd8ab0a5 100644 --- a/codecov.yml +++ b/codecov.yml @@ -13,31 +13,16 @@ coverage: # basic target: auto threshold: 3% - base: auto flags: - unit - - gpu paths: - "heat" - # advanced settings - branches: - - main - if_ci_failed: error #success, failure, error, ignore - informational: false - only_pulls: false patch: default: # basic target: auto threshold: 3% - base: auto - # advanced - branches: - - main - if_ci_failed: error #success, failure, error, ignore - only_pulls: false flags: - "unit" - - "gpu" paths: - "heat" diff --git a/heat/core/_operations.py b/heat/core/_operations.py index 75217f6b2d..08f04e305c 100644 --- a/heat/core/_operations.py +++ b/heat/core/_operations.py @@ -422,7 +422,7 @@ def __reduce_op( balanced = x.balanced # if local tensor is empty, replace it with the identity element - if 0 in x.lshape and (axis is None or (x.split in axis)): + if x.is_distributed() and 0 in x.lshape and (axis is None or split in axis): if neutral is None: neutral = float("nan") neutral_shape = x.gshape[:split] + (1,) + x.gshape[split + 1 :] diff --git a/heat/core/logical.py b/heat/core/logical.py index 9b4dcc8f79..a6be081ea7 100644 --- a/heat/core/logical.py +++ b/heat/core/logical.py @@ -91,6 +91,9 @@ def all( def local_all(t, *args, **kwargs): return torch.all(t != 0, *args, **kwargs) + if keepdim and axis is None: + axis = tuple(range(x.ndim)) + return _operations.__reduce_op( x, local_all, MPI.LAND, axis=axis, out=out, neutral=1, keepdim=keepdim ) @@ -196,6 +199,9 @@ def any( def local_any(t, *args, **kwargs): return torch.any(t != 0, *args, **kwargs) + if keepdim and axis is None: + axis = tuple(range(x.ndim)) + return _operations.__reduce_op( x, local_any, MPI.LOR, axis=axis, out=out, neutral=0, keepdim=keepdim ) diff --git a/heat/core/statistics.py b/heat/core/statistics.py index 7608da7387..d6c0305a34 100644 --- a/heat/core/statistics.py +++ b/heat/core/statistics.py @@ -206,14 +206,12 @@ def average( Axis or axes along which to average ``x``. The default, ``axis=None``, will average over all of the elements of the input array. If axis is negative it counts from the last to the first axis. - #TODO Issue #351: If axis is a tuple of ints, averaging is performed on all of the axes - specified in the tuple instead of a single axis or all the axes as - before. weights : DNDarray, optional An array of weights associated with the values in ``x``. Each value in ``x`` contributes to the average according to its associated weight. The weights array can either be 1D (in which case its length must be the size of ``x`` along the given axis) or of the same shape as ``x``. + Weighted average over tuple axis requires weights array to be of the same shape as ``x``. If ``weights=None``, then all data in ``x`` are assumed to have a weight equal to one, the result is equivalent to :func:`mean`. returned : bool, optional @@ -269,7 +267,9 @@ def average( if axis is None: raise TypeError("Axis must be specified when shapes of x and weights differ.") elif isinstance(axis, tuple): - raise NotImplementedError("Weighted average over tuple axis not implemented yet.") + raise ValueError( + "Weighted average over tuple axis requires weights to be of the same shape as x." + ) if weights.ndim != 1: raise TypeError("1D weights expected when shapes of x and weights differ.") if weights.gshape[0] != x.gshape[axis]: diff --git a/heat/core/tests/test_arithmetics.py b/heat/core/tests/test_arithmetics.py index f1fc79d2d3..9e89a7e119 100644 --- a/heat/core/tests/test_arithmetics.py +++ b/heat/core/tests/test_arithmetics.py @@ -646,6 +646,10 @@ def test_prod(self): self.assertEqual(shape_split_axis_tuple_prod.split, None) self.assertTrue((shape_split_axis_tuple_prod == expected_result).all()) + # empty array + empty = ht.array([]) + self.assertEqual(ht.prod(empty), ht.array([1.0])) + # exceptions with self.assertRaises(ValueError): ht.ones(array_len).prod(axis=1) @@ -792,6 +796,10 @@ def test_sum(self): self.assertEqual(shape_split_axis_tuple_sum.split, None) self.assertTrue((shape_split_axis_tuple_sum == expected_result).all()) + # empty array + empty = ht.array([]) + self.assertEqual(ht.sum(empty), ht.array([0.0])) + # exceptions with self.assertRaises(ValueError): ht.ones(array_len).sum(axis=1) diff --git a/heat/core/tests/test_logical.py b/heat/core/tests/test_logical.py index a995d53db3..691df7ec62 100644 --- a/heat/core/tests/test_logical.py +++ b/heat/core/tests/test_logical.py @@ -140,6 +140,32 @@ def test_all(self): out_noaxis = ht.zeros((1, 2, 3, 5), split=1) ht.all(ones_noaxis_split_axis_neg, axis=-2, out=out_noaxis) + # test keepdim + ones_2d = ht.ones((1, 1)) + self.assertEqual(ones_2d.all(keepdim=True).shape, ones_2d.shape) + + ones_2d_split = ht.ones((2, 2), split=0) + keepdim_is_one = ones_2d_split.all(keepdim=True) + self.assertEqual(keepdim_is_one.shape, (1, 1)) + self.assertEqual(keepdim_is_one.split, None) + keepdim_is_one = ones_2d_split.all(axis=0, keepdim=True) + self.assertEqual(keepdim_is_one.shape, (1, 2)) + self.assertEqual(keepdim_is_one.split, None) + keepdim_is_one = ones_2d_split.all(axis=1, keepdim=True) + self.assertEqual(keepdim_is_one.shape, (2, 1)) + self.assertEqual(keepdim_is_one.split, 0) + + ones_2d_split = ht.ones((2, 2), split=1) + keepdim_is_one = ones_2d_split.all(keepdim=True) + self.assertEqual(keepdim_is_one.shape, (1, 1)) + self.assertEqual(keepdim_is_one.split, None) + keepdim_is_one = ones_2d_split.all(axis=0, keepdim=True) + self.assertEqual(keepdim_is_one.shape, (1, 2)) + self.assertEqual(keepdim_is_one.split, 1) + keepdim_is_one = ones_2d_split.all(axis=1, keepdim=True) + self.assertEqual(keepdim_is_one.shape, (2, 1)) + self.assertEqual(keepdim_is_one.split, None) + # exceptions with self.assertRaises(ValueError): ht.ones(array_len).all(axis=1) @@ -212,6 +238,32 @@ def test_any(self): self.assertEqual(any_tensor.dtype, ht.bool) self.assertTrue(ht.equal(any_tensor, res)) + # test keepdim + ones_2d = ht.ones((1, 1)) + self.assertEqual(ones_2d.any(keepdim=True).shape, ones_2d.shape) + + ones_2d_split = ht.ones((2, 2), split=0) + keepdim_any = ones_2d_split.any(keepdim=True) + self.assertEqual(keepdim_any.shape, (1, 1)) + self.assertEqual(keepdim_any.split, None) + keepdim_any = ones_2d_split.any(axis=0, keepdim=True) + self.assertEqual(keepdim_any.shape, (1, 2)) + self.assertEqual(keepdim_any.split, None) + keepdim_any = ones_2d_split.any(axis=1, keepdim=True) + self.assertEqual(keepdim_any.shape, (2, 1)) + self.assertEqual(keepdim_any.split, 0) + + ones_2d_split = ht.ones((2, 2), split=1) + keepdim_any = ones_2d_split.any(keepdim=True) + self.assertEqual(keepdim_any.shape, (1, 1)) + self.assertEqual(keepdim_any.split, None) + keepdim_any = ones_2d_split.any(axis=0, keepdim=True) + self.assertEqual(keepdim_any.shape, (1, 2)) + self.assertEqual(keepdim_any.split, 1) + keepdim_any = ones_2d_split.any(axis=1, keepdim=True) + self.assertEqual(keepdim_any.shape, (2, 1)) + self.assertEqual(keepdim_any.split, None) + def test_isclose(self): size = ht.communication.MPI_WORLD.size a = ht.float32([[2, 2], [2, 2]]) diff --git a/setup.py b/setup.py index 5aaea7ff3e..e97991ec3b 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires=[ "mpi4py>=3.0.0", "numpy>=1.13.0", - "torch>=1.7.0, <=1.12.1", + "torch>=1.7.0, <1.13", "scipy>=0.14.0", "pillow>=6.0.0", "torchvision>=0.8.0",