From 85c9a40c2dac6b83a687f97639f8abfe0ab17f46 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Wed, 25 Sep 2019 02:40:54 +0100 Subject: [PATCH 1/3] CI environments overhaul (#3340) * Rationalize and align CI environments. Add many optional dependencies to individual CI suites. * pynio and cdms2 are not available on Windows * cfgrib causes Python interpreter crash on Windows * dtype of np.arange defaults to int64 on Linux and int32 on Windows * Suppress failure to delete file on Windows * Mark hypotesis tests as @slow --- azure-pipelines.yml | 14 +-------- ci/azure/unit-tests.yml | 2 +- ci/requirements/py36-hypothesis.yml | 29 ------------------- ci/requirements/py36.yml | 44 ++++++++++++++++------------- ci/requirements/py37-windows.yml | 32 +++++++++++++++------ ci/requirements/py37.yml | 36 +++++++++++++---------- properties/README.md | 2 +- properties/test_encode_decode.py | 6 ++++ setup.cfg | 2 +- xarray/tests/test_backends.py | 5 +++- xarray/tests/test_sparse.py | 4 +-- 11 files changed, 85 insertions(+), 91 deletions(-) delete mode 100644 ci/requirements/py36-hypothesis.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 55c4ecb05f5..1c325d7734f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,7 +17,7 @@ jobs: py37-upstream-dev: conda_env: py37 upstream_dev: true - py36-flakey: + py36-flaky: conda_env: py36 pytest_extra_flags: --run-flaky --run-network-tests allow_failure: true @@ -90,15 +90,3 @@ jobs: cd doc sphinx-build -n -j auto -b html -d _build/doctrees . _build/html displayName: Build HTML docs - -- job: LinuxHypothesis - variables: - conda_env: py36-hypothesis - pool: - vmImage: 'ubuntu-16.04' - steps: - - template: ci/azure/install.yml - - bash: | - source activate xarray-tests - pytest properties - displayName: Property based tests diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index 7023d3119b8..43be5a79d47 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -11,7 +11,7 @@ steps: # https://github.com/microsoft/azure-pipelines-tasks/issues/9302 - bash: | source activate xarray-tests - pytest xarray \ + pytest \ --junitxml=junit/test-results.xml \ --cov=xarray \ --cov-report=xml \ diff --git a/ci/requirements/py36-hypothesis.yml b/ci/requirements/py36-hypothesis.yml deleted file mode 100644 index 57f4a236845..00000000000 --- a/ci/requirements/py36-hypothesis.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: xarray-tests -channels: - - conda-forge -dependencies: - - python=3.6 - - dask - - distributed - - h5py - - h5netcdf - - matplotlib - - netcdf4 - - pytest - - pytest-cov - - pytest-env - - coveralls - - hypothesis - - flake8 - - numpy>=1.12 - - pandas>=0.19 - - pip - - scipy - - seaborn - - toolz - - rasterio - - boto3 - - bottleneck - - zarr - - pydap - - lxml diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index 53da54e3b90..7e852b5c260 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -3,36 +3,40 @@ channels: - conda-forge dependencies: - python=3.6 + - black + - boto3 + - bottleneck + - cdms2 + - cfgrib>=0.9.2 - cftime + - coveralls - dask - distributed - - h5py + - flake8 - h5netcdf + - h5py + - hypothesis + - iris>=1.10 + - lxml - matplotlib - - netcdf4 - - pytest - - pytest-cov - - pytest-env - - coveralls - - flake8 - mypy==0.720 # Must match .pre-commit-config.yaml + - nc-time-axis + - netcdf4 + - numba - numpy>=1.12 - pandas>=0.19 - pip + - pseudonetcdf>=3.0.1 + - pydap + - pynio + - pytest + - pytest-cov + - pytest-env + - rasterio - scipy - seaborn + - sparse - toolz - - rasterio - - boto3 - - bottleneck - zarr - - pseudonetcdf>=3.0.1 - - cfgrib>=0.9.2 - - cdms2 - # Disabled due to conflicting dependencies: - # https://github.com/pydata/xarray/issues/3154 - # - pynio - - iris>=1.10 - - pydap - - lxml - + - pip: + - numbagg diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 51936279134..31af80357ad 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -3,24 +3,40 @@ channels: - conda-forge dependencies: - python=3.7 + - black + - boto3 + - bottleneck + # - cdms2 # Not available on Windows + # - cfgrib>=0.9.2 # Causes Python interpreter crash on Windows - cftime - - nc-time-axis + - coveralls - dask - distributed - - h5py + - flake8 - h5netcdf + - h5py + - hypothesis + - iris>=1.10 + - lxml - matplotlib + - mypy==0.720 # Must match .pre-commit-config.yaml + - nc-time-axis - netcdf4 - - pytest - - pytest-cov - - pytest-env - - coveralls + - numba - numpy>=1.12 - pandas>=0.19 - pip + - pseudonetcdf>=3.0.1 + - pydap + # - pynio # Not available on Windows + - pytest + - pytest-cov + - pytest-env + - rasterio - scipy - seaborn + - sparse - toolz - - rasterio - - boto3 - zarr + - pip: + - numbagg diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 538d4679a79..72b6c76ea37 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -3,34 +3,40 @@ channels: - conda-forge dependencies: - python=3.7 + - black + - boto3 + - bottleneck + - cdms2 + - cfgrib>=0.9.2 - cftime - - nc-time-axis + - coveralls - dask - distributed - - h5py + - flake8 - h5netcdf + - h5py + - hypothesis + - iris>=1.10 + - lxml - matplotlib - - netcdf4 - - pytest - - pytest-cov - - pytest-env - - coveralls - - flake8 - mypy==0.720 # Must match .pre-commit-config.yaml + - nc-time-axis + - netcdf4 + - numba - numpy>=1.12 - pandas>=0.19 - pip + - pseudonetcdf>=3.0.1 + - pydap + - pynio + - pytest + - pytest-cov + - pytest-env + - rasterio - scipy - seaborn - sparse - toolz - - rasterio - - boto3 - - bottleneck - zarr - - pseudonetcdf>=3.0.1 - - cfgrib>=0.9.2 - - lxml - - pydap - pip: - numbagg diff --git a/properties/README.md b/properties/README.md index 711062a2473..86c1d41d81d 100644 --- a/properties/README.md +++ b/properties/README.md @@ -3,7 +3,7 @@ This directory contains property-based tests using a library called [Hypothesis](https://github.com/HypothesisWorks/hypothesis-python). -The property tests for Xarray are a work in progress - more are always welcome. +The property tests for xarray are a work in progress - more are always welcome. They are stored in a separate directory because they tend to run more examples and thus take longer, and so that local development can run a test suite without needing to `pip install hypothesis`. diff --git a/properties/test_encode_decode.py b/properties/test_encode_decode.py index b8f52e3de7a..c7839608981 100644 --- a/properties/test_encode_decode.py +++ b/properties/test_encode_decode.py @@ -4,6 +4,10 @@ These ones pass, just as you'd hope! """ +import pytest + +pytest.importorskip("hypothesis") + import hypothesis.extra.numpy as npst import hypothesis.strategies as st from hypothesis import given, settings @@ -23,6 +27,7 @@ ) +@pytest.mark.slow @given(st.data(), an_array) def test_CFMask_coder_roundtrip(data, arr): names = data.draw( @@ -36,6 +41,7 @@ def test_CFMask_coder_roundtrip(data, arr): xr.testing.assert_identical(original, roundtripped) +@pytest.mark.slow @given(st.data(), an_array) def test_CFScaleOffset_coder_roundtrip(data, arr): names = data.draw( diff --git a/setup.cfg b/setup.cfg index 114f71f4a9f..e95834949b4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ universal = 1 [tool:pytest] python_files=test_*.py -testpaths=xarray/tests +testpaths=xarray/tests properties # Fixed upstream in https://github.com/kwgoodman/bottleneck/pull/199 filterwarnings = ignore:Using a non-tuple sequence for multidimensional indexing is deprecated:FutureWarning diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index f6254b32f4f..04801a64c7f 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -3487,7 +3487,10 @@ def test_uamiv_format_write(self): "example.uamiv", engine="pseudonetcdf", backend_kwargs=fmtkw ) with self.roundtrip( - expected, save_kwargs=fmtkw, open_kwargs={"backend_kwargs": fmtkw} + expected, + save_kwargs=fmtkw, + open_kwargs={"backend_kwargs": fmtkw}, + allow_cleanup_failure=True, ) as actual: assert_identical(expected, actual) diff --git a/xarray/tests/test_sparse.py b/xarray/tests/test_sparse.py index 80f80a93a1c..9393d073cb7 100644 --- a/xarray/tests/test_sparse.py +++ b/xarray/tests/test_sparse.py @@ -656,7 +656,7 @@ def test_dataarray_repr(self): a = xr.DataArray( sparse.COO.from_numpy(np.ones(4)), dims=["x"], - coords={"y": ("x", sparse.COO.from_numpy(np.arange(4)))}, + coords={"y": ("x", sparse.COO.from_numpy(np.arange(4, dtype="i8")))}, ) expected = dedent( """\ @@ -671,7 +671,7 @@ def test_dataarray_repr(self): def test_dataset_repr(self): ds = xr.Dataset( data_vars={"a": ("x", sparse.COO.from_numpy(np.ones(4)))}, - coords={"y": ("x", sparse.COO.from_numpy(np.arange(4)))}, + coords={"y": ("x", sparse.COO.from_numpy(np.arange(4, dtype="i8")))}, ) expected = dedent( """\ From ea101f571f3b2f13ab02c7b57453571af559f940 Mon Sep 17 00:00:00 2001 From: Tom Nicholas <35968931+TomNicholas@users.noreply.github.com> Date: Thu, 26 Sep 2019 10:51:58 +0200 Subject: [PATCH 2/3] Bugfix/plot accept coord dim (#3345) Bug in plot.line fixed by ensuring 1D coords are cast down to their associated dims. There was previously two particular cases where this would not happen. --- doc/whats-new.rst | 8 ++++++-- xarray/plot/plot.py | 8 ++++++-- xarray/tests/test_plot.py | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d8d3382675e..828a66db6ab 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -24,6 +24,9 @@ Bug fixes reinstated for :class:`DataArray` and :class:`Dataset` objects only. Internal xarray objects remain unaddressable by weakref in order to save memory. (:issue:`3317`) by `Guido Imperiale `_. +- Line plots with the ``x`` or ``y`` argument set to a 1D non-dimensional coord + now plot the correct data for 2D DataArrays. + (:issue:`3334`). By `Tom Nicholas `_. Documentation ~~~~~~~~~~~~~ @@ -544,8 +547,9 @@ Bug fixes from higher frequencies to lower frequencies. Datapoints outside the bounds of the original time coordinate are now filled with NaN (:issue:`2197`). By `Spencer Clark `_. -- Line plots with the ``x`` argument set to a non-dimensional coord now plot the correct data for 1D DataArrays. - (:issue:`27251`). By `Tom Nicholas `_. +- Line plots with the ``x`` argument set to a non-dimensional coord now plot + the correct data for 1D DataArrays. + (:issue:`2725`). By `Tom Nicholas `_. - Subtracting a scalar ``cftime.datetime`` object from a :py:class:`CFTimeIndex` now results in a :py:class:`pandas.TimedeltaIndex` instead of raising a ``TypeError`` (:issue:`2671`). By `Spencer Clark diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 8ca62ef58f1..7938f9b027b 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -83,7 +83,9 @@ def _infer_line_data(darray, x, y, hue): ) else: - yplt = darray.transpose(xname, huename) + xdim, = darray[xname].dims + huedim, = darray[huename].dims + yplt = darray.transpose(xdim, huedim) else: yname, huename = _infer_xy_labels(darray=darray, x=y, y=hue) @@ -100,7 +102,9 @@ def _infer_line_data(darray, x, y, hue): ) else: - xplt = darray.transpose(yname, huename) + ydim, = darray[yname].dims + huedim, = darray[huename].dims + xplt = darray.transpose(ydim, huedim) huelabel = label_from_attrs(darray[huename]) hueplt = darray[huename] diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 020a49b0114..5ce75ad2396 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -170,6 +170,23 @@ def test_infer_line_data(self): line = current.plot.line()[0] assert_array_equal(line.get_xdata(), current.coords["t"].values) + def test_line_plot_along_1d_coord(self): + # Test for bug in GH #3334 + x_coord = xr.DataArray(data=[0.1, 0.2], dims=["x"]) + t_coord = xr.DataArray(data=[10, 20], dims=["t"]) + + da = xr.DataArray( + data=np.array([[0, 1], [5, 9]]), + dims=["x", "t"], + coords={"x": x_coord, "time": t_coord}, + ) + + line = da.plot(x="time", hue="x")[0] + assert_array_equal(line.get_xdata(), da.coords["time"].values) + + line = da.plot(y="time", hue="x")[0] + assert_array_equal(line.get_ydata(), da.coords["time"].values) + def test_2d_line(self): with raises_regex(ValueError, "hue"): self.darray[:, :, 0].plot.line() From 16fdac93c09036b44f90201d9078b17808f819e0 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Thu, 26 Sep 2019 10:38:46 +0100 Subject: [PATCH 3/3] CI test suites with pinned minimum dependencies (#3346) * CI test suites with pinned minimum dependencies * code review * Clarity re lxml --- azure-pipelines.yml | 8 +++- .../{py35-min.yml => py35-bare-minimum.yml} | 0 ci/requirements/py36-min-all-deps.yml | 43 +++++++++++++++++++ ci/requirements/py36-min-nep18.yml | 17 ++++++++ ci/requirements/py36.yml | 3 +- ci/requirements/py37-windows.yml | 3 +- ci/requirements/py37.yml | 3 +- xarray/tests/test_plot.py | 2 +- 8 files changed, 73 insertions(+), 6 deletions(-) rename ci/requirements/{py35-min.yml => py35-bare-minimum.yml} (100%) create mode 100644 ci/requirements/py36-min-all-deps.yml create mode 100644 ci/requirements/py36-min-nep18.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1c325d7734f..d023aa317c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,8 +8,12 @@ jobs: - job: Linux strategy: matrix: - py35-min: - conda_env: py35-min + py35-bare-minimum: + conda_env: py35-bare-minimum + py36-min-all-deps: + conda_env: py36-min-all-deps + py36-min-nep18: + conda_env: py36-min-nep18 py36: conda_env: py36 py37: diff --git a/ci/requirements/py35-min.yml b/ci/requirements/py35-bare-minimum.yml similarity index 100% rename from ci/requirements/py35-min.yml rename to ci/requirements/py35-bare-minimum.yml diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml new file mode 100644 index 00000000000..1829f2a11e3 --- /dev/null +++ b/ci/requirements/py36-min-all-deps.yml @@ -0,0 +1,43 @@ +name: xarray-tests +channels: + - conda-forge +dependencies: + - python=3.6.7 + - black + - boto3=1.9.235 + - bottleneck=1.2.1 + - cdms2=3.1.3 + - cfgrib=0.9.7.2 + - cftime=1.0.3.4 + - coveralls + - dask=2.4.0 + - distributed=2.4.0 + - flake8 + - h5netcdf=0.7.4 + - h5py=2.10.0 + - hdf5=1.10.5 + - hypothesis + - iris=2.2.0 + - lxml=4.4.1 # optional dep of pydap + - matplotlib=3.1.1 + - mypy==0.720 # Must match .pre-commit-config.yaml + - nc-time-axis=1.2.0 + - netcdf4=1.5.1.2 + - numba=0.45.1 + - numpy=1.17.2 + - pandas=0.25.1 + - pip + - pseudonetcdf=3.0.2 + - pydap=3.2.2 + - pynio=1.5.5 + - pytest + - pytest-cov + - pytest-env + - rasterio=1.0.28 + - scipy=1.3.1 + - seaborn=0.9.0 + # - sparse # See py36-min-nep18.yml + - toolz=0.10.0 + - zarr=2.3.2 + - pip: + - numbagg==0.1 diff --git a/ci/requirements/py36-min-nep18.yml b/ci/requirements/py36-min-nep18.yml new file mode 100644 index 00000000000..8680e412a99 --- /dev/null +++ b/ci/requirements/py36-min-nep18.yml @@ -0,0 +1,17 @@ +name: xarray-tests +channels: + - conda-forge +dependencies: + # Optional dependencies that require NEP18, such as sparse, + # require drastically newer packages than everything else + - python=3.6.7 + - coveralls + - dask=2.4.0 + - distributed=2.4.0 + - numpy=1.17 + - pandas=0.25 + - pytest + - pytest-cov + - pytest-env + - scipy=1.3 + - sparse=0.8.0 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index 7e852b5c260..187a9c79fbf 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -15,9 +15,10 @@ dependencies: - flake8 - h5netcdf - h5py + - hdf5 - hypothesis - iris>=1.10 - - lxml + - lxml # optional dep of pydap - matplotlib - mypy==0.720 # Must match .pre-commit-config.yaml - nc-time-axis diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 31af80357ad..24297327393 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -15,9 +15,10 @@ dependencies: - flake8 - h5netcdf - h5py + - hdf5 - hypothesis - iris>=1.10 - - lxml + - lxml # optional dep of pydap - matplotlib - mypy==0.720 # Must match .pre-commit-config.yaml - nc-time-axis diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 72b6c76ea37..5a328c64cf9 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -15,9 +15,10 @@ dependencies: - flake8 - h5netcdf - h5py + - hdf5 - hypothesis - iris>=1.10 - - lxml + - lxml # optional dep of pydap - matplotlib - mypy==0.720 # Must match .pre-commit-config.yaml - nc-time-axis diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 5ce75ad2396..99a72d68ad8 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -1962,7 +1962,7 @@ def test_datetime_hue(self, hue_style): ds2.plot.scatter(x="A", y="B", hue="hue", hue_style=hue_style) def test_facetgrid_hue_style(self): - # Can't move this to pytest.mark.parametrize because py35-min + # Can't move this to pytest.mark.parametrize because py35-bare-minimum # doesn't have mpl. for hue_style, map_type in zip( ["discrete", "continuous"], [list, mpl.collections.PathCollection]