From 05e9d95921b60073aee35db06b9ad0c0d813e37b Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 22 Apr 2022 19:25:05 -0400 Subject: [PATCH 01/15] bump pandas min from 0.22.0 to 0.25.0 --- benchmarks/asv.conf.json | 2 +- ci/requirements-py36-min.yml | 2 +- ci/requirements-py36.yml | 2 +- ci/requirements-py37.yml | 2 +- ci/requirements-py38.yml | 2 +- ci/requirements-py39.yml | 2 +- docs/sphinx/source/whatsnew/v0.9.2.rst | 5 ++++- setup.py | 2 +- 8 files changed, 11 insertions(+), 8 deletions(-) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index bf632065a5..79773e928f 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -116,7 +116,7 @@ { "python": "3.6", "numpy": "1.16.0", - "pandas": "0.22.0", + "pandas": "0.25.0", "scipy": "1.2.0", // Note: these don't have a minimum in setup.py "h5py": "2.10.0", diff --git a/ci/requirements-py36-min.yml b/ci/requirements-py36-min.yml index 495099cf8f..dfff0a9f97 100644 --- a/ci/requirements-py36-min.yml +++ b/ci/requirements-py36-min.yml @@ -16,7 +16,7 @@ dependencies: - dataclasses - h5py==3.1.0 - numpy==1.16.0 - - pandas==0.22.0 + - pandas==0.25.0 - scipy==1.2.0 - pytest-rerunfailures # conda version is >3.6 - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1 diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index 295bedda7c..596a2e0bbd 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -11,7 +11,7 @@ dependencies: - nose - numba - numpy >= 1.16.0 - - pandas >= 0.22.0 + - pandas >= 0.25.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 241cafe128..8a971d728a 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -11,7 +11,7 @@ dependencies: - nose - numba - numpy >= 1.16.0 - - pandas >= 0.22.0 + - pandas >= 0.25.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py38.yml b/ci/requirements-py38.yml index 6ff5d2da0f..db763c3d73 100644 --- a/ci/requirements-py38.yml +++ b/ci/requirements-py38.yml @@ -11,7 +11,7 @@ dependencies: - nose - numba - numpy >= 1.16.0 - - pandas >= 0.22.0 + - pandas >= 0.25.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py39.yml b/ci/requirements-py39.yml index 32abde067a..3621a737b6 100644 --- a/ci/requirements-py39.yml +++ b/ci/requirements-py39.yml @@ -11,7 +11,7 @@ dependencies: - nose # - numba # python 3.9 compat in early 2021 - numpy >= 1.16.0 - - pandas >= 0.22.0 + - pandas >= 0.25.0 - pip - pytest - pytest-cov diff --git a/docs/sphinx/source/whatsnew/v0.9.2.rst b/docs/sphinx/source/whatsnew/v0.9.2.rst index 1982759bf5..650567f2ba 100644 --- a/docs/sphinx/source/whatsnew/v0.9.2.rst +++ b/docs/sphinx/source/whatsnew/v0.9.2.rst @@ -13,7 +13,8 @@ Bug fixes ~~~~~~~~~ * :py:func:`pvlib.irradiance.get_total_irradiance` and :py:func:`pvlib.solarposition.spa_python` now raise an error instead - of silently ignoring unknown parameters (:ghpull:`1437`) + of silently ignoring unknown parameters (:pull:`1437`) + Testing ~~~~~~~ @@ -23,8 +24,10 @@ Documentation Benchmarking ~~~~~~~~~~~~~ * Updated version of numba in asv.conf from 0.36.1 to 0.40.0 to solve numba/numpy conflict. (:issue:`1439`, :pull:`1440`) + Requirements ~~~~~~~~~~~~ +* Minimum pandas version increased to v0.25.0, released July 18, 2019. Contributors ~~~~~~~~~~~~ diff --git a/setup.py b/setup.py index 892f23579c..96b4737515 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ URL = 'https://github.com/pvlib/pvlib-python' INSTALL_REQUIRES = ['numpy >= 1.16.0', - 'pandas >= 0.22.0', + 'pandas >= 0.25.0', 'pytz', 'requests', 'scipy >= 1.2.0', From 0fbc5340d0aeec9db7e29912a68a21d309eb317b Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 22 Apr 2022 19:32:18 -0400 Subject: [PATCH 02/15] fix buggy test__check_pandas_assert_kwargs don't use monkeypatch and mocker in the same test function. https://github.com/pytest-dev/pytest-mock/issues/289 --- pvlib/tests/test_conftest.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pvlib/tests/test_conftest.py b/pvlib/tests/test_conftest.py index a42d28d463..0892c52a47 100644 --- a/pvlib/tests/test_conftest.py +++ b/pvlib/tests/test_conftest.py @@ -52,22 +52,17 @@ def test_use_fixture_with_decorator(some_data): 'assert_frame_equal']) @pytest.mark.parametrize('pd_version', ['1.0.0', '1.1.0']) @pytest.mark.parametrize('check_less_precise', [True, False]) -def test__check_pandas_assert_kwargs(mocker, monkeypatch, - function_name, pd_version, +def test__check_pandas_assert_kwargs(mocker, function_name, pd_version, check_less_precise): # test that conftest._check_pandas_assert_kwargs returns appropriate # kwargs for the assert_x_equal functions - # patch the pandas assert; not interested in actually calling them: - def patched_assert(*args, **kwargs): - pass - - monkeypatch.setattr(pandas.testing, function_name, patched_assert) - # then attach a spy to it so we can see what args it is called with: - mocked_function = mocker.spy(pandas.testing, function_name) + # patch the pandas assert; not interested in actually calling them, + # plus we want to spy on how they get called. + spy = mocker.patch('pandas.testing.' + function_name) # patch pd.__version__ to exercise the two branches in # conftest._check_pandas_assert_kwargs - monkeypatch.setattr(pandas, '__version__', pd_version) + mocker.patch('pandas.__version__', new=pd_version) # finally, run the function and check what args got passed to pandas: assert_function = getattr(conftest, function_name) @@ -79,4 +74,4 @@ def patched_assert(*args, **kwargs): else: expected_kwargs = {'check_less_precise': check_less_precise} - mocked_function.assert_called_with(*args, **expected_kwargs) + spy.assert_called_once_with(*args, **expected_kwargs) From 23b3b79f9cb85ebe0524336b2c08f4e6dc4d20e8 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 22 Apr 2022 19:39:13 -0400 Subject: [PATCH 03/15] fix psm3 test (apparent_zenith -> solar_zenith) --- pvlib/tests/iotools/test_psm3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py index 2c20155cdd..d151cfa6da 100644 --- a/pvlib/tests/iotools/test_psm3.py +++ b/pvlib/tests/iotools/test_psm3.py @@ -170,7 +170,7 @@ def test_read_psm3_map_variables(): data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) columns_mapped = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'dhi', 'dni', 'ghi', 'dhi_clear', 'dni_clear', 'ghi_clear', - 'Cloud Type', 'Dew Point', 'apparent_zenith', + 'Cloud Type', 'Dew Point', 'solar_zenith', 'Fill Flag', 'albedo', 'wind_speed', 'precipitable_water', 'wind_direction', 'relative_humidity', 'temp_air', 'pressure'] From 258a2aa59f79091af42926a1e733bac1159581c5 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 22 Apr 2022 19:49:09 -0400 Subject: [PATCH 04/15] whatsnew --- docs/sphinx/source/whatsnew/v0.9.2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.2.rst b/docs/sphinx/source/whatsnew/v0.9.2.rst index 650567f2ba..dccc6fc17a 100644 --- a/docs/sphinx/source/whatsnew/v0.9.2.rst +++ b/docs/sphinx/source/whatsnew/v0.9.2.rst @@ -27,7 +27,7 @@ Benchmarking Requirements ~~~~~~~~~~~~ -* Minimum pandas version increased to v0.25.0, released July 18, 2019. +* Minimum pandas version increased to v0.25.0, released July 18, 2019. (:pull:`1448`) Contributors ~~~~~~~~~~~~ From 1ae6deedd633613e482bb70aa303f9e23865c016 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 25 Apr 2022 14:09:28 -0400 Subject: [PATCH 05/15] better UTC conversion in sun_rise_set_transit_ephem --- pvlib/solarposition.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pvlib/solarposition.py b/pvlib/solarposition.py index 93465607bb..4bb551860f 100644 --- a/pvlib/solarposition.py +++ b/pvlib/solarposition.py @@ -22,6 +22,7 @@ import pandas as pd import scipy.optimize as so import warnings +import pytz from pvlib import atmosphere from pvlib.tools import datetime_to_djd, djd_to_datetime @@ -576,7 +577,7 @@ def sun_rise_set_transit_ephem(times, latitude, longitude, thetime = thetime.to_pydatetime() # pyephem drops timezone when converting to its internal datetime # format, so handle timezone explicitly here - obs.date = ephem.Date(thetime - thetime.utcoffset()) + obs.date = ephem.Date(thetime.astimezone(pytz.UTC)) sunrise.append(_ephem_to_timezone(rising(sun), tzinfo)) sunset.append(_ephem_to_timezone(setting(sun), tzinfo)) trans.append(_ephem_to_timezone(transit(sun), tzinfo)) From efc38739321dd48635e0b3e993981c84322b947b Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 25 Apr 2022 17:14:32 -0400 Subject: [PATCH 06/15] helpful comments --- pvlib/solarposition.py | 5 +++-- pvlib/tests/test_conftest.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pvlib/solarposition.py b/pvlib/solarposition.py index 4bb551860f..7ef602fe4f 100644 --- a/pvlib/solarposition.py +++ b/pvlib/solarposition.py @@ -575,8 +575,9 @@ def sun_rise_set_transit_ephem(times, latitude, longitude, trans = [] for thetime in times: thetime = thetime.to_pydatetime() - # pyephem drops timezone when converting to its internal datetime - # format, so handle timezone explicitly here + # older versions of pyephem ignore timezone when converting to its + # internal datetime format, so convert to UTC here to support + # all versions. GH #1449 obs.date = ephem.Date(thetime.astimezone(pytz.UTC)) sunrise.append(_ephem_to_timezone(rising(sun), tzinfo)) sunset.append(_ephem_to_timezone(setting(sun), tzinfo)) diff --git a/pvlib/tests/test_conftest.py b/pvlib/tests/test_conftest.py index 0892c52a47..e848ed19c1 100644 --- a/pvlib/tests/test_conftest.py +++ b/pvlib/tests/test_conftest.py @@ -57,6 +57,10 @@ def test__check_pandas_assert_kwargs(mocker, function_name, pd_version, # test that conftest._check_pandas_assert_kwargs returns appropriate # kwargs for the assert_x_equal functions + # NOTE: be careful about mixing mocker.patch and pytest.MonkeyPatch! + # they do not coordinate their cleanups, so it is safest to only + # use one or the other. GH #1447 + # patch the pandas assert; not interested in actually calling them, # plus we want to spy on how they get called. spy = mocker.patch('pandas.testing.' + function_name) From f1195a0903768df509ff56050b32fc1559b2499a Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 25 Apr 2022 17:14:38 -0400 Subject: [PATCH 07/15] more whatsnew --- docs/sphinx/source/whatsnew/v0.9.2.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.2.rst b/docs/sphinx/source/whatsnew/v0.9.2.rst index dccc6fc17a..d6bce8a5ac 100644 --- a/docs/sphinx/source/whatsnew/v0.9.2.rst +++ b/docs/sphinx/source/whatsnew/v0.9.2.rst @@ -14,7 +14,12 @@ Bug fixes * :py:func:`pvlib.irradiance.get_total_irradiance` and :py:func:`pvlib.solarposition.spa_python` now raise an error instead of silently ignoring unknown parameters (:pull:`1437`) - +* Fix a bug in :py:func:`pvlib.solarposition.sun_rise_set_transit_ephem` + where passing localized timezones with large UTC offsets could return + rise/set/transit times for the wrong day in recent versions of ``ephem`` + (:issue:`1449`, :pull:`1448`) + + Testing ~~~~~~~ From e02d9d3f1c15a81615557a736a4f104527a52eac Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 25 Apr 2022 17:19:56 -0400 Subject: [PATCH 08/15] '3.0' -> '3' in read_crn test? --- pvlib/tests/iotools/test_crn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_crn.py b/pvlib/tests/iotools/test_crn.py index b19888dda1..8d880e0432 100644 --- a/pvlib/tests/iotools/test_crn.py +++ b/pvlib/tests/iotools/test_crn.py @@ -83,7 +83,7 @@ def test_read_crn_problems(testfile_problems, columns_mapped, dtypes): '2020-07-06 13:10:00'], freq=None).tz_localize('UTC') values = np.array([ - [92821, 20200706, 1200, 20200706, 700, '3.0', -80.69, 28.62, 24.9, + [92821, 20200706, 1200, 20200706, 700, '3', -80.69, 28.62, 24.9, 0.0, np.nan, 0, 25.5, 'C', 0, 93.0, 0, nan, nan, 990, 0, 1.57, 0], [92821, 20200706, 1310, 20200706, 810, '2.623', -80.69, 28.62, 26.9, 0.0, 430.0, 0, 30.2, 'C', 0, 87.0, 0, nan, nan, 989, 0, From cb486a311a4c79706d9f9cb6c9a796d2a71afa16 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 25 Apr 2022 17:49:09 -0400 Subject: [PATCH 09/15] apply dtypes during parsing in read_crn --- pvlib/iotools/crn.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pvlib/iotools/crn.py b/pvlib/iotools/crn.py index fe46bb69d2..82b72c9af5 100644 --- a/pvlib/iotools/crn.py +++ b/pvlib/iotools/crn.py @@ -108,11 +108,9 @@ def read_crn(filename, map_variables=True): # read in data data = pd.read_fwf(filename, header=None, names=HEADERS, widths=WIDTHS, - na_values=NAN_DICT) + na_values=NAN_DICT, dtype=dict(zip(HEADERS, DTYPES))) # Remove rows with all nans data = data.dropna(axis=0, how='all') - # set dtypes here because dtype kwarg not supported in read_fwf until 0.20 - data = data.astype(dict(zip(HEADERS, DTYPES))) # set index # UTC_TIME does not have leading 0s, so must zfill(4) to comply From d97f067b91dd7a795bf4cb963eb0f5c09ab3f3a1 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 25 Apr 2022 17:55:55 -0400 Subject: [PATCH 10/15] move dropna() post-processing into read_fwf call --- pvlib/iotools/crn.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/crn.py b/pvlib/iotools/crn.py index 82b72c9af5..b82572eeb3 100644 --- a/pvlib/iotools/crn.py +++ b/pvlib/iotools/crn.py @@ -108,9 +108,8 @@ def read_crn(filename, map_variables=True): # read in data data = pd.read_fwf(filename, header=None, names=HEADERS, widths=WIDTHS, - na_values=NAN_DICT, dtype=dict(zip(HEADERS, DTYPES))) - # Remove rows with all nans - data = data.dropna(axis=0, how='all') + na_values=NAN_DICT, dtype=dict(zip(HEADERS, DTYPES)), + skip_blank_lines=True) # set index # UTC_TIME does not have leading 0s, so must zfill(4) to comply From 5d64cb8aa5f4ef1fa1fae6abc10ce356b17dcfc3 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 25 Apr 2022 21:06:38 -0400 Subject: [PATCH 11/15] fix read_crn for pandas<1.2.0 --- pvlib/iotools/crn.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/crn.py b/pvlib/iotools/crn.py index b82572eeb3..90e1d6d6b4 100644 --- a/pvlib/iotools/crn.py +++ b/pvlib/iotools/crn.py @@ -2,6 +2,7 @@ """ import pandas as pd +import numpy as np HEADERS = [ @@ -107,9 +108,23 @@ def read_crn(filename, map_variables=True): """ # read in data + # TODO: instead of parsing as strings and then post-processing, switch to + # pd.read_fwf(..., dtype=dict(zip(HEADERS, DTYPES)), skip_blank_lines=True) + # when our minimum pandas >= 1.2.0 (skip_blank_lines bug for <1.2.0). + # As a workaround, parse all values as strings, then drop NaN, then cast + # to the appropriate dtypes, and mask "sentinal" NaN (e.g. -9999.0) data = pd.read_fwf(filename, header=None, names=HEADERS, widths=WIDTHS, - na_values=NAN_DICT, dtype=dict(zip(HEADERS, DTYPES)), - skip_blank_lines=True) + dtype=str) + + # drop empty (bad) lines + data = data.dropna(axis=0, how='all') + + # can't set dtypes in read_fwf because int cols can't contain NaN, so + # do it here instead + data = data.astype(dict(zip(HEADERS, DTYPES))) + + # finally, replace -999 values with NaN + data = data.replace(NAN_DICT, value=np.nan) # set index # UTC_TIME does not have leading 0s, so must zfill(4) to comply From 12eb84ab02ea59d23d337a9169a6b2ccf8c54726 Mon Sep 17 00:00:00 2001 From: Kevin Anderson <57452607+kanderso-nrel@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:00:49 -0400 Subject: [PATCH 12/15] Update pvlib/solarposition.py Co-authored-by: Will Holmgren --- pvlib/solarposition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/solarposition.py b/pvlib/solarposition.py index 7ef602fe4f..6878b7d078 100644 --- a/pvlib/solarposition.py +++ b/pvlib/solarposition.py @@ -578,7 +578,7 @@ def sun_rise_set_transit_ephem(times, latitude, longitude, # older versions of pyephem ignore timezone when converting to its # internal datetime format, so convert to UTC here to support # all versions. GH #1449 - obs.date = ephem.Date(thetime.astimezone(pytz.UTC)) + obs.date = ephem.Date(thetime.astimezone(datetime.timezone.UTC)) sunrise.append(_ephem_to_timezone(rising(sun), tzinfo)) sunset.append(_ephem_to_timezone(setting(sun), tzinfo)) trans.append(_ephem_to_timezone(transit(sun), tzinfo)) From d000079327cf0f36ddd97e49c68b648a7295a797 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Thu, 28 Apr 2022 16:35:19 -0400 Subject: [PATCH 13/15] nix pytz --- pvlib/solarposition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/solarposition.py b/pvlib/solarposition.py index 6878b7d078..ab0576e368 100644 --- a/pvlib/solarposition.py +++ b/pvlib/solarposition.py @@ -22,7 +22,7 @@ import pandas as pd import scipy.optimize as so import warnings -import pytz +import datetime from pvlib import atmosphere from pvlib.tools import datetime_to_djd, djd_to_datetime From 2fbccdd7c633ad8e8b924d23e77554396d4d6681 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Thu, 28 Apr 2022 17:02:32 -0400 Subject: [PATCH 14/15] UTC -> utc --- pvlib/solarposition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/solarposition.py b/pvlib/solarposition.py index ab0576e368..cdcacd7ec6 100644 --- a/pvlib/solarposition.py +++ b/pvlib/solarposition.py @@ -578,7 +578,7 @@ def sun_rise_set_transit_ephem(times, latitude, longitude, # older versions of pyephem ignore timezone when converting to its # internal datetime format, so convert to UTC here to support # all versions. GH #1449 - obs.date = ephem.Date(thetime.astimezone(datetime.timezone.UTC)) + obs.date = ephem.Date(thetime.astimezone(datetime.timezone.utc)) sunrise.append(_ephem_to_timezone(rising(sun), tzinfo)) sunset.append(_ephem_to_timezone(setting(sun), tzinfo)) trans.append(_ephem_to_timezone(transit(sun), tzinfo)) From 6494ea02495582b7338578298768cf2c5d280de5 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Thu, 28 Apr 2022 19:09:37 -0400 Subject: [PATCH 15/15] address simd arccos issue in tracking.singleaxis --- pvlib/tracking.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pvlib/tracking.py b/pvlib/tracking.py index 732108dec2..951f2e886e 100644 --- a/pvlib/tracking.py +++ b/pvlib/tracking.py @@ -510,6 +510,9 @@ def singleaxis(apparent_zenith, apparent_azimuth, # Calculate surface_tilt dotproduct = (panel_norm_earth * projected_normal).sum(axis=1) + # for edge cases like axis_tilt=90, numpy's SIMD can produce values like + # dotproduct = (1 + 2e-16). Clip off the excess so that arccos works: + dotproduct = np.clip(dotproduct, -1, 1) surface_tilt = 90 - np.degrees(np.arccos(dotproduct)) # Bundle DataFrame for return values and filter for sun below horizon.