From 3419c6d5f8285f46ecf200a88a8b83d92fa257c4 Mon Sep 17 00:00:00 2001 From: Caitlin O'Connor Date: Sun, 14 Jul 2024 12:40:27 -0700 Subject: [PATCH 01/16] fix to allow wind extrapolation for weathering --- py_gnome/gnome/environment/wind.py | 11 ++++++----- .../tests/unit_tests/test_environment/test_wind.py | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/py_gnome/gnome/environment/wind.py b/py_gnome/gnome/environment/wind.py index 70a5aa147..15c546f2f 100644 --- a/py_gnome/gnome/environment/wind.py +++ b/py_gnome/gnome/environment/wind.py @@ -509,7 +509,7 @@ def get_value(self, time): return tuple(data[0]['value']) - def at(self, points, time, *, units=None, extrapolate=False, coord_sys='uv',_auto_align=True, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, coord_sys='uv',_auto_align=True, **kwargs): # fixme: this isn't quite aligned with the Environment base class signature. # it should be: # def at(self, points, time, units=None, extrapolate=False, coord_sys='uv', _auto_align=True): @@ -553,10 +553,11 @@ def at(self, points, time, *, units=None, extrapolate=False, coord_sys='uv',_aut else: raise ValueError('invalid coordinate system {0}'.format(coord_sys)) - if extrapolate: - self.extrapolation_is_allowed = True - else: - self.extrapolation_is_allowed = False + if extrapolate is not None: # passed in a value, assume this is what we want + self.extrapolation_is_allowed = extrapolate + else: # otherwise probably set on the wind_mover + if self.extrapolation_is_allowed is None: + self.extrapolation_is_allowed = False #print(f"{self.extrapolation_is_allowed=}") try: data = self.get_wind_data(time, 'm/s', cs)[0]['value'] diff --git a/py_gnome/tests/unit_tests/test_environment/test_wind.py b/py_gnome/tests/unit_tests/test_environment/test_wind.py index f01921c94..acefdf10a 100755 --- a/py_gnome/tests/unit_tests/test_environment/test_wind.py +++ b/py_gnome/tests/unit_tests/test_environment/test_wind.py @@ -238,10 +238,6 @@ def test_at_out_of_bounds(wind_circ): result = wind.at([(0, 0, 0)], wind.data_stop) assert np.array_equal(result, [[1., 0., 0.]]) - # but should work if extrapolate=True - result = wind.at([(0, 0, 0)], wind.data_stop + gs.hours(1), extrapolate=True) - assert np.array_equal(result, [[1., 0., 0.]]) - # after the data_stop with pytest.raises(ValueError) as excinfo: result = wind.at([(0, 0, 0)], wind.data_stop + gs.hours(1)) @@ -252,6 +248,10 @@ def test_at_out_of_bounds(wind_circ): result = wind.at([(0, 0, 0)], wind.data_start - gs.hours(1)) assert "not within the bounds" in str(excinfo.value) + # but should work if extrapolate=True + result = wind.at([(0, 0, 0)], wind.data_stop + gs.hours(1), extrapolate=True) + assert np.array_equal(result, [[1., 0., 0.]]) + @pytest.fixture(scope='module') def wind_rand(rq_rand): From d3c0ab9a16254933a02a10a9cbe902795524499e Mon Sep 17 00:00:00 2001 From: Caitlin O'Connor Date: Tue, 16 Jul 2024 14:00:54 -0700 Subject: [PATCH 02/16] fix to extrapolate for ossm wind --- py_gnome/gnome/environment/wind.py | 7 ++++++- py_gnome/tests/unit_tests/test_environment/test_wind.py | 8 ++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/py_gnome/gnome/environment/wind.py b/py_gnome/gnome/environment/wind.py index 15c546f2f..5a19bbc00 100644 --- a/py_gnome/gnome/environment/wind.py +++ b/py_gnome/gnome/environment/wind.py @@ -553,10 +553,13 @@ def at(self, points, time, *, units=None, extrapolate=None, coord_sys='uv',_auto else: raise ValueError('invalid coordinate system {0}'.format(coord_sys)) + # since extrapolation_is_allowed is not passed into the C code we need + # to temporarily set it if extrapolate is passed in, and then change it back + original_extrapolation = self.extrapolation_is_allowed # save value if extrapolate is not None: # passed in a value, assume this is what we want self.extrapolation_is_allowed = extrapolate else: # otherwise probably set on the wind_mover - if self.extrapolation_is_allowed is None: + if self.extrapolation_is_allowed is None: # shouldn't happen self.extrapolation_is_allowed = False #print(f"{self.extrapolation_is_allowed=}") try: @@ -565,6 +568,8 @@ def at(self, points, time, *, units=None, extrapolate=None, coord_sys='uv',_auto # CyTimeseries is raising an IndexError raise ValueError(f'time specified ({time}) is not within the bounds of the time: ' f'({self.data_start} to {self.data_stop})') + self.extrapolation_is_allowed = original_extrapolation # put it back the way it was + if idx is None: ret_data[:, 0] = data[0] ret_data[:, 1] = data[1] diff --git a/py_gnome/tests/unit_tests/test_environment/test_wind.py b/py_gnome/tests/unit_tests/test_environment/test_wind.py index acefdf10a..f01921c94 100755 --- a/py_gnome/tests/unit_tests/test_environment/test_wind.py +++ b/py_gnome/tests/unit_tests/test_environment/test_wind.py @@ -238,6 +238,10 @@ def test_at_out_of_bounds(wind_circ): result = wind.at([(0, 0, 0)], wind.data_stop) assert np.array_equal(result, [[1., 0., 0.]]) + # but should work if extrapolate=True + result = wind.at([(0, 0, 0)], wind.data_stop + gs.hours(1), extrapolate=True) + assert np.array_equal(result, [[1., 0., 0.]]) + # after the data_stop with pytest.raises(ValueError) as excinfo: result = wind.at([(0, 0, 0)], wind.data_stop + gs.hours(1)) @@ -248,10 +252,6 @@ def test_at_out_of_bounds(wind_circ): result = wind.at([(0, 0, 0)], wind.data_start - gs.hours(1)) assert "not within the bounds" in str(excinfo.value) - # but should work if extrapolate=True - result = wind.at([(0, 0, 0)], wind.data_stop + gs.hours(1), extrapolate=True) - assert np.array_equal(result, [[1., 0., 0.]]) - @pytest.fixture(scope='module') def wind_rand(rq_rand): From 5dd2f17e2d9bf5c10c0ce1b9a3d64657b672a42d Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Wed, 17 Jul 2024 11:59:04 -0700 Subject: [PATCH 03/16] added a finally to assume extrapolation flag is reset. --- py_gnome/gnome/environment/wind.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py_gnome/gnome/environment/wind.py b/py_gnome/gnome/environment/wind.py index 5a19bbc00..b71b2fa2f 100644 --- a/py_gnome/gnome/environment/wind.py +++ b/py_gnome/gnome/environment/wind.py @@ -568,7 +568,8 @@ def at(self, points, time, *, units=None, extrapolate=None, coord_sys='uv',_auto # CyTimeseries is raising an IndexError raise ValueError(f'time specified ({time}) is not within the bounds of the time: ' f'({self.data_start} to {self.data_stop})') - self.extrapolation_is_allowed = original_extrapolation # put it back the way it was + finally: # make sure it gets restored even if there's an error + self.extrapolation_is_allowed = original_extrapolation # put it back the way it was if idx is None: ret_data[:, 0] = data[0] From ab6ef6f573da0469608b35d6270e0c69ccfc32a8 Mon Sep 17 00:00:00 2001 From: Caitlin O'Connor Date: Wed, 17 Jul 2024 13:17:02 -0700 Subject: [PATCH 04/16] changed extrapolate default to None in at functions --- py_gnome/gnome/environment/environment.py | 2 +- .../gnome/environment/environment_objects.py | 28 ++++++++++++------- .../gnome/environment/gridded_objects_base.py | 21 ++++++++------ 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/py_gnome/gnome/environment/environment.py b/py_gnome/gnome/environment/environment.py index 31c7c973f..26d6c3009 100644 --- a/py_gnome/gnome/environment/environment.py +++ b/py_gnome/gnome/environment/environment.py @@ -56,7 +56,7 @@ def __init__(self, make_default_refs=True, **kwargs): self.array_types = {} super().__init__(**kwargs) - def at(self, points, time, *, units=None, extrapolate=False, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, **kwargs): """ Find the value of the property at positions P at time T diff --git a/py_gnome/gnome/environment/environment_objects.py b/py_gnome/gnome/environment/environment_objects.py index d7464bbd0..70a05ac09 100644 --- a/py_gnome/gnome/environment/environment_objects.py +++ b/py_gnome/gnome/environment/environment_objects.py @@ -119,7 +119,7 @@ def v(self): return self._v - def at(self, points, time, *, units=None, extrapolate=False, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, **kwargs): """ Find the value of the property at positions of points at time T @@ -486,7 +486,7 @@ class GridCurrent(VelocityGrid, Environment): # 'surface_northward_sea_water_velocity'], # 'w': ['upward_sea_water_velocity']} - def at(self, points, time, *, units=None, extrapolate=False, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, **kwargs): ''' Find the value of the property at positions P at time T @@ -516,7 +516,9 @@ def at(self, points, time, *, units=None, extrapolate=False, **kwargs): if res is not None: return res - extrapolate = self.extrapolation_is_allowed or extrapolate + if extrapolate is None: + extrapolate = self.extrapolation_is_allowed + value = super(GridCurrent, self).at(points, time, units=units, @@ -580,7 +582,7 @@ def __init__(self, self.wet_dry_mask = wet_dry_mask - def at(self, points, time, *, units=None, extrapolate=False, coord_sys='uv', _auto_align=True, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, coord_sys='uv', _auto_align=True, **kwargs): ''' Find the value of the property at positions P at time T @@ -628,7 +630,8 @@ def at(self, points, time, *, units=None, extrapolate=False, coord_sys='uv', _au return self.transform_result(value, coord_sys) if value is None: - extrapolate = self.extrapolation_is_allowed or extrapolate + if extrapolate is None: + extrapolate = self.extrapolation_is_allowed value = super(GridWind, self).at(pts, time, units=units, extrapolate=extrapolate, @@ -711,7 +714,7 @@ def __init__(self, *args, **kwargs): kwargs['data'] = data - def at(self, points, time, *, units=None, extrapolate=False, + def at(self, points, time, *, units=None, extrapolate=None, _hash=None, _mem=True, **kwargs): if _hash is None: @@ -728,6 +731,9 @@ def at(self, points, time, *, units=None, extrapolate=False, _time_idx = self.time.index_of(time) order = self.dimension_ordering + if extrapolate is None: + extrapolate = self.extrapolation_is_allowed + if order[0] == 'time': value = self._time_interp(points, time, units=units, extrapolate=extrapolate, _mem=_mem, _hash=_hash, **kwargs) @@ -840,8 +846,9 @@ def init_from_netCDF(self, **kwargs ) - def at(self, points, time, *, units=None, extrapolate=False, **kwargs): - extrapolate = self.extrapolation_is_allowed or extrapolate + def at(self, points, time, *, units=None, extrapolate=None, **kwargs): + if extrapolate is None: + extrapolate = self.extrapolation_is_allowed cctn = (self.ice_concentration.at(points, time, extrapolate=extrapolate, **kwargs) .copy()) @@ -907,13 +914,14 @@ def from_netCDF(cls, ice_velocity=ice_velocity, **kwargs)) - def at(self, points, time, *, units=None, extrapolate=False, min_val=0, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, min_val=0, **kwargs): """ :param min_val: Minimum value for wind speed. If computed wind speed is less than this value, it will be set to this value. :type min_val: float """ - extrapolate = self.extrapolation_is_allowed or extrapolate + if extrapolate is None: + extrapolate = self.extrapolation_is_allowed cctn = self.ice_concentration.at(points, time, extrapolate=extrapolate, **kwargs) wind_v = super(IceAwareWind, self).at(points, time, diff --git a/py_gnome/gnome/environment/gridded_objects_base.py b/py_gnome/gnome/environment/gridded_objects_base.py index 8cc8e1f8d..f461b010f 100644 --- a/py_gnome/gnome/environment/gridded_objects_base.py +++ b/py_gnome/gnome/environment/gridded_objects_base.py @@ -589,7 +589,10 @@ def from_netCDF(cls, *args, **kwargs): return var @combine_signatures - def at(self, points, time, *, units=None, extrapolate=False, unmask=True, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, unmask=True, **kwargs): + + if extrapolate is None: + extrapolate = self.extrapolation_is_allowed value = super(Variable, self).at(points, time, units=units, @@ -744,7 +747,7 @@ def __init__(self, **kwargs): super(VectorVariable, self).__init__(*args, **kwargs) self.extrapolation_is_allowed = extrapolation_is_allowed - + #Adding this so unit conversion happens properly in the components #I really don't think we will be doing mixed units in one of these #anytime soon. @@ -755,8 +758,8 @@ def __init__(self, warnings.warn("Variable {0} has units {1} which are not the same as the VectorVariable {2} units {3}.".format(var.name, var._gnome_unit, self.name, self._gnome_unit)) else: var._gnome_unit = self._gnome_unit - - + + def __repr__(self): try: @@ -939,7 +942,7 @@ def constant(cls, :param values: vector of values :type values: array-like ''' - + Grid = Grid_S Time = cls._default_component_types['time'] _node_lon = np.array(([-360, 0, 360], [-360, 0, 360], [-360, 0, 360])) @@ -952,9 +955,11 @@ def constant(cls, _vars = [Variable(grid=_grid, units=units[i], time=_time, data=d) for i, d in enumerate(_datas)] return cls(name=name, grid=_grid, time=_time, variables=_vars) - def at(self, points, time, *, units=None, extrapolate=False, unmask=True, **kwargs): + def at(self, points, time, *, units=None, extrapolate=None, unmask=True, **kwargs): units = units if units else self._gnome_unit #no need to convert here, its handled in the subcomponents - value = super(VectorVariable, self).at(points, time, + if extrapolate is None: + extrapolate = self.extrapolation_is_allowed + value = super(VectorVariable, self).at(points, time, units=units, extrapolate=extrapolate, unmask=unmask, @@ -1013,7 +1018,7 @@ def get_data_vectors(self): xt = x.shape[0] y = raw_v[:] yt = y.shape[0] - if raw_u.shape[-2:] == raw_v.shape[-2:] and raw_u.shape[-2:] == self.grid.center_mask.shape: + if raw_u.shape[-2:] == raw_v.shape[-2:] and raw_u.shape[-2:] == self.grid.center_mask.shape: #raw u/v are same shape and on center #need to padding_slice the variable since they are not interpolated from u/v x = x[(np.s_[:],) + ctr_padding_slice] From bf0b6b7829a69674b6d11406baae54c776afdd5e Mon Sep 17 00:00:00 2001 From: "James.Makela" Date: Wed, 17 Jul 2024 14:09:44 -0700 Subject: [PATCH 05/16] CI: Switch over to an ubuntu docker image for the compile task. --- .gitlab-ci.yml | 16 ++++++++++++---- dockerfile | 7 +++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3afa1b0d9..328bdbd48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: PYTHON_VER: "3.10" - IMAGE_NAME: "miniforge-python" + IMAGE_NAME: "ubuntu/ubuntuforge-python" GIT_DEPTH: 2 stages: @@ -38,10 +38,17 @@ compile_pygnome: - docker - linux image: registry.orr.noaa.gov/erd/centos-conda/$IMAGE_NAME$PYTHON_VER - script: - - echo "compiling PyGNOME" - - yum install gcc gcc-c++ make libXext libSM libXrender -y + before_script: + - apt update + - apt upgrade -y + + - echo 'tzdata tzdata/Areas select America' | debconf-set-selections + - echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections + - DEBIAN_FRONTEND="noninteractive" apt install -y tzdata + - apt install gcc gcc-c++ make libXext libSM libXrender -y + + script: # get the adios_db code: needed for tests and requirements - git clone --depth 1 -b develop https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.orr.noaa.gov/gnome/oil_database/oil_database.git - cd ./py_gnome @@ -60,6 +67,7 @@ compile_pygnome: # And there may be times we want to build a wheel file (.whl) # The following command will build a wheel in the current directory: # python -m pip wheel ./ + - echo "compiling PyGNOME" - python -m pip install ./ # need extra requirements for the docs diff --git a/dockerfile b/dockerfile index 18d4ecac3..66d2ba500 100644 --- a/dockerfile +++ b/dockerfile @@ -1,10 +1,13 @@ ARG PYTHON_VER -FROM registry.orr.noaa.gov/erd/centos-conda/miniforge-python$PYTHON_VER +FROM registry.orr.noaa.gov/erd/centos-conda/ubuntu/ubuntuforge-python$PYTHON_VER # Args declared before the FROM need to be redeclared, don't delete this ARG PYTHON_VER -RUN yum install -y libglib2.0-0 libxext6 libsm6 libxrender1 \ +RUN apt update +RUN apt upgrade -y + +RUN apt install -y libglib2.0-0 libxext6 libsm6 libxrender1 \ wget gcc make bzip2 gcc-c++ chrpath patchelf \ ca-certificates git mercurial subversion tar From 94a93520599609d6fc1a5c87a0129fc6ec999e97 Mon Sep 17 00:00:00 2001 From: "James.Makela" Date: Wed, 17 Jul 2024 14:30:38 -0700 Subject: [PATCH 06/16] CI: apt package names are a bit different in ubuntu. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 328bdbd48..6a0d93679 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,7 +46,7 @@ compile_pygnome: - echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections - DEBIAN_FRONTEND="noninteractive" apt install -y tzdata - - apt install gcc gcc-c++ make libXext libSM libXrender -y + - apt install g++ make libxext-dev libsm-dev libxrender-dev -y script: # get the adios_db code: needed for tests and requirements From 8b1c34a7911434b3a8aeec4aa88bf3dee4433eb0 Mon Sep 17 00:00:00 2001 From: "James.Makela" Date: Wed, 17 Jul 2024 14:38:59 -0700 Subject: [PATCH 07/16] CI: ubuntu image is missing git. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a0d93679..d09f0c136 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,7 +46,7 @@ compile_pygnome: - echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections - DEBIAN_FRONTEND="noninteractive" apt install -y tzdata - - apt install g++ make libxext-dev libsm-dev libxrender-dev -y + - apt install git g++ make libxext-dev libsm-dev libxrender-dev -y script: # get the adios_db code: needed for tests and requirements From 21936a52e66b1c9e2cba6b07ea0cc75f600d921b Mon Sep 17 00:00:00 2001 From: "James.Makela" Date: Wed, 17 Jul 2024 15:06:33 -0700 Subject: [PATCH 08/16] CI: I think the before_script section is messing up the test_pygnome task. --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d09f0c136..66f90c7ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ compile_pygnome: - docker - linux image: registry.orr.noaa.gov/erd/centos-conda/$IMAGE_NAME$PYTHON_VER - before_script: + script: - apt update - apt upgrade -y @@ -48,7 +48,6 @@ compile_pygnome: - apt install git g++ make libxext-dev libsm-dev libxrender-dev -y - script: # get the adios_db code: needed for tests and requirements - git clone --depth 1 -b develop https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.orr.noaa.gov/gnome/oil_database/oil_database.git - cd ./py_gnome @@ -105,7 +104,7 @@ test_pygnome_scripts: - docker - linux allow_failure: true - image: registry.orr.noaa.gov/erd/centos-conda/centos7-python$PYTHON_VER + image: registry.orr.noaa.gov/erd/centos-conda/$IMAGE_NAME$PYTHON_VER script: - cd ./py_gnome/scripts/testing_scripts && python run_all.py 2<&1 From 095ed2890bd7e6f7d08e0ccf02a7c7c036deec26 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Wed, 17 Jul 2024 16:32:43 -0700 Subject: [PATCH 09/16] pinned numpy to <2 --- py_gnome/conda_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_gnome/conda_requirements.txt b/py_gnome/conda_requirements.txt index 246359c56..487f99afd 100644 --- a/py_gnome/conda_requirements.txt +++ b/py_gnome/conda_requirements.txt @@ -38,7 +38,7 @@ pyzmq # tblib ## Base numerical libs -numpy +numpy <2 scipy ## Modeling libs From 9ccf219f7c0cf835571922934f423e56edcff219 Mon Sep 17 00:00:00 2001 From: "James.Makela" Date: Thu, 18 Jul 2024 13:24:34 -0700 Subject: [PATCH 10/16] CI: Use apt-get instead of apt because cli instability...reasons. apt-get the proper c++ stuff in the dockerfile. --- .gitlab-ci.yml | 12 ++++++------ dockerfile | 15 ++++++++++----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66f90c7ac..90c809b8c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,14 +39,14 @@ compile_pygnome: - linux image: registry.orr.noaa.gov/erd/centos-conda/$IMAGE_NAME$PYTHON_VER script: - - apt update - - apt upgrade -y + - apt-get update + - apt-get upgrade -y - - echo 'tzdata tzdata/Areas select America' | debconf-set-selections - - echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections - - DEBIAN_FRONTEND="noninteractive" apt install -y tzdata + #- echo 'tzdata tzdata/Areas select America' | debconf-set-selections + #- echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections + #- DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata - - apt install git g++ make libxext-dev libsm-dev libxrender-dev -y + - apt-get install git g++ make libxext-dev libsm-dev libxrender-dev -y # get the adios_db code: needed for tests and requirements - git clone --depth 1 -b develop https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.orr.noaa.gov/gnome/oil_database/oil_database.git diff --git a/dockerfile b/dockerfile index 66d2ba500..19810c71d 100644 --- a/dockerfile +++ b/dockerfile @@ -4,12 +4,17 @@ FROM registry.orr.noaa.gov/erd/centos-conda/ubuntu/ubuntuforge-python$PYTHON_VER # Args declared before the FROM need to be redeclared, don't delete this ARG PYTHON_VER -RUN apt update -RUN apt upgrade -y +RUN apt-get update +RUN apt-get upgrade -y -RUN apt install -y libglib2.0-0 libxext6 libsm6 libxrender1 \ - wget gcc make bzip2 gcc-c++ chrpath patchelf \ - ca-certificates git mercurial subversion tar +#RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections +#RUN echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections +#RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata + +RUN apt-get install -y \ + g++ make libxext-dev libsm-dev libxrender-dev libglib2.0-0 \ + wget chrpath bzip2 tar \ + git patchelf ca-certificates COPY ./ /pygnome/ WORKDIR /pygnome/ From 8375a1e5127d242a83a039bded89664d2f0f4467 Mon Sep 17 00:00:00 2001 From: "James.Makela" Date: Mon, 22 Jul 2024 14:28:07 -0700 Subject: [PATCH 11/16] CI: Cleanup some things in the pipeline code. --- .gitlab-ci.yml | 4 ---- dockerfile | 10 +++------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90c809b8c..f46d3d8bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,10 +42,6 @@ compile_pygnome: - apt-get update - apt-get upgrade -y - #- echo 'tzdata tzdata/Areas select America' | debconf-set-selections - #- echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections - #- DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata - - apt-get install git g++ make libxext-dev libsm-dev libxrender-dev -y # get the adios_db code: needed for tests and requirements diff --git a/dockerfile b/dockerfile index 19810c71d..31e01efc7 100644 --- a/dockerfile +++ b/dockerfile @@ -7,10 +7,6 @@ ARG PYTHON_VER RUN apt-get update RUN apt-get upgrade -y -#RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections -#RUN echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections -#RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata - RUN apt-get install -y \ g++ make libxext-dev libsm-dev libxrender-dev libglib2.0-0 \ wget chrpath bzip2 tar \ @@ -20,9 +16,9 @@ COPY ./ /pygnome/ WORKDIR /pygnome/ RUN conda install python=$PYTHON_VER \ - --file py_gnome/conda_requirements.txt \ - --file py_gnome/conda_requirements_build.txt \ - --file oil_database/adios_db/conda_requirements.txt + --file py_gnome/conda_requirements.txt \ + --file py_gnome/conda_requirements_build.txt \ + --file oil_database/adios_db/conda_requirements.txt RUN cd oil_database/adios_db && python -m pip install ./ From 595241b60f756d88505dae3be3c518a2ceec06af Mon Sep 17 00:00:00 2001 From: "aaron.racicot" Date: Wed, 24 Jul 2024 22:24:40 +0000 Subject: [PATCH 12/16] Update to support defaulting back to basic ERMA styling while still supporting... --- .../gnome/outputters/erma_data_package.py | 440 ++++++++++-------- py_gnome/tests/pytest.ini | 1 + 2 files changed, 241 insertions(+), 200 deletions(-) diff --git a/py_gnome/gnome/outputters/erma_data_package.py b/py_gnome/gnome/outputters/erma_data_package.py index f61a7b050..0c4589d07 100644 --- a/py_gnome/gnome/outputters/erma_data_package.py +++ b/py_gnome/gnome/outputters/erma_data_package.py @@ -31,7 +31,12 @@ class ERMADataPackageSchema(BaseOutputterSchema): missing=drop, save=True, update=True, test_equal=False ) # ERMA folder - folder_name = SchemaNode(String(), save=True, update=True) + base_folder_name = SchemaNode(String(), save=True, update=True) + model_folder_name = SchemaNode(String(), save=True, update=True) + # Default styling + default_erma_styling = SchemaNode( + Boolean(), missing=drop, save=True, update=True + ) # Certain layer certain_layer_name = SchemaNode(String(), save=True, update=True) include_certain_particles = SchemaNode( @@ -118,7 +123,8 @@ class ERMADataPackageOutput(Outputter): time_formatter = '%m/%d/%Y %H:%M' - def __init__(self, filename, folder_name=None, + def __init__(self, filename, base_folder_name='', model_folder_name='', + default_erma_styling=True, # Certain layer certain_layer_name=None, include_certain_particles=True, # Uncertain layer @@ -168,12 +174,19 @@ def __init__(self, filename, folder_name=None, self.filedir = str(pathlib_path.parent) # A temp dir used to do our work... self.tempdir = tempfile.TemporaryDirectory(prefix='gnome.') + # We dont know the model_start_time right off the bat, but we will fill in + # when the model run starts. + self.model_start_time = None # Default some vars if they are null - if not folder_name: - self.folder_name = "Testing Layers > Trajectories > " + self.filenamestem + '-' + self.id + if not base_folder_name: + self.base_folder_name = "Testing Layers > Trajectories > " + self.filenamestem + '-' + self.id else: - self.folder_name = folder_name + self.base_folder_name = base_folder_name + self.model_folder_name = model_folder_name + self.folder_name = self.base_folder_name + ' > ' + self.model_folder_name + # Default styling + self.default_erma_styling = default_erma_styling # Certain layer self.certain_layer_name = certain_layer_name self.include_certain_particles = include_certain_particles @@ -270,6 +283,12 @@ def prepare_for_model_run(self, spills, **kwargs) self.model_start_time = model_start_time + # By default we want to name the folder for the output based on the model + # start time. Since we are just finding that out now (if it was not + # explicitily set by the user), we need to set the folder_name here as well + if not self.model_folder_name: + self.model_folder_name = self.model_start_time.strftime(self.time_formatter) + self.folder_name = self.base_folder_name + ' > ' + self.model_folder_name self.spills = spills # This generates a cutoff struct for contours based # on our spills @@ -386,14 +405,14 @@ def build_package(self): layer_size = self.certain_boundary_size if self.certain_boundary_size else 3 layer_json.append(self.make_boundary_polygon_package_layer(next(id), False, self.shapefile_builder_certain_boundary.filename, - layer_name, 'Certain Particles Boundary', + layer_name, 'Best Estimate Boundary', layer_color, layer_size)) if self.include_certain_contours: layer_name = self.certain_contours_layer_name if self.certain_contours_layer_name else 'Certain Particles Contours' layer_size = self.certain_contours_size if self.certain_contours_size else 3 layer_json.append(self.make_contour_polygon_package_layer(next(id), self.shapefile_builder_certain_contours.filename, - layer_name, 'Certain Particles Contour', + layer_name, 'Best Estimate Contours', layer_size)) if self.include_uncertain_boundary and self.uncertain: layer_name = self.uncertain_boundary_layer_name if self.uncertain_boundary_layer_name else 'Uncertain Particles Boundary' @@ -401,7 +420,7 @@ def build_package(self): layer_size = self.uncertain_boundary_size if self.uncertain_boundary_size else 3 layer_json.append(self.make_boundary_polygon_package_layer(next(id), True, self.shapefile_builder_uncertain_boundary.filename, - layer_name, 'Uncertain Particles Boundary', + layer_name, 'Uncertainty Boundary', layer_color, layer_size)) # Now we can zip it all up zipf = zipfile.ZipFile(self.filename, 'w') @@ -575,24 +594,17 @@ def make_boundary_polygon_package_layer(self, id, uncertain, shapefile_filename, layer_template['mapfile_layer']['shapefile']['file'] = "file://source_files/" + basefile layer_template['mapfile_layer']['layer_name'] = generic_name layer_template['mapfile_layer']['layer_desc'] = generic_description - layer_template['mapfile_layer']['classitem'] = 'area' + # Get rid of a few things we dont want + layer_template['mapfile_layer']['classitem'] = None # Modify the style object polygon_template_cartoline = copy.deepcopy(polygon_template) - polygon_template_solid = copy.deepcopy(polygon_template) - polygon_template_cartoline['name'] = style_name+'_cartoline' - polygon_template_cartoline['expression'] = '[area] > 10' - polygon_template_cartoline['expression_type'] = 'M' + polygon_template_cartoline['name'] = style_name + polygon_template_cartoline['expression'] = None + polygon_template_cartoline['expression_type'] = None polygon_template_cartoline['styles'][0]['outlinecolor'] = color polygon_template_cartoline['styles'][0]['outlinesymbol'] = 'dashedcartoline' polygon_template_cartoline['styles'][0]['style_width'] = style_width - polygon_template_solid['name'] = style_name+'_solid' - polygon_template_solid['expression'] = '[area] <= 10' - polygon_template_solid['expression_type'] = 'M' - polygon_template_solid['styles'][0]['outlinesymbol'] = None - polygon_template_solid['styles'][0]['outlinecolor'] = color - polygon_template_solid['styles'][0]['style_width'] = style_width * 3 layer_template['mapfile_layer']['layer_classes'].append(polygon_template_cartoline) - layer_template['mapfile_layer']['layer_classes'].append(polygon_template_solid) else: raise ValueError("Can not write ERMA Data Package without template!!!") @@ -861,198 +873,226 @@ def make_particle_package_layer(self, id, layer_name, uncertain, shapefile_filen # More than a year, use year layer_template['time_step_override'] = int(timestepseconds / year) layer_template['time_units_override'] = 'year' - classorder = itertools.count() - spills_grouped_by_style = [] - if self.disable_legend_collapse: - # In the case we dont want to collapse, we just group - # each spill on its own - for s_id, s in enumerate(self.spills): - spills_grouped_by_style.append([s_id]) - else: - # We can try to group spills with like styling... - spills_grouped_by_style = self.group_like_spills() - # Now we loop through the spill groups and style them - for spill_group in spills_grouped_by_style: - # Since we are styling by group, we grab the first spill - # from this group - spill_id = spill_group[0] - spill = self.spills[spill_id] - # Build a string indicating the ids in this group for - # the mapserver expression - spill_group_string = (',').join([str(s) for s in spill_group]) - # For the legend, we group the names - name_string_list = [] - for s_id in spill_group: - spill_name = (f'{self.spills[s_id].name} ' - f'({self.spills[s_id].amount}' - f'{self.spills[s_id].units})') - name_string_list.append(spill_name) - # In ERMA the pipe puts them on separate lines in the legend - spill_names = ('|').join(name_string_list) - # If we have an appearance, we use that - if spill._appearance and spill._appearance.colormap: - appearance = spill._appearance.to_dict() - colormap = spill._appearance.colormap.to_dict() - requested_display_param = appearance['data'] - requested_display_unit = appearance['units'] - unit_map = {} - if requested_display_param in self.default_unit_map: - unit_map = self.default_unit_map[requested_display_param] + if not self.default_erma_styling: + classorder = itertools.count() + spills_grouped_by_style = [] + if self.disable_legend_collapse: + # In the case we dont want to collapse, we just group + # each spill on its own + for s_id, s in enumerate(self.spills): + spills_grouped_by_style.append([s_id]) + else: + # We can try to group spills with like styling... + spills_grouped_by_style = self.group_like_spills() + # Now we loop through the spill groups and style them + for spill_group in spills_grouped_by_style: + # Since we are styling by group, we grab the first spill + # from this group + spill_id = spill_group[0] + spill = self.spills[spill_id] + # Build a string indicating the ids in this group for + # the mapserver expression + spill_group_string = (',').join([str(s) for s in spill_group]) + # For the legend, we group the names + name_string_list = [] + for s_id in spill_group: + spill_name = (f'{self.spills[s_id].name} ' + f'({self.spills[s_id].amount}' + f'{self.spills[s_id].units})') + name_string_list.append(spill_name) + # In ERMA the pipe puts them on separate lines in the legend + spill_names = ('|').join(name_string_list) + # If we have an appearance, we use that + if spill._appearance and spill._appearance.colormap and not self.default_erma_styling: + appearance = spill._appearance.to_dict() + colormap = spill._appearance.colormap.to_dict() + requested_display_param = appearance['data'] + requested_display_unit = appearance['units'] + unit_map = {} + if requested_display_param in self.default_unit_map: + unit_map = self.default_unit_map[requested_display_param] + else: + raise ValueError(f'Style requested is not supported!!! {requested_display_param}') + # Looks like the domains are always in the base units... + number_scale_domain = [val for val in colormap['numberScaleDomain']] + color_scale_domain = [val for val in colormap['colorScaleDomain']] + #number_scale_domain = [uc.convert(requested_display_unit, + # unit_map['unit'], val) for val in colormap['numberScaleDomain']] + #color_scale_domain = [uc.convert(requested_display_unit, + # unit_map['unit'], val) for val in colormap['colorScaleDomain']] + min = float(number_scale_domain[0]) + max = None + data_column = unit_map['column'] + style_size = 8 * appearance['scale'] + if uncertain: + uncertain_color = '#FF0000' + # In the uncertain case, we just color everything red + floating_class_template = copy.deepcopy(default_floating_template) + beached_class_template = copy.deepcopy(default_beached_template) + floating_class_template['expression'] = ('[statuscode] = 2 AND ' + f'[spill_id] IN "{spill_group_string}"') + floating_class_template['styles'][0]['color'] = uncertain_color + floating_class_template['styles'][0]['style_size'] = style_size + floating_class_template['name'] = f'{spill_names}|Floating Uncertain' + beached_class_template['expression'] = ('[statuscode] = 3 AND ' + f'[spill_id] IN "{spill_group_string}"') + beached_class_template['styles'][0]['color'] = uncertain_color + beached_class_template['styles'][0]['outlinecolor'] = uncertain_color + beached_class_template['styles'][0]['style_size'] = style_size + beached_class_template['name'] = f'{spill_names}|Beached Uncertain' + layer_template['mapfile_layer']['layer_classes'].append(floating_class_template) + layer_template['mapfile_layer']['layer_classes'].append(beached_class_template) + else: + # Loop through the color scale and make classes + for idx, color in enumerate(colormap['colorScaleRange']): + if (idx+1) > len(color_scale_domain): + max = float(number_scale_domain[1]) + else: + max = float(color_scale_domain[idx]) + # Convert our min/max with unit conversion for labeling + if requested_display_param == 'Mass': + converted_min = convert_mass_to_mass_or_volume(unit_map['unit'], + requested_display_unit, + spill.substance.standard_density, + min) + converted_max = convert_mass_to_mass_or_volume(unit_map['unit'], + requested_display_unit, + spill.substance.standard_density, + max) + elif requested_display_param == 'Surface Concentration': + converted_min = uc.convert('oil concentration', + unit_map['unit'], + requested_display_unit, + min) + converted_max = uc.convert('oil concentration', + unit_map['unit'], + requested_display_unit, + max) + elif requested_display_param == 'Viscosity': + converted_min = uc.convert('kinematic viscosity', + unit_map['unit'], + requested_display_unit, + min) + converted_max = uc.convert('kinematic viscosity', + unit_map['unit'], + requested_display_unit, + max) + else: + converted_min = uc.convert(unit_map['unit'], + requested_display_unit, + min) + converted_max = uc.convert(unit_map['unit'], + requested_display_unit, + max) + # A mapserver class per color + floating_class_template = copy.deepcopy(default_floating_template) + beached_class_template = copy.deepcopy(default_beached_template) + units = unit_map['unit'] + class_label = '' + if idx == 0 and len(colormap['colorScaleRange']) == 1: + # Special case that we only have one range... so we show ALL + # particles, but still label it with the range + floating_class_template['expression'] = ('[statuscode] = 2 AND ' + f'[spill_id] IN "{spill_group_string}"') + beached_class_template['expression'] = ('[statuscode] = 3 AND ' + f'[spill_id] IN "{spill_group_string}"') + class_label = f'<{converted_min:#.4g} - {data_column} - {converted_max:#.4g}+ ({requested_display_unit})' + elif idx == 0: + # First one... open ended lower range + floating_class_template['expression'] = ('[statuscode] = 2 AND ' + f'[spill_id] IN "{spill_group_string}" AND ' + f'[{data_column}] <= {max}') + beached_class_template['expression'] = ('[statuscode] = 3 AND ' + f'[spill_id] IN "{spill_group_string}" AND ' + f'[{data_column}] <= {max}') + class_label = f'<{converted_min:#.4g} - {data_column} - {converted_max:#.4g} ({requested_display_unit})' + elif idx == len(colormap['colorScaleRange'])-1: + # Last one... open ended upper range + floating_class_template['expression'] = ('[statuscode] = 2 AND ' + f'[spill_id] IN "{spill_group_string}" AND ' + f'{min} <= [{data_column}]') + beached_class_template['expression'] = ('[statuscode] = 3 AND ' + f'[spill_id] IN "{spill_group_string}" AND ' + f'{min} <= [{data_column}]') + class_label = f'{converted_min:#.4g} - {data_column} - {converted_max:#.4g}+ ({requested_display_unit})' + else: + floating_class_template['expression'] = ('[statuscode] = 2 AND ' + f'[spill_id] IN "{spill_group_string}" AND ' + f'({min} < [{data_column}] AND ' + f'[{data_column}] <= {max})') + beached_class_template['expression'] = ('[statuscode] = 3 AND ' + f'[spill_id] IN "{spill_group_string}" AND ' + f'({min} < [{data_column}] AND ' + f'[{data_column}] <= {max})') + class_label = f'{converted_min:#.4g} - {data_column} - {converted_max:#.4g} ({requested_display_unit})' + colorblocklabel = colormap['colorBlockLabels'][idx] + floating_class_template['name'] = f'{spill_names}|{colorblocklabel if colorblocklabel else class_label}' + floating_class_template['ordering'] = next(classorder) + floating_class_template['styles'][0]['color'] = color + floating_class_template['styles'][0]['style_size'] = style_size + beached_class_template['name'] = f'{spill_names}|Beached' + beached_class_template['ordering'] = next(classorder) + beached_class_template['styles'][0]['color'] = color + beached_class_template['styles'][0]['outlinecolor'] = color + beached_class_template['styles'][0]['style_size'] = style_size + min = max + layer_template['mapfile_layer']['layer_classes'].append(floating_class_template) + layer_template['mapfile_layer']['layer_classes'].append(beached_class_template) + ## # Loop through the classes... and set "ordering" in reverse order + ## for idx, layer_class in enumerate(list(reversed(layer_template['mapfile_layer']['layer_classes']))): + ## layer_class['ordering'] = idx else: - raise ValueError(f'Style requested is not supported!!! {requested_display_param}') - # Looks like the domains are always in the base units... - number_scale_domain = [val for val in colormap['numberScaleDomain']] - color_scale_domain = [val for val in colormap['colorScaleDomain']] - #number_scale_domain = [uc.convert(requested_display_unit, - # unit_map['unit'], val) for val in colormap['numberScaleDomain']] - #color_scale_domain = [uc.convert(requested_display_unit, - # unit_map['unit'], val) for val in colormap['colorScaleDomain']] - min = float(number_scale_domain[0]) - max = None - data_column = unit_map['column'] - style_size = 8 * appearance['scale'] - if uncertain: - uncertain_color = '#FF0000' - # In the uncertain case, we just color everything red + # No appearance data... use a default floating_class_template = copy.deepcopy(default_floating_template) beached_class_template = copy.deepcopy(default_beached_template) floating_class_template['expression'] = ('[statuscode] = 2 AND ' f'[spill_id] IN "{spill_group_string}"') - floating_class_template['styles'][0]['color'] = uncertain_color - floating_class_template['styles'][0]['style_size'] = style_size - floating_class_template['name'] = f'{spill_names}|Floating Uncertain' beached_class_template['expression'] = ('[statuscode] = 3 AND ' f'[spill_id] IN "{spill_group_string}"') - beached_class_template['styles'][0]['color'] = uncertain_color - beached_class_template['styles'][0]['outlinecolor'] = uncertain_color - beached_class_template['styles'][0]['style_size'] = style_size - beached_class_template['name'] = f'{spill_names}|Beached Uncertain' + if uncertain: + uncertain_color = '#FF0000' + floating_class_template['styles'][0]['color'] = uncertain_color + floating_class_template['styles'][0]['outlinecolor'] = uncertain_color + floating_class_template['name'] = f'{spill_names}|Floating' + beached_class_template['styles'][0]['color'] = uncertain_color + beached_class_template['styles'][0]['outlinecolor'] = uncertain_color + beached_class_template['name'] = f'{spill_names}|Beached' + else: + floating_class_template['name'] = f'{spill_names}|Floating' + beached_class_template['name'] = f'{spill_names}|Beached' layer_template['mapfile_layer']['layer_classes'].append(floating_class_template) layer_template['mapfile_layer']['layer_classes'].append(beached_class_template) - else: - # Loop through the color scale and make classes - for idx, color in enumerate(colormap['colorScaleRange']): - if (idx+1) > len(color_scale_domain): - max = float(number_scale_domain[1]) - else: - max = float(color_scale_domain[idx]) - # Convert our min/max with unit conversion for labeling - if requested_display_param == 'Mass': - converted_min = convert_mass_to_mass_or_volume(unit_map['unit'], - requested_display_unit, - spill.substance.standard_density, - min) - converted_max = convert_mass_to_mass_or_volume(unit_map['unit'], - requested_display_unit, - spill.substance.standard_density, - max) - elif requested_display_param == 'Surface Concentration': - converted_min = uc.convert('oil concentration', - unit_map['unit'], - requested_display_unit, - min) - converted_max = uc.convert('oil concentration', - unit_map['unit'], - requested_display_unit, - max) - elif requested_display_param == 'Viscosity': - converted_min = uc.convert('kinematic viscosity', - unit_map['unit'], - requested_display_unit, - min) - converted_max = uc.convert('kinematic viscosity', - unit_map['unit'], - requested_display_unit, - max) - else: - converted_min = uc.convert(unit_map['unit'], - requested_display_unit, - min) - converted_max = uc.convert(unit_map['unit'], - requested_display_unit, - max) - # A mapserver class per color - floating_class_template = copy.deepcopy(default_floating_template) - beached_class_template = copy.deepcopy(default_beached_template) - units = unit_map['unit'] - class_label = '' - if idx == 0 and len(colormap['colorScaleRange']) == 1: - # Special case that we only have one range... so we show ALL - # particles, but still label it with the range - floating_class_template['expression'] = ('[statuscode] = 2 AND ' - f'[spill_id] IN "{spill_group_string}"') - beached_class_template['expression'] = ('[statuscode] = 3 AND ' - f'[spill_id] IN "{spill_group_string}"') - class_label = f'<{converted_min:#.4g} - {data_column} - {converted_max:#.4g}+ ({requested_display_unit})' - elif idx == 0: - # First one... open ended lower range - floating_class_template['expression'] = ('[statuscode] = 2 AND ' - f'[spill_id] IN "{spill_group_string}" AND ' - f'[{data_column}] <= {max}') - beached_class_template['expression'] = ('[statuscode] = 3 AND ' - f'[spill_id] IN "{spill_group_string}" AND ' - f'[{data_column}] <= {max}') - class_label = f'<{converted_min:#.4g} - {data_column} - {converted_max:#.4g} ({requested_display_unit})' - elif idx == len(colormap['colorScaleRange'])-1: - # Last one... open ended upper range - floating_class_template['expression'] = ('[statuscode] = 2 AND ' - f'[spill_id] IN "{spill_group_string}" AND ' - f'{min} <= [{data_column}]') - beached_class_template['expression'] = ('[statuscode] = 3 AND ' - f'[spill_id] IN "{spill_group_string}" AND ' - f'{min} <= [{data_column}]') - class_label = f'{converted_min:#.4g} - {data_column} - {converted_max:#.4g}+ ({requested_display_unit})' - else: - floating_class_template['expression'] = ('[statuscode] = 2 AND ' - f'[spill_id] IN "{spill_group_string}" AND ' - f'({min} < [{data_column}] AND ' - f'[{data_column}] <= {max})') - beached_class_template['expression'] = ('[statuscode] = 3 AND ' - f'[spill_id] IN "{spill_group_string}" AND ' - f'({min} < [{data_column}] AND ' - f'[{data_column}] <= {max})') - class_label = f'{converted_min:#.4g} - {data_column} - {converted_max:#.4g} ({requested_display_unit})' - colorblocklabel = colormap['colorBlockLabels'][idx] - floating_class_template['name'] = f'{spill_names}|{colorblocklabel if colorblocklabel else class_label}' - floating_class_template['ordering'] = next(classorder) - floating_class_template['styles'][0]['color'] = color - floating_class_template['styles'][0]['style_size'] = style_size - beached_class_template['name'] = f'{spill_names}|Beached' - beached_class_template['ordering'] = next(classorder) - beached_class_template['styles'][0]['color'] = color - beached_class_template['styles'][0]['outlinecolor'] = color - beached_class_template['styles'][0]['style_size'] = style_size - min = max - layer_template['mapfile_layer']['layer_classes'].append(floating_class_template) - layer_template['mapfile_layer']['layer_classes'].append(beached_class_template) - # Loop through the classes... and set "ordering" in reverse order - for idx, layer_class in enumerate(list(reversed(layer_template['mapfile_layer']['layer_classes']))): - layer_class['ordering'] = idx + # Loop through the classes... and set "ordering" in reverse order + for idx, layer_class in enumerate(list(reversed(layer_template['mapfile_layer']['layer_classes']))): + layer_class['ordering'] = idx + with open(output_path, "w") as o: + json.dump(layer_template, o, indent=4) + return {'shapefile_filename': shapefile_filename, + 'json_filename': dir+"/"+str(id)+".json"} + else: + # Default styling was selected... collapse everything to simple styling + # No appearance data... use a default + floating_class_template = copy.deepcopy(default_floating_template) + beached_class_template = copy.deepcopy(default_beached_template) + floating_class_template['expression'] = '[statuscode] = 2' + beached_class_template['expression'] = '[statuscode] = 3' + if uncertain: + uncertain_color = '#FF0000' + floating_class_template['styles'][0]['color'] = uncertain_color + floating_class_template['styles'][0]['outlinecolor'] = uncertain_color + floating_class_template['name'] = 'Floating' + beached_class_template['styles'][0]['color'] = uncertain_color + beached_class_template['styles'][0]['outlinecolor'] = uncertain_color + beached_class_template['name'] = 'Beached' else: - # No appearance data... use a default - floating_class_template = copy.deepcopy(default_floating_template) - beached_class_template = copy.deepcopy(default_beached_template) - floating_class_template['expression'] = ('[statuscode] = 2 AND ' - f'[spill_id] IN "{spill_group_string}"') - beached_class_template['expression'] = ('[statuscode] = 3 AND ' - f'[spill_id] IN "{spill_group_string}"') - if uncertain: - uncertain_color = '#FF0000' - floating_class_template['styles'][0]['color'] = uncertain_color - floating_class_template['styles'][0]['outlinecolor'] = uncertain_color - floating_class_template['name'] = f'{spill_names}|Floating' - beached_class_template['styles'][0]['color'] = uncertain_color - beached_class_template['styles'][0]['outlinecolor'] = uncertain_color - beached_class_template['name'] = f'{spill_names}|Beached' - else: - floating_class_template['name'] = f'{spill_names}|Floating' - beached_class_template['name'] = f'{spill_names}|Beached' + floating_class_template['name'] = 'Floating' + beached_class_template['name'] = 'Beached' layer_template['mapfile_layer']['layer_classes'].append(floating_class_template) layer_template['mapfile_layer']['layer_classes'].append(beached_class_template) - with open(output_path, "w") as o: - json.dump(layer_template, o, indent=4) - return {'shapefile_filename': shapefile_filename, - 'json_filename': dir+"/"+str(id)+".json"} + with open(output_path, "w") as o: + json.dump(layer_template, o, indent=4) + return {'shapefile_filename': shapefile_filename, + 'json_filename': dir+"/"+str(id)+".json"} else: raise ValueError("Can not write ERMA Data Package without template!!!") diff --git a/py_gnome/tests/pytest.ini b/py_gnome/tests/pytest.ini index e38acbf09..59b2e4d76 100644 --- a/py_gnome/tests/pytest.ini +++ b/py_gnome/tests/pytest.ini @@ -3,3 +3,4 @@ filterwarnings = ignore:.*BlockManager to :DeprecationWarning ignore:.*Possibly due to too larger :RuntimeWarning +addopts = --ignore=profiling From da4605fbcafe9efefe13be7a2cb1ab16ced5804a Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Thu, 25 Jul 2024 07:24:51 +0000 Subject: [PATCH 13/16] Addressed all Warnings triggered by the tests. --- py_gnome/conda_requirements.txt | 2 +- py_gnome/gnome/cy_gnome/cy_cats_mover.pyx | 3 + py_gnome/gnome/cy_gnome/cy_ossm_time.pyx | 3 +- py_gnome/gnome/cy_gnome/cy_shio_time.pyx | 36 +++++----- py_gnome/gnome/spills/spill.py | 58 ++++++++------- py_gnome/gnome/utilities/cache.py | 31 ++++---- py_gnome/gnome/weatherers/roc.py | 10 +-- py_gnome/gnome/weatherers/spreading.py | 5 +- py_gnome/tests/unit_tests/conftest.py | 71 +++++++++---------- .../unit_tests/test_cy/test_cy_wind_mover.py | 6 +- .../test_environment_objects.py | 2 +- .../test_environment/test_gridcur.py | 2 +- .../tests/unit_tests/test_maps/test_map.py | 14 ++-- .../unit_tests/test_maps/test_tideflat_map.py | 9 +-- py_gnome/tests/unit_tests/test_model.py | 47 ++++++------ .../test_model_integration/test_model_map.py | 8 +-- .../tests/unit_tests/test_model_multiproc.py | 4 +- .../test_movers/test_simple_mover.py | 39 +--------- .../unit_tests/test_movers/test_wind_mover.py | 15 ++-- .../tests/unit_tests/test_ops/test_density.py | 28 ++++---- .../unit_tests/test_ops/test_viscosity.py | 6 +- .../test_outputters/test_current_outputter.py | 63 ++++++---------- .../test_outputters/test_erma_data_package.py | 4 +- .../test_outputters/test_geojson.py | 5 +- .../test_ice_image_outputter.py | 5 +- .../test_ice_json_outputter.py | 45 ++++-------- .../test_outputters/test_ice_outputter.py | 35 +++++++-- .../unit_tests/test_outputters/test_kmz.py | 5 +- .../test_outputters/test_netcdf_outputter.py | 9 +-- .../test_oil_budget_outputter.py | 4 +- .../test_outputters/test_outputter.py | 8 +-- .../test_outputters/test_renderer.py | 4 +- .../unit_tests/test_outputters/test_shape.py | 4 +- .../test_weathering_outputter.py | 4 +- .../test_persist/test_model_save_load.py | 4 +- .../test_persist/test_old_savefiles.py | 2 + .../unit_tests/test_persist/test_save_load.py | 60 ++++++++-------- .../unit_tests/test_persist/test_schemas.py | 10 ++- .../unit_tests/test_spills/test_release.py | 8 +-- .../test_spills/test_release_in_model.py | 12 ++-- .../test_spatial_release_spreading.py | 2 +- py_gnome/tests/unit_tests/test_tideflats.py | 4 +- .../tests/unit_tests/test_update_from_dict.py | 10 +-- .../unit_tests/test_utilities/test_cache.py | 24 +++++-- .../test_weatherers/test_cleanup.py | 4 +- .../test_weatherers/test_evaporation.py | 6 +- .../test_weatherers/test_spreading.py | 13 ++-- 47 files changed, 373 insertions(+), 380 deletions(-) diff --git a/py_gnome/conda_requirements.txt b/py_gnome/conda_requirements.txt index 487f99afd..8924007aa 100644 --- a/py_gnome/conda_requirements.txt +++ b/py_gnome/conda_requirements.txt @@ -38,7 +38,7 @@ pyzmq # tblib ## Base numerical libs -numpy <2 +numpy >=1.24,<2 scipy ## Modeling libs diff --git a/py_gnome/gnome/cy_gnome/cy_cats_mover.pyx b/py_gnome/gnome/cy_gnome/cy_cats_mover.pyx index 02211c4a5..442a45d70 100644 --- a/py_gnome/gnome/cy_gnome/cy_cats_mover.pyx +++ b/py_gnome/gnome/cy_gnome/cy_cats_mover.pyx @@ -200,6 +200,9 @@ cdef class CyCatsMover(CyCurrentMover): Takes a CyShioTime object as input and sets C++ Cats mover properties from the Shio object. """ + print("in set_shio") + print("CyShioTime:", CyShioTime) + print("cy_shio:", cy_shio) self.cats.SetTimeDep(cy_shio.shio) if cy_shio.station_location is not None and self.ref_point is None: diff --git a/py_gnome/gnome/cy_gnome/cy_ossm_time.pyx b/py_gnome/gnome/cy_gnome/cy_ossm_time.pyx index ce2a8d48a..bd0613dc3 100644 --- a/py_gnome/gnome/cy_gnome/cy_ossm_time.pyx +++ b/py_gnome/gnome/cy_gnome/cy_ossm_time.pyx @@ -155,7 +155,8 @@ cdef class CyOSSMTime(object): wp['z'][:] = wp['z'] g_wp = np.zeros((1,), dtype=basic_types.world_point) - g_wp[0] = (wp['long'], wp['lat'], 0) + g_wp[0] = (wp['long'][0], wp['lat'][0], 0) + return tuple(g_wp[0]) diff --git a/py_gnome/gnome/cy_gnome/cy_shio_time.pyx b/py_gnome/gnome/cy_gnome/cy_shio_time.pyx index aeb3f17d0..11d58fda4 100644 --- a/py_gnome/gnome/cy_gnome/cy_shio_time.pyx +++ b/py_gnome/gnome/cy_gnome/cy_shio_time.pyx @@ -144,25 +144,27 @@ cdef class CyShioTime(CyOSSMTime): 'yeardata={0.yeardata!r}' ')'.format(self)) - def __str__(self): - """Return string representation of this object""" - """info = {'Long': round(g_wp[0]['long'], 2), - 'Lat': round( g_wp[0]['lat'], 2), - 'StationName': sName, 'StationType': sType, - 'DaylightSavingsOff': self.shio.daylight_savings_off} - """ - - info = ("CyShioTime object - Info read from file:\n" - " File: {1.path_filename} \n" - " StationName : {0[StationName]}," - " StationType : {0[StationType]}\n" - " (Long, Lat) : ({0[Long]}, {0[Lat]})\n" - " DaylightSavingsOff : {0[DaylightSavingsOff]}" - "".format(self.get_info(), self)) - - return info + # this is broken -- no get_info method ???? + # def __str__(self): + # """Return string representation of this object""" + # """info = {'Long': round(g_wp[0]['long'], 2), + # 'Lat': round( g_wp[0]['lat'], 2), + # 'StationName': sName, 'StationType': sType, + # 'DaylightSavingsOff': self.shio.daylight_savings_off} + # """ + + # info = ("CyShioTime object - Info read from file:\n" + # " File: {1.path_filename} \n" + # " StationName : {0[StationName]}," + # " StationType : {0[StationType]}\n" + # " (Long, Lat) : ({0[Long]}, {0[Lat]})\n" + # " DaylightSavingsOff : {0[DaylightSavingsOff]}" + # "".format(self.get_info(), self)) + + # return info def __eq(self, CyShioTime other): + # used by __richcmp__ attrs = ('filename', 'daylight_savings_off', 'scale_factor', 'station', 'station_type', 'station_location', 'yeardata') diff --git a/py_gnome/gnome/spills/spill.py b/py_gnome/gnome/spills/spill.py index 50cac9b19..e1545f034 100644 --- a/py_gnome/gnome/spills/spill.py +++ b/py_gnome/gnome/spills/spill.py @@ -522,37 +522,40 @@ def _setup_spill(release, return spill def point_line_spill(num_elements, - start_position, - release_time, - end_position=None, - end_release_time=None, - substance=None, - amount=0, - units='kg', - water=None, - on=True, - windage_range=None, - windage_persist=None, - name='Point or Line Release'): + start_position, + release_time, + end_position=None, + end_release_time=None, + substance=None, + amount=0, + units='kg', + water=None, + on=True, + windage_range=None, + windage_persist=None, + name='Point-Line Spill'): ''' - Helper function returns a Spill object + Helper function to create a spill from a point or line source. + + Returns a configured Spill object :param num_elements: total number of elements to be released :type num_elements: integer :param start_position: initial location the elements are released - :type start_position: 3-tuple of floats (long, lat, postive depth) or - 2-tuple of floats (lon,lat) in which case depth will default to 0 + :type start_position: 3-tuple of floats (long, lat, positive depth) or + 2-tuple of floats (lon, lat) in which case depth will default to 0. - :param release_time: time the LEs are released (datetime object) + :param release_time: time the elements are released :type release_time: datetime.datetime :param end_position=None: Optional. For moving source, the end position If None, then release is from a point source :type end_position: 2-tuple of floats (long, lat) - :param end_release_time=None: optional -- for a time varying release, - the end release time. If None, then release is instantaneous + :param end_release_time=None: Optional -- for a time varying release, + the end release time. If None, then + the release is instantaneous :type end_release_time: datetime.datetime :param substance=None: Type of oil spilled. @@ -566,15 +569,15 @@ def point_line_spill(num_elements, :param tuple windage_range=(.01, .04): Percentage range for windage. Active only for surface particles - when a mind mover is added - :type windage_range: tuple + when a wind mover is added + :type windage_range: 2-tuple in fraction of wind speed the elements will move. :param windage_persist=900: Persistence for windage values in seconds. - Use -1 for inifinite, otherwise it is - randomly reset on this time scale - :type windage_persist: int + Use -1 for infinite, otherwise it is + randomly reset on this time scale + :type windage_persist: int seconds. - :param name='Point/Line Spill': a name for the spill + :param name='Point-Line Spill': a name for the spill :type name: str ''' # make positions 3d, with depth = 0 if they are not already @@ -617,7 +620,9 @@ def surface_point_line_spill(num_elements, windage_persist=None, name='Surface Point or Line Release'): ''' - Helper function returns a Spill object + Helper function for a surface spill at a point or over a line. + + Returns a configured Spill object :param num_elements: total number of elements to be released :type num_elements: integer @@ -658,7 +663,8 @@ def surface_point_line_spill(num_elements, :param name='Surface Point/Line Spill': a name for the spill :type name: str ''' - warn('This helper function is deprecated in favor of point_line_spill.', DeprecationWarning) + warn('The `surface_point_line_spill` helper function is deprecated in favor of point_line_spill.', + DeprecationWarning) # make positions 3d, with depth = 0 if they are not already start_position = *start_position[:2], 0 diff --git a/py_gnome/gnome/utilities/cache.py b/py_gnome/gnome/utilities/cache.py index 8c2f2f4f9..6b165fbb2 100644 --- a/py_gnome/gnome/utilities/cache.py +++ b/py_gnome/gnome/utilities/cache.py @@ -23,16 +23,21 @@ # this should happen once, on first import # it will get cleaned up when python exits # all individual cache dirs go in this one. +# FixME: why have one temp dir per Python instance, +# rather than one per cache instance? -_cache_dir = tempfile.mkdtemp(prefix='gnome.') +# global_cache_dir = tempfile.mkdtemp(prefix='gnome_cache.') +_CacheDirObject = tempfile.TemporaryDirectory(prefix='gnome_cache.') +global_cache_dir = _CacheDirObject.name class CacheError(Exception): 'Here so we can be sure the user knows the error is coming from here' pass - -def clean_up_cache(dir_name=_cache_dir): +# No longer needed -- tempfile.TemporaryDirectory takes care of this for us +# but it's used in a test, so ... +def clean_up_cache(dir_name=global_cache_dir): """ Deletes a cache dir. @@ -52,13 +57,11 @@ def clean_up_cache(dir_name=_cache_dir): warnings.warn(repr(excp)) -# need to clean up temp directories at exit: -# this will clean up the master temp dir, and anything in it if -# something went wrong with __del__ in the individual objects -atexit.register(clean_up_cache) +# no longer needed -- tempfile.TempDir takes care of this +# atexit.register(clean_up_cache) -class ElementCache(object): +class ElementCache(): """ Cache for element data -- i.e. the data associated with the particles. This caches UncertainSpillContainerPair @@ -78,7 +81,9 @@ def __init__(self, cache_dir=None, enabled=True): If not provided, a temp dir will be created by the python tempfile module """ - self.create_new_dir(cache_dir) + # if cache_dir is None: + # cache_dir = tempfile.mkdtemp(dir=global_cache_dir) + self._create_new_dir(cache_dir) # dict to hold recent data so we don't need to pull from the # file system @@ -110,9 +115,11 @@ def _make_filename(self, step_num, uncertain=False): return os.path.join(self._cache_dir, 'step_%06i.npz' % step_num) - def create_new_dir(self, cache_dir=None): + def _create_new_dir(self, cache_dir=None): if cache_dir is None: - self._cache_dir = tempfile.mkdtemp(dir=_cache_dir) + self.cache_dir_obj = tempfile.TemporaryDirectory(dir=global_cache_dir) + self._cache_dir = self.cache_dir_obj.name + # self._cache_dir = tempfile.mkdtemp(dir=global_cache_dir) else: self._cache_dir = cache_dir return True @@ -245,4 +252,4 @@ def rewind(self): # clean out the disk cache if os.path.isdir(self._cache_dir): shutil.rmtree(self._cache_dir) - self.create_new_dir() + self._create_new_dir() diff --git a/py_gnome/gnome/weatherers/roc.py b/py_gnome/gnome/weatherers/roc.py index cec5a6170..c66fa6341 100644 --- a/py_gnome/gnome/weatherers/roc.py +++ b/py_gnome/gnome/weatherers/roc.py @@ -1408,11 +1408,12 @@ def simulate_plane(self, sc, time_step, model_time): treated_possible = disp_actual * self.disp_oil_ratio mass_treatable = None - if np.isnan(np.mean(sc['density'][self.dispersable_oil_idxs(sc) - ])): + disp_oil_idx = self.dispersable_oil_idxs(sc) + + if not len(disp_oil_idx) or np.isnan(np.mean(sc['density'][disp_oil_idx])): mass_treatable = 0 else: - mass_treatable = np.mean(sc['density'][self.dispersable_oil_idxs(sc)]) * treated_possible + mass_treatable = np.mean(sc['density'][disp_oil_idx]) * treated_possible oil_avail = self.dispersable_oil_amount(sc, 'kg') @@ -1563,7 +1564,8 @@ def dispersable_oil_idxs(self, sc): def dispersable_oil_amount(self, sc, units='gal'): idxs = self.dispersable_oil_idxs(sc) - + if not len(idxs): + return 0 if units in _valid_vol_units: tot_vol = np.sum(sc['mass'][idxs] / sc['density'][idxs]) return max(0, uc.convert('m^3', units, tot_vol)) diff --git a/py_gnome/gnome/weatherers/spreading.py b/py_gnome/gnome/weatherers/spreading.py index 4134c95a5..a06a0d58e 100644 --- a/py_gnome/gnome/weatherers/spreading.py +++ b/py_gnome/gnome/weatherers/spreading.py @@ -866,8 +866,9 @@ def weather_elements(self, sc, time_step, model_time): # of oil released at same time, from same spill, sum # the 'fay_area' array for elements that belong to same oil # blob. - thickness = (data['bulk_init_volume'][s_mask][0] / - data['fay_area'][s_mask].sum()) + with np.errstate(divide='ignore'): # OK to get inf -- it will do the right thing later. + thickness = (data['bulk_init_volume'][s_mask][0] + / data['fay_area'][s_mask].sum()) # assume only one type of oil is modeled so thickness_limit is # already set and constant for all diff --git a/py_gnome/tests/unit_tests/conftest.py b/py_gnome/tests/unit_tests/conftest.py index 731b4d882..50aefd115 100644 --- a/py_gnome/tests/unit_tests/conftest.py +++ b/py_gnome/tests/unit_tests/conftest.py @@ -31,9 +31,10 @@ from gnome.gnomeobject import class_from_objtype, GnomeId from gnome.spills.substance import NonWeatheringSubstance from gnome.spills.gnome_oil import GnomeOil +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import Release, VerticalPlumeRelease from gnome.spill_container import SpillContainer - base_dir = os.path.dirname(__file__) # test_oil = u'ALASKA NORTH SLOPE (MIDDLE PIPELINE)' @@ -223,10 +224,7 @@ def sample_sc_release(num_elements=10, current_time = release_time if spill is None: - spill = gnome.spills.surface_point_line_spill(num_elements, - start_pos, - release_time, - amount=0) + spill = point_line_spill(num_elements, start_pos, release_time, amount=0) spill.units = units spill.amount = amount_per_element * num_elements @@ -561,14 +559,14 @@ def sample_release_spill(): :returns: a tuple containing (spill, start_positions). start_positions should be equal to spill.start_positions """ - from gnome.spills import Release + start_positions = ((0., 0., 0.), (28.0, -75.0, 0.), (-15, 12, 4.0), (80, -80, 100.0)) rel = Release(release_time=datetime(2012, 1, 1, 1), custom_positions=start_positions) - sp = gnome.spills.Spill(release=rel) + sp = Spill(release=rel) return (sp, start_positions) @@ -578,7 +576,6 @@ def sample_vertical_plume_spill(): ''' creates an example VerticalPlumeSource object ''' - from gnome.spills import VerticalPlumeRelease, Spill from gnome.utilities.plume import get_plume_data release_time = datetime.now() @@ -597,7 +594,7 @@ def sample_vertical_plume_spill(): @pytest.fixture(scope='function') def sample_sc_no_uncertainty(): """ - Sample spill container with 2 surface_point_line_spill spills: + Sample spill container with 2 surface point_line_spill spills: - release_time for 2nd spill is 1 hour delayed - 2nd spill takes 4 hours to release and end_position is different so it @@ -619,23 +616,23 @@ def sample_sc_no_uncertainty(): end_position = (24.0, -79.5, 1.0) end_release_time = datetime(2012, 1, 1, 12) + timedelta(hours=4) - spills = [gnome.spills.surface_point_line_spill(num_elements, - start_position, - release_time, - amount=10, units='l', - water=water), - gnome.spills.surface_point_line_spill(num_elements, - start_position, - release_time_2, - end_position, - end_release_time, - water=water), - ] + spills = [ + point_line_spill(num_elements, + start_position, + release_time, + amount=10, + units='l', + water=water), + point_line_spill(num_elements, + start_position, + release_time_2, + end_position, + end_release_time, + water=water), + ] sc.spills.add(spills) return sc - - # @pytest.fixture(scope='module') def sample_model_fixture_base(): """ @@ -789,13 +786,13 @@ def sample_model_weathering(sample_model_fcn, sub = GnomeOil(oil) start_time = model.start_time + timedelta(hours=1) end_time = start_time + timedelta(seconds=model.time_step * 3) - spill = gnome.spills.surface_point_line_spill(num_les, - rel_pos, - start_time, - end_release_time=end_time, - substance=sub, - amount=100, - units='kg') + spill = point_line_spill(num_les, + rel_pos, + start_time, + end_release_time=end_time, + substance=sub, + amount=100, + units='kg') model.spills += spill # define environment objects that weatherers require @@ -815,13 +812,13 @@ def sample_model_weathering2(sample_model_fcn2, oil, temp=311.16): sub = GnomeOil(oil) start_time = model.start_time end_time = start_time - spill = gnome.spills.surface_point_line_spill(100, - rel_pos, - start_time, - end_release_time=end_time, - substance=sub, - amount=10000, - units='kg') + spill = point_line_spill(100, + rel_pos, + start_time, + end_release_time=end_time, + substance=sub, + amount=10000, + units='kg') model.spills += spill return model diff --git a/py_gnome/tests/unit_tests/test_cy/test_cy_wind_mover.py b/py_gnome/tests/unit_tests/test_cy/test_cy_wind_mover.py index 694106ac8..f9aaa5ef5 100644 --- a/py_gnome/tests/unit_tests/test_cy/test_cy_wind_mover.py +++ b/py_gnome/tests/unit_tests/test_cy/test_cy_wind_mover.py @@ -6,9 +6,6 @@ designed to be run with py.test """ - - - import os import numpy as np @@ -79,7 +76,8 @@ def test_eq(): # use following constant wind for testing -const_wind = np.zeros((1, ), dtype=velocity_rec) +# can't figure out how to get a scalar other than create a array and extract ... +const_wind = np.zeros((1, ), dtype=velocity_rec)[0] const_wind['u'] = 50 # meters per second? const_wind['v'] = 100 diff --git a/py_gnome/tests/unit_tests/test_environment/test_environment_objects.py b/py_gnome/tests/unit_tests/test_environment/test_environment_objects.py index 4a2056747..c1894c815 100644 --- a/py_gnome/tests/unit_tests/test_environment/test_environment_objects.py +++ b/py_gnome/tests/unit_tests/test_environment/test_environment_objects.py @@ -125,7 +125,7 @@ def test_SteadyUniformCurrent_in_model(): duration=gs.hours(1), time_step=gs.minutes(10), ) - model.spills += gs.surface_point_line_spill(num_elements=10, + model.spills += gs.point_line_spill(num_elements=10, release_time=start_time, start_position=(0,0)) suc = SteadyUniformCurrent(math.sqrt(2), 45, units='knots') diff --git a/py_gnome/tests/unit_tests/test_environment/test_gridcur.py b/py_gnome/tests/unit_tests/test_environment/test_gridcur.py index 1304bf1a5..e4361b1ee 100644 --- a/py_gnome/tests/unit_tests/test_environment/test_gridcur.py +++ b/py_gnome/tests/unit_tests/test_environment/test_gridcur.py @@ -15,7 +15,7 @@ from gnome.environment import gridcur from gnome.movers import CurrentMover -from gnome.spills import grid_spill +from gnome.spills.spill import grid_spill import gnome.scripting as gs diff --git a/py_gnome/tests/unit_tests/test_maps/test_map.py b/py_gnome/tests/unit_tests/test_maps/test_map.py index 3684197ac..cbb70e36c 100644 --- a/py_gnome/tests/unit_tests/test_maps/test_map.py +++ b/py_gnome/tests/unit_tests/test_maps/test_map.py @@ -172,7 +172,7 @@ def test_adding_a_map(self): gmap += GnomeMap() - +@pytest.mark.filterwarnings("ignore:ParamMap") class Test_ParamMap: ''' WIP @@ -607,6 +607,8 @@ def test_to_geojson(self): for c in coord_coll[0]: assert len(c) == 2 + # this is result of the serialization saving the bounds, just in case they changed. + @pytest.mark.filterwarnings("ignore:Provided map bounds superscede map bounds") def test_serialize_deserialize(self): """ test create new object from to_dict @@ -962,12 +964,11 @@ def test_serialize(self): assert ser['spillable_area'] is None - #need to add the file to file server - @pytest.mark.xfail() + # @pytest.mark.xfail() def test_serialize_from_blob_old(self): # this one uses the "old" name, before moving the map module. json_data = {'approximate_raster_interval': 53.9608870724, - 'filename': u'/Users/chris.barker/Hazmat/GitLab/pygnome/py_gnome/tests/unit_tests/sample_data/florida_with_lake_small.bna', + 'filename': bna_with_lake, 'id': u'b3590b7d-aab1-11ea-8899-1e00b098d304', 'map_bounds': [(-82.8609915978, 24.5472415066), (-82.8609915978, 28.1117673335), @@ -993,11 +994,10 @@ def test_serialize_from_blob_old(self): assert map.spillable_area is None assert len(map.map_bounds) == 4 - @pytest.mark.xfail() - def test_serialize_from_blob_new(self): + # @pytest.mark.xfail() def test_serialize_from_blob_new(self): # this one uses the "new" name, after moving the map module. json_data = {'approximate_raster_interval': 53.9608870724, - 'filename': u'/Users/chris.barker/Hazmat/GitLab/pygnome/py_gnome/tests/unit_tests/sample_data/florida_with_lake_small.bna', + 'filename': bna_with_lake, 'id': u'b3590b7d-aab1-11ea-8899-1e00b098d304', 'map_bounds': [(-82.8609915978, 24.5472415066), (-82.8609915978, 28.1117673335), diff --git a/py_gnome/tests/unit_tests/test_maps/test_tideflat_map.py b/py_gnome/tests/unit_tests/test_maps/test_tideflat_map.py index b8a3ad202..d01041d9e 100644 --- a/py_gnome/tests/unit_tests/test_maps/test_tideflat_map.py +++ b/py_gnome/tests/unit_tests/test_maps/test_tideflat_map.py @@ -4,11 +4,7 @@ tests for the tidal flat map """ - - - - -# a few tests that show that the delation to the underlying map works: +# a few tests that show that the delegation to the underlying map works: import os from datetime import datetime @@ -72,7 +68,7 @@ def simple_model(output_dir): model.map = MapFromBNA(bna_file) model.movers += gs.constant_point_wind_mover(10, 300, "m/s") - model.spills += gs.surface_point_line_spill(num_elements=100, + model.spills += gs.point_line_spill(num_elements=100, start_position=(5.4, 53.38, 0), end_position=(5.8, 53.4, 0), release_time=start_time, @@ -256,6 +252,7 @@ def test_model_run_with_tideflat(simple_model): # to make it run faster model.time_step = gs.hours(2) + model.outputters[0].output_timestep = gs.hours(2) for step in model: print("step_num", step['step_num']) diff --git a/py_gnome/tests/unit_tests/test_model.py b/py_gnome/tests/unit_tests/test_model.py index ffc98bcb6..1a5b9fbcd 100644 --- a/py_gnome/tests/unit_tests/test_model.py +++ b/py_gnome/tests/unit_tests/test_model.py @@ -22,9 +22,8 @@ from gnome.environment import Wind, Tide, constant_wind, Water, Waves, GridWind from gnome.model import Model -from gnome.spills import (Spill, - surface_point_line_spill, - Release) +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import Release from gnome.movers import SimpleMover, RandomMover, PointWindMover, CatsMover, WindMover @@ -246,7 +245,7 @@ def test_release_end_of_step(duration): duration=timedelta(hours=duration)) end_release_time = model.start_time + model.duration - model.spills += surface_point_line_spill(10, (0.0, 0.0, 0.0), + model.spills += point_line_spill(10, (0.0, 0.0, 0.0), model.start_time, end_release_time=end_release_time) @@ -314,7 +313,7 @@ def test_simple_run_rewind(): model.movers += a_mover assert len(model.movers) == 1 - spill = surface_point_line_spill(num_elements=10, + spill = point_line_spill(num_elements=10, start_position=(0., 0., 0.), release_time=start_time) @@ -361,7 +360,7 @@ def test_simple_run_backward_rewind(): model.movers += a_mover assert len(model.movers) == 1 - spill = surface_point_line_spill(num_elements=10, + spill = point_line_spill(num_elements=10, start_position=(0., 0., 0.), release_time=start_time) @@ -406,7 +405,7 @@ def test_simple_run_with_map(): model.movers += a_mover assert len(model.movers) == 1 - spill = surface_point_line_spill(num_elements=10, + spill = point_line_spill(num_elements=10, start_position=(0., 0., 0.), release_time=start_time) @@ -638,7 +637,7 @@ def test_all_movers(start_time, release_delay, duration): release_time = (start_time + timedelta(seconds=model.time_step * release_delay)) - model.spills += surface_point_line_spill(num_elements=10, + model.spills += point_line_spill(num_elements=10, start_position=start_loc, release_time=release_time) @@ -723,7 +722,7 @@ def test_linearity_of_wind_movers(wind_persist): model1.duration = timedelta(hours=1) model1.time_step = timedelta(hours=1) model1.start_time = start_time - sp = surface_point_line_spill(num_elements=num_LEs, + sp = point_line_spill(num_elements=num_LEs, start_position=(1., 2., 0.), release_time=start_time, substance=NonWeatheringSubstance(windage_persist=wind_persist)) @@ -737,7 +736,7 @@ def test_linearity_of_wind_movers(wind_persist): model2.duration = timedelta(hours=10) model2.time_step = timedelta(hours=1) model2.start_time = start_time - model2.spills += surface_point_line_spill(num_elements=num_LEs, + model2.spills += point_line_spill(num_elements=num_LEs, start_position=(1., 2., 0.), release_time=start_time, substance=NonWeatheringSubstance(windage_persist=wind_persist)) @@ -795,13 +794,13 @@ def test_model_release_after_start(): # add a spill that starts after the run begins. release_time = start_time + timedelta(hours=1) - model.spills += surface_point_line_spill(num_elements=5, + model.spills += point_line_spill(num_elements=5, start_position=(0, 0, 0), release_time=release_time) # and another that starts later.. - model.spills += surface_point_line_spill(num_elements=4, + model.spills += point_line_spill(num_elements=4, start_position=(0, 0, 0), release_time=(start_time + timedelta(hours=2)) @@ -833,13 +832,13 @@ def test_model_release_after_start_backwards(): # add a spill that starts after the run begins. release_time = start_time - timedelta(hours=1) - model.spills += surface_point_line_spill(num_elements=5, + model.spills += point_line_spill(num_elements=5, start_position=(0, 0, 0), release_time=release_time) # and another that starts later.. - model.spills += surface_point_line_spill(num_elements=4, + model.spills += point_line_spill(num_elements=4, start_position=(0, 0, 0), release_time=(start_time - timedelta(hours=2)) @@ -881,7 +880,7 @@ def test_release_at_right_time(): # add a spill that starts right when the run begins - model.spills += surface_point_line_spill(num_elements=12, + model.spills += point_line_spill(num_elements=12, start_position=(0, 0, 0), release_time=start_time, end_release_time=start_time + gs.hours(6), @@ -1102,7 +1101,7 @@ def test_contains_object(sample_model_fcn): water, wind = Water(), constant_wind(1., 0) model.environment += [water, wind] - sp = surface_point_line_spill(500, (0, 0, 0), + sp = point_line_spill(500, (0, 0, 0), rel_time + timedelta(hours=1), substance=model.spills[0].substance, amount=100, @@ -1200,7 +1199,7 @@ def test_staggered_spills_weathering(sample_model_fcn, delay): model.cache = True model.outputters += gnome.outputters.WeatheringOutput() - cs = surface_point_line_spill(500, (0, 0, 0), + cs = point_line_spill(500, (0, 0, 0), rel_time + delay, end_release_time=(rel_time + delay + timedelta(hours=1)), @@ -1278,7 +1277,7 @@ def test_two_substance_same(sample_model_fcn, s0=test_oil, s1=test_oil): rel_time = model.spills[0].release_time model.duration = timedelta(days=1) - cs = surface_point_line_spill(500, (0, 0, 0), + cs = point_line_spill(500, (0, 0, 0), rel_time, end_release_time=(rel_time + timedelta(hours=1)), @@ -1359,7 +1358,7 @@ def test_two_substance_different(sample_model_fcn, s0=test_oil, s1="oil_crude"): rel_time = model.spills[0].release_time model.duration = timedelta(days=1) - cs = surface_point_line_spill(500, (0, 0, 0), + cs = point_line_spill(500, (0, 0, 0), rel_time, end_release_time=(rel_time + timedelta(hours=1)), @@ -1402,8 +1401,8 @@ def test_weathering_data_attr(): s1_rel = datetime.now().replace(microsecond=0) s2_rel = s1_rel + timedelta(seconds=ts) - s = [surface_point_line_spill(10, (0, 0, 0), s1_rel), - surface_point_line_spill(10, (0, 0, 0), s2_rel)] + s = [point_line_spill(10, (0, 0, 0), s1_rel), + point_line_spill(10, (0, 0, 0), s2_rel)] model = Model(time_step=ts, start_time=s1_rel) model.spills += s @@ -1483,6 +1482,8 @@ def test_merge_from_empty_model(self, model): for item in getattr(m, oc): assert item in getattr(model, oc) + + @pytest.mark.filterwarnings("ignore:Provided map bounds superscede map bounds") def test_load_location_file(self, saveloc_, model): ''' create a model @@ -1493,7 +1494,7 @@ def test_load_location_file(self, saveloc_, model): m.environment += [Water(), constant_wind(1., 0.)] m.weatherers += Evaporation(m.environment[0], m.environment[-1]) # has to have the same substance as the sample model - m.spills += surface_point_line_spill(10, (0, 0, 0), + m.spills += point_line_spill(10, (0, 0, 0), datetime(2014, 1, 1, 12, 0), substance=test_oil) @@ -1703,7 +1704,7 @@ def test_water_missing(self): def test_get_spill_property(): model = Model() - model.spills += gs.surface_point_line_spill(num_elements=10, + model.spills += gs.point_line_spill(num_elements=10, start_position=(0.0, 0.0), release_time=model.start_time, ) diff --git a/py_gnome/tests/unit_tests/test_model_integration/test_model_map.py b/py_gnome/tests/unit_tests/test_model_integration/test_model_map.py index 30c0811e8..bdc078158 100644 --- a/py_gnome/tests/unit_tests/test_model_integration/test_model_map.py +++ b/py_gnome/tests/unit_tests/test_model_integration/test_model_map.py @@ -31,7 +31,7 @@ def test_off_map_removed_GnomeMap(): (-0.1, -0.1), (-0.1, 0.1)]) - model.spills += gs.surface_point_line_spill(num_elements=10, + model.spills += gs.point_line_spill(num_elements=10, start_position=(0.0, 0.0), release_time=model.start_time, end_release_time=model.start_time + gs.hours(5), @@ -84,7 +84,7 @@ def test_off_map_removed_MapFromBNA(): # ) - model.spills += gs.surface_point_line_spill(num_elements=10, + model.spills += gs.point_line_spill(num_elements=10, start_position=(-126.2, 48.0), release_time=model.start_time, end_release_time=model.start_time + gs.hours(5), @@ -146,7 +146,7 @@ def test_mass_balance_MapFromBNA(): # [-127.465333 47.44727 ]] # This goes off the map - model.spills += gs.surface_point_line_spill(num_elements=10, + model.spills += gs.point_line_spill(num_elements=10, start_position=(-126.2, 48.2), release_time=model.start_time, end_release_time=model.start_time + gs.hours(5), @@ -154,7 +154,7 @@ def test_mass_balance_MapFromBNA(): ) # this one hits land - model.spills += gs.surface_point_line_spill(num_elements=10, + model.spills += gs.point_line_spill(num_elements=10, start_position=(-127.0, 47.7), release_time=model.start_time, end_release_time=model.start_time + gs.hours(5), diff --git a/py_gnome/tests/unit_tests/test_model_multiproc.py b/py_gnome/tests/unit_tests/test_model_multiproc.py index d9bc8c27e..1a88b9552 100644 --- a/py_gnome/tests/unit_tests/test_model_multiproc.py +++ b/py_gnome/tests/unit_tests/test_model_multiproc.py @@ -19,7 +19,7 @@ from gnome.maps import MapFromBNA from gnome.environment import Wind, Water, Waves, Tide -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.movers import RandomMover, PointWindMover, CatsMover from gnome.weatherers import Evaporation, ChemicalDispersion, Burn, Skimmer @@ -63,7 +63,7 @@ def make_model(uncertain=False, duration=timedelta(hours=48), time_step=3600, map=gnome_map, uncertain=uncertain, cache_enabled=False) - spill = surface_point_line_spill(num_elements=1000, + spill = point_line_spill(num_elements=1000, start_position=(-72.419992, 41.202120, 0.0), release_time=start_time, diff --git a/py_gnome/tests/unit_tests/test_movers/test_simple_mover.py b/py_gnome/tests/unit_tests/test_movers/test_simple_mover.py index d9bbe76f0..9f96de099 100644 --- a/py_gnome/tests/unit_tests/test_movers/test_simple_mover.py +++ b/py_gnome/tests/unit_tests/test_movers/test_simple_mover.py @@ -66,7 +66,7 @@ def test_basic_move_backward(): expected = proj.meters_to_lonlat((-100.0, -1000.0, 0.0), (0.0, 0.0, 0.0)) - assert np.alltrue(delta == expected) + assert np.all(delta == expected) def test_north(): @@ -127,8 +127,7 @@ def test_uncertainty_backward(): # expected = np.zeros_like(delta) - expected = proj.meters_to_lonlat((-1000.0, -1000.0, 0.0), (0.0, 0.0, - 0.0)) + expected = proj.meters_to_lonlat((-1000.0, -1000.0, 0.0), (0.0, 0.0, 0.0)) assert np.all(delta == expected) @@ -139,36 +138,4 @@ def test_uncertainty_backward(): # the mean should be close: # this is the smallest tolerance that consistently passed -- good enough? - assert np.allclose(np.mean(delta, 0), np.mean(u_delta, 0), - rtol=1.7e-1) - - -def test_uncertainty_backward(): - sp = sample_sc_release(num_elements=1000, start_pos=(0.0, 0.0, - 0.0)) - - u_sp = sample_sc_release(num_elements=1000, start_pos=(0.0, 0.0, - 0.0), uncertain=True) - - mover = simple_mover.SimpleMover(velocity=(10.0, 10.0, 0.0)) - - delta = mover.get_move(sp, time_step=-100, model_time=None) - u_delta = mover.get_move(u_sp, time_step=-100, model_time=None) - - # expected = np.zeros_like(delta) - - expected = proj.meters_to_lonlat((-1000.0, -1000.0, 0.0), (0.0, 0.0, - 0.0)) - - assert np.alltrue(delta == expected) - - # but uncertain spills should be different: - - assert not np.alltrue(u_delta == expected) - - # the mean should be close: - - assert np.allclose(np.mean(delta, 0), np.mean(u_delta, 0), - rtol=1.7e-1) - - + assert np.allclose(np.mean(delta, 0), np.mean(u_delta, 0), rtol=1.7e-1) diff --git a/py_gnome/tests/unit_tests/test_movers/test_wind_mover.py b/py_gnome/tests/unit_tests/test_movers/test_wind_mover.py index 03899d018..769905327 100644 --- a/py_gnome/tests/unit_tests/test_movers/test_wind_mover.py +++ b/py_gnome/tests/unit_tests/test_movers/test_wind_mover.py @@ -17,7 +17,7 @@ from gnome.environment import Wind -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.spill_container import SpillContainer from gnome.spills.substance import NonWeatheringSubstance @@ -534,13 +534,12 @@ def test_windage_index(): rel_time = datetime(2013, 1, 1, 0, 0) timestep = 30 for i in range(2): - spill = surface_point_line_spill(num_elements=5, - start_position=(0., 0., 0.), - release_time=rel_time + i * timedelta(hours=1), - substance=NonWeatheringSubstance(windage_range=(i * .01 + - .01, i * .01 + .01), - windage_persist=900) - ) + spill = point_line_spill( + num_elements=5, + start_position=(0., 0., 0.), + release_time=rel_time + i * timedelta(hours=1), + substance=NonWeatheringSubstance(windage_range=(i * .01 + .01, i * .01 + .01), + windage_persist=900)) sc.spills.add(spill) #windage = ['windages', 'windage_range', 'windage_persist'] diff --git a/py_gnome/tests/unit_tests/test_ops/test_density.py b/py_gnome/tests/unit_tests/test_ops/test_density.py index 019e90c11..c9c2ec2ae 100644 --- a/py_gnome/tests/unit_tests/test_ops/test_density.py +++ b/py_gnome/tests/unit_tests/test_ops/test_density.py @@ -42,12 +42,12 @@ def test_recalc_density(): # nw_subs = NonWeatheringSubstance(standard_density=900) # sc.substance = nw_subs # fix it ********** - spill = gs.surface_point_line_spill(num_elements=100, - start_position=(0.0, 0.0, 0.0), - release_time=datetime(2014, 1, 1, 0, 0), - amount=100, - units='bbl', - substance = NonWeatheringSubstance(standard_density=900)) + spill = gs.point_line_spill(num_elements=100, + start_position=(0.0, 0.0, 0.0), + release_time=datetime(2014, 1, 1, 0, 0), + amount=100, + units='bbl', + substance=NonWeatheringSubstance(standard_density=900)) sc.spills.add(spill) # fix it ********** sc.prepare_for_model_run(weathering_array_types) @@ -60,8 +60,8 @@ def test_recalc_density(): default_water = Water() assert sc.mass_balance['avg_density'] == 0 - #Nonweathering density should not get recalculated. - #Aggregation should still occur. + # Nonweathering density should not get recalculated. + # Aggregation should still occur. recalc_density(sc, water=default_water, aggregate=True) assert np.all(sc['density'] == 900) assert sc.mass_balance['avg_density'] == 900 @@ -72,12 +72,12 @@ def test_recalc_density(): # sc.substance = new_subs # fix it ********** sc = SpillContainer() - spill = gs.surface_point_line_spill(num_elements=100, - start_position=(0.0, 0.0, 0.0), - release_time=datetime(2014, 1, 1, 0, 0), - amount=100, - units='bbl', - substance=GnomeOil('oil_crude')) + spill = gs.point_line_spill(num_elements=100, + start_position=(0.0, 0.0, 0.0), + release_time=datetime(2014, 1, 1, 0, 0), + amount=100, + units='bbl', + substance=GnomeOil('oil_crude')) sc.spills.add(spill) # fix it ********** sc.prepare_for_model_run(weathering_array_types) diff --git a/py_gnome/tests/unit_tests/test_ops/test_viscosity.py b/py_gnome/tests/unit_tests/test_ops/test_viscosity.py index e2363b4e6..27cd6e078 100644 --- a/py_gnome/tests/unit_tests/test_ops/test_viscosity.py +++ b/py_gnome/tests/unit_tests/test_ops/test_viscosity.py @@ -35,7 +35,7 @@ def test_recalc_viscosity(): # sc.substance = NonWeatheringSubstance() # fix it ********** sc = SpillContainer() - spill = gs.surface_point_line_spill(num_elements=100, + spill = gs.point_line_spill(num_elements=100, start_position=(0.0, 0.0, 0.0), release_time=datetime(2014, 1, 1, 0, 0), amount=100, @@ -63,7 +63,7 @@ def test_recalc_viscosity(): # sc.substance = new_subs # fix it ********** sc = SpillContainer() - spill = gs.surface_point_line_spill(num_elements=100, + spill = gs.point_line_spill(num_elements=100, start_position=(0.0, 0.0, 0.0), release_time=datetime(2014, 1, 1, 0, 0), amount=100, @@ -87,4 +87,4 @@ def test_recalc_viscosity(): #temp went down so density goes up. assert np.all(sc['viscosity'] > init_kv) assert sc.mass_balance['avg_viscosity'] > init_kv - \ No newline at end of file + diff --git a/py_gnome/tests/unit_tests/test_outputters/test_current_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_current_outputter.py index 523f3031f..c6783a586 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_current_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_current_outputter.py @@ -1,6 +1,3 @@ -''' -tests for geojson outputter -''' from datetime import datetime @@ -10,55 +7,39 @@ # from gnome.basic_types import oil_status from gnome.utilities import time_utils +from gnome.model import Model +from gnome.maps.map import GnomeMap from gnome.environment import Tide -from gnome.spills import Release, Spill, surface_point_line_spill +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import Release from gnome.movers import CatsMover from gnome.outputters.json import CurrentJsonOutput from ..conftest import testdata + td = Tide(filename=testdata['CatsMover']['tide']) c_cats = CatsMover(testdata['CatsMover']['curr'], tide=td) -rel_time = datetime(2012, 9, 15, 12) -time_step = 15 * 60 # seconds -model_time = time_utils.sec_to_date(time_utils.date_to_sec(rel_time)) - - @pytest.fixture(scope='function') -def model(sample_model, output_dir): - model = sample_model['model'] - rel_start_pos = sample_model['release_start_pos'] - rel_end_pos = sample_model['release_end_pos'] - - model.cache_enabled = True - model.uncertain = True - - N = 10 # a line of ten points - line_pos = np.zeros((N, 3), dtype=np.float64) - line_pos[:, 0] = np.linspace(rel_start_pos[0], rel_end_pos[0], N) - line_pos[:, 1] = np.linspace(rel_start_pos[1], rel_end_pos[1], N) - - # print start_points - model.environment += td +def model(output_dir): + """ + a simple model with only the mover / environment object needed + """ + rel_time = datetime(2012, 9, 15, 12) + time_step = 15 * 60 # seconds + # model_time = time_utils.sec_to_date(time_utils.date_to_sec(rel_time)) - model.spills += surface_point_line_spill(1, - start_position=rel_start_pos, - release_time=model.start_time, - end_position=rel_end_pos) + model = Model(start_time=rel_time, + time_step=time_step) - release = Release(custom_positions=line_pos, - release_time=model.start_time) - - model.spills += Spill(release) + model.cache_enabled = True + # model.environment += td model.movers += c_cats - model.outputters += CurrentJsonOutput([c_cats]) - model.rewind() - return model @@ -70,13 +51,8 @@ def test_init(): def test_current_grid_json_output(model): ''' - test geojson outputter with a model since simplest to do that + test geojson outputter with a model since simplest to do that ''' - # default is to round data - model.rewind() - - - for step in model: assert 'step_num' in step assert 'CurrentJsonOutput' in step @@ -87,7 +63,10 @@ def test_current_grid_json_output(model): # We just want to verify here that our keys exist in the movers # collection. for k in list(fcs.keys()): - assert model.movers.index(k) > 0 + try: + model.movers[k] + except KeyError: + assert False, "mover is not in the movers collection" # Check that our structure is correct. for fc in list(fcs.values()): diff --git a/py_gnome/tests/unit_tests/test_outputters/test_erma_data_package.py b/py_gnome/tests/unit_tests/test_outputters/test_erma_data_package.py index 5caabe196..09eada4a9 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_erma_data_package.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_erma_data_package.py @@ -13,7 +13,7 @@ from gnome.outputters import ERMADataPackageOutput import gnome.scripting as gs -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.spill_container import SpillContainerPair # file extension to use for test output files @@ -30,7 +30,7 @@ def model(sample_model_fcn, output_filename): model.cache_enabled = True model.uncertain = True - model.spills += surface_point_line_spill(10, + model.spills += point_line_spill(10, start_position=rel_start_pos, release_time=model.start_time, end_position=rel_end_pos) diff --git a/py_gnome/tests/unit_tests/test_outputters/test_geojson.py b/py_gnome/tests/unit_tests/test_outputters/test_geojson.py index 6dc735a3a..a87bf21ec 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_geojson.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_geojson.py @@ -15,7 +15,8 @@ import pytest from gnome.outputters import TrajectoryGeoJsonOutput -from gnome.spills import PolygonRelease, Spill, surface_point_line_spill +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import PolygonRelease from gnome.basic_types import oil_status from gnome.environment import constant_wind, Water from gnome.weatherers import Evaporation @@ -44,7 +45,7 @@ def model(sample_model_fcn, output_dir): # print start_points - model.spills += surface_point_line_spill(1, + model.spills += point_line_spill(1, start_position=rel_start_pos, release_time=model.start_time, end_position=rel_end_pos, diff --git a/py_gnome/tests/unit_tests/test_outputters/test_ice_image_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_ice_image_outputter.py index c3c26dbd2..7108cd8fd 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_ice_image_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_ice_image_outputter.py @@ -13,7 +13,8 @@ from gnome.utilities import time_utils from gnome.utilities.projections import FlatEarthProjection, GeoProjection -from gnome.spills import PolygonRelease, Spill, surface_point_line_spill +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import PolygonRelease from gnome.movers import IceMover from gnome.outputters import IceImageOutput @@ -45,7 +46,7 @@ def make_model(): # line_pos[:, 1] = np.linspace(rel_start_pos[1], rel_end_pos[1], N) # start_pos = (-164.01696, 72.921024, 0) - # model.spills += surface_point_line_spill(1, + # model.spills += point_line_spill(1, # start_position=start_pos, # release_time=model.start_time, # end_position=start_pos) diff --git a/py_gnome/tests/unit_tests/test_outputters/test_ice_json_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_ice_json_outputter.py index dea7321e7..7116e90e4 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_ice_json_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_ice_json_outputter.py @@ -2,14 +2,16 @@ tests for geojson outputter ''' - import time from datetime import datetime import numpy as np import pytest -from gnome.spills import Release, Spill, surface_point_line_spill +from gnome.model import Model +from gnome.maps.map import GnomeMap +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import Release from gnome.movers import IceMover from gnome.outputters import IceJsonOutput @@ -22,40 +24,20 @@ @pytest.fixture(scope='function') -def model(sample_model, output_dir): - model = sample_model['model'] - rel_start_pos = sample_model['release_start_pos'] - rel_end_pos = sample_model['release_end_pos'] - - model.start_time = datetime(2015, 5, 14, 0) - model.cache_enabled = True - model.uncertain = True - - N = 10 # a line of ten points - line_pos = np.zeros((N, 3), dtype=np.float64) - line_pos[:, 0] = np.linspace(rel_start_pos[0], rel_end_pos[0], N) - line_pos[:, 1] = np.linspace(rel_start_pos[1], rel_end_pos[1], N) - - start_pos = (-164.01696, 72.921024, 0) - model.spills += surface_point_line_spill(1, - start_position=start_pos, - release_time=model.start_time, - end_position=start_pos) - - release = Release(custom_positions=line_pos, - release_time=model.start_time) +def model(output_dir): + """ + A simple model with only the mover / environment object needed + """ + time_step = 15 * 60 # seconds - model.spills += Spill(release) + model = Model(start_time = datetime(2015, 5, 14, 0), + time_step=time_step) model.movers += c_ice_mover - model.outputters += IceJsonOutput([c_ice_mover]) - model.rewind() - return model - def test_init(): 'simple initialization passes' g = IceJsonOutput([c_ice_mover]) @@ -84,7 +66,10 @@ def test_ice_geojson_output(model): # We just want to verify here that our keys exist in the movers # collection. for k in list(fcs.keys()): - assert model.movers.index(k) > 0 + try: + model.movers[k] + except KeyError: + assert False, "mover is not in the movers collection" # Check that our structure is correct. for fc_list in list(fcs.values()): diff --git a/py_gnome/tests/unit_tests/test_outputters/test_ice_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_ice_outputter.py index 85af5573d..4a7fccf39 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_ice_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_ice_outputter.py @@ -11,13 +11,15 @@ # from gnome.basic_types import oil_status from gnome.utilities import time_utils -from gnome.spills import Release, Spill, surface_point_line_spill +from gnome.maps.map import GnomeMap +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import Release from gnome.movers import IceMover from gnome.outputters import IceGeoJsonOutput, IceJsonOutput from ..conftest import testdata -pytest.mark.skip("ice outputter not currently useful -- tests slow") +pytestmark = pytest.mark.skip("ice outputter not currently useful -- tests slow") curr_file = testdata['IceMover']['ice_curr_curv'] @@ -25,9 +27,14 @@ c_ice_mover = IceMover(curr_file, topology_file) +# NOTE: we really don't need a full model here +# unit tests should test as little as possible ... @pytest.fixture(scope='function') def model(sample_model, output_dir): model = sample_model['model'] + + #reset map to water world, so we dont get overlapping issues + model.map = GnomeMap() rel_start_pos = sample_model['release_start_pos'] rel_end_pos = sample_model['release_end_pos'] @@ -41,7 +48,7 @@ def model(sample_model, output_dir): line_pos[:, 1] = np.linspace(rel_start_pos[1], rel_end_pos[1], N) start_pos = (-164.01696, 72.921024, 0) - model.spills += surface_point_line_spill(1, + model.spills += point_line_spill(1, start_position=start_pos, release_time=model.start_time, end_position=start_pos) @@ -60,6 +67,23 @@ def model(sample_model, output_dir): return model +@pytest.fixture(scope='function') +def model(output_dir): + """ + A simple model with only the mover / environment object needed + """ + time_step = 15 * 60 # seconds + + model = Model(start_time = datetime(2015, 5, 14, 0), + time_step=time_step) + + model.movers += c_ice_mover + + model.outputters += IceGeoJsonOutput([c_ice_mover]) + + return model + + def test_init(): 'simple initialization passes' g = IceGeoJsonOutput([c_ice_mover]) @@ -88,7 +112,10 @@ def test_ice_geojson_output(model): # We just want to verify here that our keys exist in the movers # collection. for k in list(fcs.keys()): - assert model.movers.index(k) > 0 + try: + model.movers[k] + except KeyError: + assert False, "mover is not in the movers collection" # Check that our structure is correct. for fc_list in list(fcs.values()): diff --git a/py_gnome/tests/unit_tests/test_outputters/test_kmz.py b/py_gnome/tests/unit_tests/test_outputters/test_kmz.py index 6c251369a..609535319 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_kmz.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_kmz.py @@ -14,7 +14,8 @@ from gnome.outputters import kmz_templates -from gnome.spills import PolygonRelease, Spill, surface_point_line_spill +from gnome.spills.spill import Spill, point_line_spill +from gnome.spills.release import PolygonRelease from gnome.spill_container import SpillContainerPair from gnome.movers import RandomMover, constant_point_wind_mover from gnome.model import Model @@ -41,7 +42,7 @@ def model(sample_model, output_filename): model.cache_enabled = True model.uncertain = True - model.spills += surface_point_line_spill(2, + model.spills += point_line_spill(2, start_position=rel_start_pos, release_time=model.start_time, end_position=rel_end_pos) diff --git a/py_gnome/tests/unit_tests/test_outputters/test_netcdf_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_netcdf_outputter.py index ad3084bd4..870003976 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_netcdf_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_netcdf_outputter.py @@ -14,7 +14,8 @@ import netCDF4 as nc -from gnome.spills import surface_point_line_spill, Spill, Release +from gnome.spills.spill import point_line_spill, Spill +from gnome.spills.release import Release from gnome.spill_container import SpillContainerPair from gnome.weatherers import Evaporation from gnome.environment import Water @@ -38,7 +39,7 @@ def model(sample_model_fcn, output_filename): model.cache_enabled = True model.spills += \ - surface_point_line_spill(num_elements=5, + point_line_spill(num_elements=5, start_position=sample_model_fcn['release_start_pos'], release_time=model.start_time, end_release_time=model.start_time + model.duration, @@ -564,7 +565,7 @@ def test_read_all_arrays(model): # 2nd time around, look at uncertain filename so toggle uncertain flag uncertain = True - +@pytest.mark.filterwarnings("ignore: Outputter output timestep") @pytest.mark.slow @pytest.mark.parametrize("output_ts_factor", [1, 2]) def test_write_output_post_run(model, output_ts_factor): @@ -665,7 +666,7 @@ def test_serialize_deserialize(output_filename): ''' s_time = datetime(2014, 1, 1, 1, 1, 1) model = Model(start_time=s_time) - model.spills += surface_point_line_spill(num_elements=5, + model.spills += point_line_spill(num_elements=5, start_position=(0, 0, 0), release_time=model.start_time) diff --git a/py_gnome/tests/unit_tests/test_outputters/test_oil_budget_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_oil_budget_outputter.py index c0e0c9360..8f83bd2f1 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_oil_budget_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_oil_budget_outputter.py @@ -23,7 +23,7 @@ Skimmer, Burn, ) -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.outputters import OilBudgetOutput @@ -55,7 +55,7 @@ def model(sample_model): model.duration = gs.days(1) end_time = start_time + gs.hours(1) - spill = surface_point_line_spill(100, + spill = point_line_spill(100, start_position=rel_start_pos, release_time=start_time, end_release_time=start_time + gs.hours(1), diff --git a/py_gnome/tests/unit_tests/test_outputters/test_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_outputter.py index 94dc88027..d7d68a1b9 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_outputter.py @@ -7,7 +7,7 @@ import pytest -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.outputters import Outputter import gnome.scripting as gs @@ -23,7 +23,7 @@ def model(sample_model): model.cache_enabled = True - model.spills += surface_point_line_spill( + model.spills += point_line_spill( num_elements=10, start_position=sample_model['release_start_pos'], release_time=model.start_time, @@ -58,8 +58,8 @@ def test_rewind(): params = [(model_ts, item) for item in output_ts] params.extend([(timedelta(hours=6), (timedelta(days=1), 4, 1))]) - - +# this is testing out-of-sync timesteps, so we ignore the warning. +@pytest.mark.filterwarnings("ignore: Outputter output timestep") @pytest.mark.parametrize(("model_ts", "output_ts"), params) def test_output_timestep(model, model_ts, output_ts): """ diff --git a/py_gnome/tests/unit_tests/test_outputters/test_renderer.py b/py_gnome/tests/unit_tests/test_outputters/test_renderer.py index be3b535d8..cd02f7816 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_renderer.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_renderer.py @@ -418,7 +418,7 @@ def test_animation_in_model(output_dir): """ model = gs.Model() model.movers += gs.RandomMover() - model.spills += gs.surface_point_line_spill(num_elements=100, + model.spills += gs.point_line_spill(num_elements=100, start_position=(0, 0), release_time=model.start_time, ) @@ -488,4 +488,4 @@ def test_particle_color_with_depth(output_dir): formats=['png'], depth_colors='inferno') - assert np.alltrue(r1._color_ramp.color_index == r2._color_ramp.color_index) + assert np.all(r1._color_ramp.color_index == r2._color_ramp.color_index) diff --git a/py_gnome/tests/unit_tests/test_outputters/test_shape.py b/py_gnome/tests/unit_tests/test_outputters/test_shape.py index a1e6b70cc..b83734ccd 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_shape.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_shape.py @@ -14,7 +14,7 @@ from gnome.outputters import ShapeOutput import gnome.scripting as gs -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.spill_container import SpillContainerPair @@ -32,7 +32,7 @@ def model(sample_model_fcn, output_filename): model.cache_enabled = True model.uncertain = True - model.spills += surface_point_line_spill(2, + model.spills += point_line_spill(2, start_position=rel_start_pos, release_time=model.start_time, end_position=rel_end_pos) diff --git a/py_gnome/tests/unit_tests/test_outputters/test_weathering_outputter.py b/py_gnome/tests/unit_tests/test_outputters/test_weathering_outputter.py index 1ba1d01e0..ea2bf2e88 100644 --- a/py_gnome/tests/unit_tests/test_outputters/test_weathering_outputter.py +++ b/py_gnome/tests/unit_tests/test_outputters/test_weathering_outputter.py @@ -14,7 +14,7 @@ from gnome.environment import constant_wind, Water, Waves from gnome.weatherers import Evaporation, ChemicalDispersion, Skimmer, Burn -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.outputters import WeatheringOutput @@ -45,7 +45,7 @@ def model(sample_model): model.duration = timedelta(hours=12) end_time = start_time + timedelta(hours=1) - spill = surface_point_line_spill(100, + spill = point_line_spill(100, start_position=rel_start_pos, release_time=start_time, end_release_time=start_time + hours(1), diff --git a/py_gnome/tests/unit_tests/test_persist/test_model_save_load.py b/py_gnome/tests/unit_tests/test_persist/test_model_save_load.py index 2eb093707..a7e200ffa 100644 --- a/py_gnome/tests/unit_tests/test_persist/test_model_save_load.py +++ b/py_gnome/tests/unit_tests/test_persist/test_model_save_load.py @@ -24,7 +24,7 @@ from gnome.maps import MapFromBNA from gnome.environment import Wind, Tide, Water from gnome.model import Model -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.movers import RandomMover, PointWindMover, CatsMover, IceMover from gnome.weatherers import Evaporation, Skimmer, Burn from gnome.outputters import CurrentJsonOutput, IceJsonOutput @@ -57,7 +57,7 @@ def make_model(uncertain=False, mode='gnome'): end_release_time = start_time + timedelta(hours=6) spill_amount = 1000.0 spill_units = 'kg' - model.spills += surface_point_line_spill(num_elements=1000, + model.spills += point_line_spill(num_elements=1000, start_position=start_position, release_time=start_time, end_release_time=end_release_time, diff --git a/py_gnome/tests/unit_tests/test_persist/test_old_savefiles.py b/py_gnome/tests/unit_tests/test_persist/test_old_savefiles.py index 6f9d22855..ebe8668a4 100644 --- a/py_gnome/tests/unit_tests/test_persist/test_old_savefiles.py +++ b/py_gnome/tests/unit_tests/test_persist/test_old_savefiles.py @@ -18,6 +18,8 @@ save_files = SAMPLE_DATA_DIR.glob("*.gnome") +pytestmark = pytest.mark.filterwarnings("ignore:Provided map bounds superscede map bounds found in file") + @pytest.mark.parametrize('savefilename', save_files) def test_old_save_files(savefilename): print("testing loading of:", savefilename) diff --git a/py_gnome/tests/unit_tests/test_persist/test_save_load.py b/py_gnome/tests/unit_tests/test_persist/test_save_load.py index 96d00dcab..a448ced97 100644 --- a/py_gnome/tests/unit_tests/test_persist/test_save_load.py +++ b/py_gnome/tests/unit_tests/test_persist/test_save_load.py @@ -16,7 +16,8 @@ Tide, Water, constant_wind, - GridCurrent) + GridCurrent, + GridWind) from gnome.movers import (constant_point_wind_mover, SimpleMover, @@ -42,7 +43,7 @@ TrajectoryGeoJsonOutput) from gnome.maps import MapFromBNA -from gnome.spills.spill import (surface_point_line_spill, +from gnome.spills.spill import (point_line_spill, PointLineRelease, ) @@ -59,6 +60,8 @@ from testfixtures import LogCapture from ..conftest import test_oil +# reloading maps often triggers this one: +pytestmark = pytest.mark.filterwarnings("ignore:Provided map bounds superscede map bounds found in file") def test_warning_logged(): ''' @@ -83,7 +86,7 @@ def test_class_from_objtype(): def test_exceptions(): - a = 1 + a = 1.1 # need to use something other than an int -- ints are interned. refs = References() refs.reference(a, 'a') refs.reference(a, 'a') # should not do anything @@ -101,7 +104,7 @@ def test_reference_object(): ''' get a reference to an object,then retrieve the object by reference ''' - a = 1 + a = 1.1 refs = References() r1 = refs.reference(a) obj = refs.retrieve(r1) @@ -149,41 +152,40 @@ def test_gnome_obj_reference(): Tide(testdata['CatsMover']['tide']), # Wind(filename=testdata['ComponentMover']['wind']), constant_wind(5., 270, 'knots'), - Wind(timeseries=(sec_to_date(24 * 60 * 60), - (0, 0)), units='mps'), + Wind(timeseries=(sec_to_date(24 * 60 * 60), (0, 0)), units='mps'), Water(temperature=273), - RandomMover(), CatsMover(testdata['CatsMover']['curr']), - CatsMover(testdata['CatsMover']['curr'], - tide=Tide(testdata['CatsMover']['tide'])), + CatsMover(testdata['CatsMover']['curr'], tide=Tide(testdata['CatsMover']['tide'])), ComponentMover(testdata['ComponentMover']['curr']), - ComponentMover(testdata['ComponentMover']['curr'], - wind=constant_wind(5., 270, 'knots')), - # wind=Wind(filename=testdata['ComponentMover']['wind'])), - WindMover(testdata['c_GridWindMover']['wind_rect']), - #WindMover(testdata['c_GridWindMover']['wind_curv']), #variable names wrong - CurrentMover(testdata['c_GridCurrentMover']['curr_tri']), - CurrentMover(testdata['c_GridCurrentMover']['curr_reg']), + ComponentMover(testdata['ComponentMover']['curr'], wind=constant_wind(5., 270, 'knots')), + # wind=Wind(filename=testdata['ComponentMover']['wind'])), + WindMover(GridWind.from_netCDF(testdata['c_GridWindMover']['wind_rect'])), + #WindMover(testdata['c_GridWindMover']['wind_curv']), #variable names wrong + CurrentMover(GridCurrent.from_netCDF(testdata['c_GridCurrentMover']['curr_tri'])), + CurrentMover(GridCurrent.from_netCDF(testdata['c_GridCurrentMover']['curr_reg'])), RandomMover3D(), SimpleMover(velocity=(10.0, 10.0, 0.0)), - MapFromBNA(testdata['MapFromBNA']['testmap'], 6), - NetCDFOutput(os.path.join(base_dir, 'xtemp.nc')), Renderer(testdata['Renderer']['bna_sample'], - os.path.join(base_dir, 'output_dir')), + NetCDFOutput(os.path.join(base_dir, 'xtemp.nc')), + Renderer(testdata['Renderer']['bna_sample'], os.path.join(base_dir, 'output_dir')), WeatheringOutput(), - PointLineRelease(release_time=datetime.now(), - num_elements=10, - start_position=(0, 0, 0)), - surface_point_line_spill(10, (0, 0, 0), datetime.now()), + PointLineRelease(release_time=datetime.now(), num_elements=10, start_position=(0, 0, 0)), + point_line_spill(10, (0, 0, 0), datetime.now()), Substance(windage_range=(0.05, 0.07)), GnomeOil(test_oil, windage_range=(0.05, 0.07)), NonWeatheringSubstance(windage_range=(0.05, 0.07)), - Skimmer(amount=100, efficiency=0.3, active_range=(datetime(2014, 1, 1, 0, 0), datetime(2014, 1, 1, 4, 0)), units='kg'), - Burn(area=100, thickness=1, active_range=(datetime(2014, 1, 1, 0, 0), datetime(2014, 1, 1, 4, 0)), - efficiency=.9), - ChemicalDispersion(fraction_sprayed=.2, active_range=(datetime(2014, 1, 1, 0, 0), datetime(2014, 1, 1, 4, 0)), - efficiency=.3), + Skimmer(amount=100, + efficiency=0.3, + active_range=(datetime(2014, 1, 1, 0, 0), datetime(2014, 1, 1, 4, 0)), + units='kg'), + Burn(area=100, + thickness=1, + active_range=(datetime(2014, 1, 1, 0, 0), datetime(2014, 1, 1, 4, 0)), + efficiency=.9), + ChemicalDispersion(fraction_sprayed=.2, + active_range=(datetime(2014, 1, 1, 0, 0), datetime(2014, 1, 1, 4, 0)), + efficiency=.3), # todo: ask Caitlin how to fix # movers.RiseVelocityMover(), # todo: This is incomplete - no _schema for @@ -192,7 +194,6 @@ def test_gnome_obj_reference(): TrajectoryGeoJsonOutput(), ) - @pytest.mark.parametrize("obj", g_objects) def test_serial_deserial(saveloc_, obj): 'test save/load functionality' @@ -204,7 +205,6 @@ def test_serial_deserial(saveloc_, obj): assert obj == obj2 - @pytest.mark.parametrize("obj", g_objects) def test_save_load_fun(saveloc_, obj): 'test save/load functionality' diff --git a/py_gnome/tests/unit_tests/test_persist/test_schemas.py b/py_gnome/tests/unit_tests/test_persist/test_schemas.py index 664126e15..108a9fe51 100644 --- a/py_gnome/tests/unit_tests/test_persist/test_schemas.py +++ b/py_gnome/tests/unit_tests/test_persist/test_schemas.py @@ -28,9 +28,15 @@ def test_datetimes(): assert dt == dt2 -@pytest.mark.skip("these are known not to work") +# @pytest.mark.skip("these are known not to work") + +@pytest.mark.xfail(reason="cfdatetime is not supported") def test_cfdatetimes(): + """ + test of serializing cfdatetimes + It does not work, but keeping this, in case we decide to support it one day + """ dt = [cfdatetime(2020, 10, 10 + i, 12, 30) for i in range(10)] serial = tss.serialize(dt) @@ -43,5 +49,3 @@ def test_cfdatetimes(): assert dt == dt2 - assert False - diff --git a/py_gnome/tests/unit_tests/test_spills/test_release.py b/py_gnome/tests/unit_tests/test_spills/test_release.py index 7914ac7ce..5fee997e5 100644 --- a/py_gnome/tests/unit_tests/test_spills/test_release.py +++ b/py_gnome/tests/unit_tests/test_spills/test_release.py @@ -14,10 +14,10 @@ import numpy as np from gnome.spills import (Release, - PointLineRelease, - PolygonRelease, - #GridRelease, - ) + PointLineRelease, + PolygonRelease, + #GridRelease, + ) from gnome.spills.release import release_from_splot_data from gnome.spills.le import LEData diff --git a/py_gnome/tests/unit_tests/test_spills/test_release_in_model.py b/py_gnome/tests/unit_tests/test_spills/test_release_in_model.py index e72db4756..103a9f807 100644 --- a/py_gnome/tests/unit_tests/test_spills/test_release_in_model.py +++ b/py_gnome/tests/unit_tests/test_spills/test_release_in_model.py @@ -10,7 +10,7 @@ from datetime import datetime from gnome.model import Model -from gnome.spills.spill import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.outputters.memory_outputter import MemoryOutputter import gnome.scripting as gs @@ -49,7 +49,7 @@ def empty_model(): @pytest.fixture def inst_spill(): - spill = surface_point_line_spill( + spill = point_line_spill( num_elements=NUM_ELEMENTS, start_position=(0, 0, 0), release_time=START_TIME, @@ -64,7 +64,7 @@ def inst_spill(): @pytest.fixture def cont_spill(): - spill = surface_point_line_spill( + spill = point_line_spill( num_elements=NUM_ELEMENTS, start_position=(0, 0, 0), release_time=START_TIME, @@ -176,7 +176,7 @@ def test_one_element_per_timestep(): ) model.outputters += MemoryOutputter() - model.spills += surface_point_line_spill( + model.spills += point_line_spill( num_elements=10, start_position=(0, 0, 0), release_time=model.start_time, @@ -207,7 +207,7 @@ def test_two_elements_per_timestep(): ) model.outputters += MemoryOutputter() - model.spills += surface_point_line_spill( + model.spills += point_line_spill( num_elements=20, start_position=(0, 0, 0), release_time=model.start_time, @@ -240,7 +240,7 @@ def test_fractional_elements_per_timestep(num_per): ) model.outputters += MemoryOutputter() - model.spills += surface_point_line_spill( + model.spills += point_line_spill( num_elements= int(10 * num_per), start_position=(0, 0, 0), release_time=model.start_time, diff --git a/py_gnome/tests/unit_tests/test_spills/test_spatial_release_spreading.py b/py_gnome/tests/unit_tests/test_spills/test_spatial_release_spreading.py index 5aa340fd2..5aa67a0e7 100644 --- a/py_gnome/tests/unit_tests/test_spills/test_spatial_release_spreading.py +++ b/py_gnome/tests/unit_tests/test_spills/test_spatial_release_spreading.py @@ -177,7 +177,7 @@ def test_model_run_integrated(self, amount): model.environment += [constant_wind(1, 0), Water(temperature = 30.+273.15), Waves()] # step 2 ---- add a spill object - spill = gs.surface_point_line_spill(num_elements=1000, + spill = gs.point_line_spill(num_elements=1000, start_position=(0.0, 0.0, 0.0), release_time=datetime.datetime(2000, 1, 1, 1), amount=amount, diff --git a/py_gnome/tests/unit_tests/test_tideflats.py b/py_gnome/tests/unit_tests/test_tideflats.py index c7e444d3f..e83d7dc6c 100644 --- a/py_gnome/tests/unit_tests/test_tideflats.py +++ b/py_gnome/tests/unit_tests/test_tideflats.py @@ -25,7 +25,7 @@ constant_point_wind_mover, point_wind_mover_from_file) -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from .conftest import sample_sc_release, testdata @@ -214,7 +214,7 @@ def test_full_model_run(): model.movers += RandomMover() # defaults are fine - model.spills += surface_point_line_spill(num_elements=10, + model.spills += point_line_spill(num_elements=10, start_position=(0.0, 0.0, 0.0), release_time=start_time, ) diff --git a/py_gnome/tests/unit_tests/test_update_from_dict.py b/py_gnome/tests/unit_tests/test_update_from_dict.py index 441f02f6a..c3366fad9 100644 --- a/py_gnome/tests/unit_tests/test_update_from_dict.py +++ b/py_gnome/tests/unit_tests/test_update_from_dict.py @@ -10,7 +10,7 @@ import numpy as np from gnome.model import Model -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.movers import SimpleMover from gnome.movers import RandomMover @@ -19,16 +19,16 @@ -l_spills = [surface_point_line_spill(10, (0, 0, 0), +l_spills = [point_line_spill(10, (0, 0, 0), datetime.now().replace(microsecond=0), name='sp1'), - surface_point_line_spill(15, (0, 0, 0), + point_line_spill(15, (0, 0, 0), datetime.now().replace(microsecond=0), name='sp2'), - surface_point_line_spill(20, (0, 0, 0), + point_line_spill(20, (0, 0, 0), datetime.now().replace(microsecond=0), name='sp3'), - surface_point_line_spill(5, (0, 0, 0), + point_line_spill(5, (0, 0, 0), datetime.now().replace(microsecond=0), name='sp4')] l_mv = [SimpleMover(velocity=(1, 2, 3)), RandomMover()] diff --git a/py_gnome/tests/unit_tests/test_utilities/test_cache.py b/py_gnome/tests/unit_tests/test_utilities/test_cache.py index 0f8a8690c..4ead4e601 100644 --- a/py_gnome/tests/unit_tests/test_utilities/test_cache.py +++ b/py_gnome/tests/unit_tests/test_utilities/test_cache.py @@ -26,14 +26,28 @@ tdelta = timedelta(hours=1) + # unraisableexception.py:80: PytestUnraisableExceptionWarning: Exception ignored in: + + # Traceback (most recent call last): + # File "/Users/chris.barker/miniforge3/envs/gnome-dev/lib/python3.10/zipfile.py", line 1833, in __del__ + # self.close() + # File "/Users/chris.barker/miniforge3/envs/gnome-dev/lib/python3.10/zipfile.py", line 1850, in close + # self.fp.seek(self.start_dir) + # File "/Users/chris.barker/miniforge3/envs/gnome-dev/lib/python3.10/tempfile.py", line 795, in seek + # return self._file.seek(*args) + # ValueError: I/O operation on closed file. +# only happens when run alongside other utilities tests ... +@pytest.mark.skip("this is raising mysterious errors") def test_init(): """ can we even create one? """ c = cache.ElementCache() + print(c._cache_dir) + print(cache.global_cache_dir) assert True -@pytest.mark.skip("these are intermittently failing -- and we're not using the cache anyway") +# @pytest.mark.skip("these are intermittently failing -- and we're not using the cache anyway") def test_cache_clear_on_delete(): c1 = cache.ElementCache() @@ -55,11 +69,11 @@ def test_cache_clear_on_delete(): del c2 assert not os.path.isdir(d2) -# del c3 -# assert not os.path.isdir(d3) + del c3 + assert not os.path.isdir(d3) -# del c4 -# assert not os.path.isdir(d4) + del c4 + assert not os.path.isdir(d4) def test_write(): diff --git a/py_gnome/tests/unit_tests/test_weatherers/test_cleanup.py b/py_gnome/tests/unit_tests/test_weatherers/test_cleanup.py index b1a816bde..e8a9741d1 100644 --- a/py_gnome/tests/unit_tests/test_weatherers/test_cleanup.py +++ b/py_gnome/tests/unit_tests/test_weatherers/test_cleanup.py @@ -19,7 +19,7 @@ ChemicalDispersion, weatherer_sort) from gnome.spill_container import SpillContainer -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.utilities.inf_datetime import InfDateTime from gnome.environment import Waves, constant_wind, Water @@ -61,7 +61,7 @@ def mk_test_objs(cls, water=None): sc = SpillContainer() print("******************") print("Adding a spill to spill container") - sc.spills += surface_point_line_spill(10, + sc.spills += point_line_spill(10, (0, 0, 0), rel_time, substance=test_oil, diff --git a/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py b/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py index 66712460e..30df95dfe 100644 --- a/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py +++ b/py_gnome/tests/unit_tests/test_weatherers/test_evaporation.py @@ -12,7 +12,7 @@ import numpy as np from gnome.model import Model -from gnome.spills import surface_point_line_spill +from gnome.spills.spill import point_line_spill from gnome.environment import constant_wind, Water, Wind from gnome.weatherers import Evaporation from gnome.outputters import WeatheringOutput @@ -145,7 +145,7 @@ def setup_test(self, end_time_delay, num_les, ts=900.): m1 = Model(start_time=stime, time_step=ts) m1.environment += [constant_wind(0, 0), Water()] m1.weatherers += Evaporation() - m1.spills += surface_point_line_spill(num_les[0], st_pos, stime, + m1.spills += point_line_spill(num_les[0], st_pos, stime, end_release_time=etime, substance=oil, amount=36000, units='kg') @@ -154,7 +154,7 @@ def setup_test(self, end_time_delay, num_les, ts=900.): m2 = Model(start_time=stime, time_step=ts) m2.environment += [constant_wind(0, 0), Water()] m2.weatherers += Evaporation() - m2.spills += surface_point_line_spill(num_les[1], st_pos, stime, + m2.spills += point_line_spill(num_les[1], st_pos, stime, end_release_time=etime, substance=oil, amount=36000, units='kg') diff --git a/py_gnome/tests/unit_tests/test_weatherers/test_spreading.py b/py_gnome/tests/unit_tests/test_weatherers/test_spreading.py index 0915dd46e..4aabc10ef 100644 --- a/py_gnome/tests/unit_tests/test_weatherers/test_spreading.py +++ b/py_gnome/tests/unit_tests/test_weatherers/test_spreading.py @@ -1,6 +1,3 @@ -''' -Test Langmuir() - very simple object with only one method -''' import sys @@ -270,7 +267,7 @@ def test_two_spills(self): time_step=15 * 60, uncertain=False) - spill_0 = gs.surface_point_line_spill(num_elements=16, + spill_0 = gs.point_line_spill(num_elements=16, start_position=(-164.791878561, 69.6252597267, 0.0), release_time=start_time, @@ -279,7 +276,7 @@ def test_two_spills(self): substance=test_oil, units='bbl') - spill_1 = gs.surface_point_line_spill(num_elements=16, + spill_1 = gs.point_line_spill(num_elements=16, start_position=(-164.791878561, 69.6252597267, 0.0), release_time=start_time, @@ -288,7 +285,7 @@ def test_two_spills(self): substance=test_oil, units='bbl') - spill_2 = gs.surface_point_line_spill(num_elements=16, + spill_2 = gs.point_line_spill(num_elements=16, start_position=(-164.791878561, 69.6252597267, 0.0), release_time=start_time, @@ -297,7 +294,7 @@ def test_two_spills(self): substance=test_oil, units='bbl') - spill_3 = gs.surface_point_line_spill(num_elements=16, + spill_3 = gs.point_line_spill(num_elements=16, start_position=(-164.791878561, 69.6252597267, 0.0), release_time=start_time, @@ -354,7 +351,7 @@ def test_constant_area(self, area): time_step=15 * 60, uncertain=False) - spill = gs.surface_point_line_spill(num_elements=num_elements, + spill = gs.point_line_spill(num_elements=num_elements, start_position=(-164.791878561, 69.6252597267, 0.0), release_time=start_time, From ed5eaf2a650a24667002d31629d73a09cc5f72a3 Mon Sep 17 00:00:00 2001 From: Dylan Righi Date: Thu, 25 Jul 2024 20:58:00 +0000 Subject: [PATCH 14/16] Update README.rst --- README.rst | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/README.rst b/README.rst index e25712163..c54e9c656 100644 --- a/README.rst +++ b/README.rst @@ -56,39 +56,6 @@ and it is used in our development and testing process. `Install using Anaconda <./Installing.rst>`_ -Compiling PyGnome -================= -Move to the repository directory and activate the ``gnome`` conda environment with: - - (base) conda activate gnome - -Delete ``py3buildenv_.txt`` if re-compiling code after making software changes. The text file `py3buildenv_.txt` is used to cache software versions to avoid repitious prompts but will cause path problems if changes have been made and needs to be deleted to avoid these erroneous path defs. - -Copy ``python Py3Env.py`` to an untracked file that defines local setup, e.g. ``python Py3Env_RDM.py``, and edit -``places`` to point to the right directory on your local system: - - places = [r"put-your-VisualStudio-vcvars64.bat-directory-path-here"] - -This file can remain untracked in version control because it is unique to each user and easily reproducible from the tracked file ``Py3Env.py`` that will be updated with future advancements. Once this path is corrected to local path, run the script in gnome environment: - - (base) conda activate gnome - - (gnome) python Py3Env_RDM.py - -A command terminal ``cmd.exe`` prompt will open. Change directories and run ``setup.py`` - - (gnome) cd py_gnome - - (gnome) python setup.py develop - -After many, many messages, you will see something like: - - Installed c:\users\rachael.mueller\projects\pygnome\py_gnome - - Processing dependencies for pyGnome==1.1.7 - - Finished processing dependencies for pyGnome==1.1.7 - The WebGNOME Interface: ======================= From 726d7c0a30dbe5e41c9657bfc869a9448d9b6100 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Mon, 29 Jul 2024 16:13:12 -0700 Subject: [PATCH 15/16] switch windows tests to manual -- the Windows Runner is broken --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f46d3d8bd..aff47ee62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,6 +115,7 @@ test_pygnome_scripts: test_pygnome_windows: stage: test + when: manual tags: - windows - visual-studio-22 From d263aeacd0488d85c368e5c0451f2999e25e7865 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Tue, 30 Jul 2024 18:50:20 -0700 Subject: [PATCH 16/16] updated to build -- smaller sdist and dynamic version --- py_gnome/conda_requirements_build.txt | 5 +++-- py_gnome/gnome/__init__.py | 2 +- py_gnome/pyproject.toml | 14 ++++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/py_gnome/conda_requirements_build.txt b/py_gnome/conda_requirements_build.txt index 5dead964a..f7c5dca4f 100644 --- a/py_gnome/conda_requirements_build.txt +++ b/py_gnome/conda_requirements_build.txt @@ -3,9 +3,10 @@ cython=3.* setuptools>=62 # older versions uses different build dirs -setuptools-scm>=6.2 +# setuptools-scm>=6.2 gitpython python-build cmake ninja -scikit-build-core>=0.7.* +scikit-build-core>=0.9.9 +# hatchling diff --git a/py_gnome/gnome/__init__.py b/py_gnome/gnome/__init__.py index f27b6e7e5..5c963fd48 100644 --- a/py_gnome/gnome/__init__.py +++ b/py_gnome/gnome/__init__.py @@ -22,7 +22,7 @@ # just so it will be in the namespace. from .gnomeobject import GnomeId, AddLogger -__version__ = '1.1.12' +__version__ = "1.1.13dev" if os.name == 'nt': diff --git a/py_gnome/pyproject.toml b/py_gnome/pyproject.toml index 61139df6d..385c74214 100644 --- a/py_gnome/pyproject.toml +++ b/py_gnome/pyproject.toml @@ -28,10 +28,14 @@ classifiers = [ "Topic :: Scientific/Engineering :: Oceanography", ] -version = "1.1.12" readme = "README.rst" -# dynamic = ["version", "readme"] +dynamic = ["version"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "gnome/__init__.py" + # really hard to do with conda compatibility :-) # dependencies = [ @@ -54,5 +58,11 @@ namespaces = false # to disable scanning PEP 420 namespaces (true by default) [tool.scikit-build] wheel.py-api = "py3" +# sdist.include = ["tests/unit_tests"] +sdist.exclude = ["scripts", + "tests/", # until we can figure out how to exclude the downloaded data files. + "gnome/cy_gnome/*.cpp" + ] + # [project.optional-dependencies]