diff --git a/.github/workflows/build_docker.yaml b/.github/workflows/build_docker.yaml index cbf8ce0..fa6d8ae 100644 --- a/.github/workflows/build_docker.yaml +++ b/.github/workflows/build_docker.yaml @@ -90,7 +90,7 @@ jobs: run: | cd /opt/project cp .env.example .env - python -m pytest -r a -v tests/integration/test_domain_json.py + python -m pytest -r a -v tests/integration/test_domain.py env: CDSAPI_KEY: ${{ secrets.CDSAPI_ADS_KEY }} CDSAPI_URL: https://ads.atmosphere.copernicus.eu/api diff --git a/changelog/53.trivial.md b/changelog/53.trivial.md new file mode 100644 index 0000000..8132b9b --- /dev/null +++ b/changelog/53.trivial.md @@ -0,0 +1 @@ +Remove scripts to transform NetCDF files to JSON which are no longer needed \ No newline at end of file diff --git a/scripts/omDomainJSON.py b/scripts/omDomainJSON.py deleted file mode 100644 index f3eec36..0000000 --- a/scripts/omDomainJSON.py +++ /dev/null @@ -1,117 +0,0 @@ -# -# Copyright 2023 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Generate a JSON file describing the domain and grid.""" - -import json - -import numpy as np - -from openmethane_prior.config import PriorConfig, load_config_from_env - - -def _make_point(x, y): - return [float(x), float(y)] - - -def write_domain_json(config: PriorConfig, output_file): - """ - Write a JSON file describing the domain and grid. - - The JSON file contains the following information: - * CRS description - * Grid properties (number of rows, columns, cell size, and center lat/lon) - * List of grid cells, each with the following properties: - * Projection x and y coordinates - * Landmask value - * Center lat/lon - * Corner lat/lon values - - This file is ingested by the frontend and is static for a given domain. - - Parameters - ---------- - output_file - File to read - """ - # Load raster land-use data - print("converting domain grid details to JSON") - - domain_ds = config.domain_dataset() - - domain = { - "crs": { - "projection_type": "lambert_conformal_conic", - "standard_parallel": float(domain_ds.attrs["TRUELAT1"]), - "standard_parallel_2": float(domain_ds.attrs["TRUELAT2"]), - "longitude_of_central_meridian": float(domain_ds.attrs["STAND_LON"]), - "latitude_of_projection_origin": float(domain_ds.attrs["MOAD_CEN_LAT"]), - "projection_origin_x": float(domain_ds.attrs["XORIG"]), - "projection_origin_y": float(domain_ds.attrs["YORIG"]), - "proj4": config.domain_projection().to_proj4(), - }, - "grid_properties": { - "rows": domain_ds.sizes["ROW"], - "cols": domain_ds.sizes["COL"], - "cell_x_size": float(domain_ds.attrs["DX"]), - "cell_y_size": float(domain_ds.attrs["DY"]), - "center_latlon": _make_point(domain_ds.attrs["XCENT"], domain_ds.attrs["YCENT"]), - }, - "grid_cells": [], - } - - if ( - domain_ds.sizes["ROW_D"] != domain_ds.sizes["ROW"] + 1 - or domain_ds.sizes["COL_D"] != domain_ds.sizes["COL"] + 1 - ): - raise RuntimeError("Cell corners dimension must be one greater than number of cells") - - domain_slice = domain_ds.sel(TSTEP=0, LAY=0) - # Add projection coordinates and WGS84 lat/lon for each grid cell - for (y, x), _ in np.ndenumerate(domain_slice["LANDMASK"]): - cell_properties = { - "projection_x_coordinate": int(x), - "projection_y_coordinate": int(y), - "landmask": int(domain_slice["LANDMASK"].item(y, x)), - "center_latlon": _make_point( - domain_slice["LAT"].item(y, x), domain_slice["LON"].item(y, x) - ), - "corner_latlons": [ - _make_point(domain_slice["LATD"].item(y, x), domain_slice["LOND"].item(y, x)), - _make_point( - domain_slice["LATD"].item(y, x + 1), domain_slice["LOND"].item(y, x + 1) - ), - _make_point( - domain_slice["LATD"].item(y + 1, x + 1), domain_slice["LOND"].item(y + 1, x + 1) - ), - _make_point( - domain_slice["LATD"].item(y + 1, x), domain_slice["LOND"].item(y + 1, x) - ), - ], - } - domain["grid_cells"].append(cell_properties) - - json.dump(domain, output_file) - - -if __name__ == "__main__": - config = load_config_from_env() - output_file = config.as_output_file("om-domain.json") - - with open(output_file, "w") as fp: - write_domain_json(config, fp) diff --git a/scripts/omGeoJSON.py b/scripts/omGeoJSON.py deleted file mode 100644 index ca80cd4..0000000 --- a/scripts/omGeoJSON.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Copyright 2023 The Superpower Institute Ltd. -# -# This file is part of OpenMethane. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Utilities related to GEOJSON files""" - -import json - -import netCDF4 as nc -import numpy as np -from geojson import Feature, FeatureCollection, Polygon, dumps - -from openmethane_prior.config import PriorConfig, load_config_from_env -from openmethane_prior.layers import layer_names - -prior_layers = [f"OCH4_{layer.upper()}" for layer in [*layer_names, "total"]] - - -class NumpyEncoder(json.JSONEncoder): - """Numpy encoder for JSON serialization""" - - def default(self, obj): - """Convert numpy arrays to lists""" - if isinstance(obj, np.ndarray): - return obj.tolist() - return json.JSONEncoder.default(self, obj) - - -def processGeoJSON(config: PriorConfig): - """ - Process the gridded prior to produce a GeoJSON file for the frontend. - - Parameters - ---------- - config - Application configuration - """ - geojson_output_path = config.output_path / "om-prior.json" - - """Convert the gridded prior to GeoJSON format""" - print("converting gridded prior to GeoJSON") - - # Load domain - print("Loading output file") - - ds = nc.Dataset(config.output_domain_file, "r") - - # There is a better way to do this but this will work for now - # Using xarray wasn't straightforward because the layers don't use - # the same attribute names, ie mixing x/y and lat/long. - ds_slice = { - "LANDMASK": ds["LANDMASK"][:][0], - "LATD": ds["LATD"][:][0][0], - "LOND": ds["LOND"][:][0][0], - } - - max_values = {} - max_values_float = {} - for layer_name in prior_layers: - # extract the meaningful dimensions from the NetCDF variables - ds_slice[layer_name] = ds[layer_name][:][0][0] - # find the max emission value in a single cell for each layer - max_values[layer_name] = np.max(ds_slice[layer_name]) - max_values_float[layer_name] = float(max_values[layer_name]) - - # Add GeoJSON Polygon feature for each grid location - features = [] - - print("Gathering cell data") - for (y, x), _ in np.ndenumerate(ds_slice["LANDMASK"]): - properties = { - "x": x, - "y": y, - "landmask": int(ds_slice["LANDMASK"][y][x]), - # left for backward compatibility with previous format - "m": float(ds_slice["OCH4_TOTAL"][y][x]), - "rm": float(ds_slice["OCH4_TOTAL"][y][x] / max_values["OCH4_TOTAL"]), - } - for layer_name in prior_layers: - properties[layer_name] = float(ds_slice[layer_name][y][x]) - - features.append( - Feature( - geometry=Polygon( - ( - [ - (float(ds_slice["LOND"][y][x]), float(ds_slice["LATD"][y][x])), - (float(ds_slice["LOND"][y][x + 1]), float(ds_slice["LATD"][y][x + 1])), - ( - float(ds_slice["LOND"][y + 1][x + 1]), - float(ds_slice["LATD"][y + 1][x + 1]), - ), - (float(ds_slice["LOND"][y + 1][x]), float(ds_slice["LATD"][y + 1][x])), - (float(ds_slice["LOND"][y][x]), float(ds_slice["LATD"][y][x])), - ], - ) - ), - properties=properties, - ) - ) - - feature_collection = FeatureCollection(features) - feature_collection.metadata = { - "max_values": max_values_float, - } - - print("Writing output to", geojson_output_path) - with open(geojson_output_path, "w") as fp: - fp.write(dumps(feature_collection)) - - -if __name__ == "__main__": - config = load_config_from_env() - - processGeoJSON(config) diff --git a/tests/integration/test_domain.py b/tests/integration/test_domain.py new file mode 100644 index 0000000..3c2b1f9 --- /dev/null +++ b/tests/integration/test_domain.py @@ -0,0 +1,27 @@ +import numpy + +def test_domain_attributes(input_domain): + # Check domain matches projection used in calculations + assert type(input_domain.attrs["TRUELAT1"]) == numpy.float32 + assert type(input_domain.attrs["TRUELAT2"]) == numpy.float32 + assert type(input_domain.attrs["MOAD_CEN_LAT"]) == numpy.float32 + assert type(input_domain.attrs["STAND_LON"]) == numpy.float32 + assert type(input_domain.attrs["XCENT"]) == numpy.float64 + assert type(input_domain.attrs["YCENT"]) == numpy.float64 + + assert type(input_domain.attrs['DX']) == numpy.float32 + assert input_domain.attrs['DX'] > 0 + assert type(input_domain.attrs['DX']) == numpy.float32 + assert input_domain.attrs['DY'] > 0 + +# Check domain matches projection used in calculations +def test_domain_projection(config, input_domain): + assert str(input_domain.attrs["TRUELAT1"]) in str(config.crs) + assert str(input_domain.attrs["TRUELAT2"]) in str(config.crs) + assert str(input_domain.attrs["MOAD_CEN_LAT"]) in str(config.crs) + assert str(input_domain.attrs["STAND_LON"]) in str(config.crs) + assert str(input_domain.attrs["XCENT"]) in str(config.crs) + # TODO: is this a problem? + # assert str(input_domain.attrs["YCENT"]) in str(config.crs) + + \ No newline at end of file diff --git a/tests/integration/test_domain_json.py b/tests/integration/test_domain_json.py deleted file mode 100644 index 1a2bb3f..0000000 --- a/tests/integration/test_domain_json.py +++ /dev/null @@ -1,53 +0,0 @@ -# work around until folder structure is updated -import json -from io import StringIO - -from scripts.omDomainJSON import write_domain_json - - -def test_001_json_structure(config, input_domain): - outfile = StringIO() - - # generate the JSON, writing to a memory buffer - write_domain_json(config, outfile) - - outfile.seek(0) - domain = json.load(outfile) - - # spot check some known values - assert domain["crs"] == { - "projection_type": "lambert_conformal_conic", - "standard_parallel": -15.0, - "standard_parallel_2": -40.0, - "longitude_of_central_meridian": 133.302001953125, - "latitude_of_projection_origin": -27.643997192382812, - "projection_origin_x": -2270000, - "projection_origin_y": -2165629.25, - "proj4": "+proj=lcc +lat_0=-27.6439971923828 +lon_0=133.302001953125 +lat_1=-15 +lat_2=-40 +x_0=0 +y_0=0 +R=6370000 +units=m +no_defs", # noqa: E501 - } - assert domain["grid_properties"] == { - "rows": 430, - "cols": 454, - "cell_x_size": 10000.0, - "cell_y_size": 10000.0, - "center_latlon": [133.302001953125, -27.5], - } - - # Check the number of cells - assert ( - len(domain["grid_cells"]) - == domain["grid_properties"]["rows"] * domain["grid_properties"]["cols"] - ) - # check a single grid cell for known values - assert domain["grid_cells"][0] == { - "projection_x_coordinate": 0, - "projection_y_coordinate": 0, - "landmask": 0, - "center_latlon": [-44.73386001586914, 105.03723907470703], - "corner_latlons": [ - [-44.76662826538086, 104.96293640136719], - [-44.78663635253906, 105.08344268798828], - [-44.70106506347656, 105.11154174804688], - [-44.681068420410156, 104.99114990234375], - ], - }