From 630f218f6faef9ba6d8a7510c6f45e8071711700 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 15 Apr 2024 14:09:01 +1000 Subject: [PATCH 01/25] chore: Correct requirements.txt --- README.md | 8 ++++++++ requirements.txt | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c83d14b..0a2ebe7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,14 @@ Copy the `.env.example` file to `.env` and customise the paths as you need. ### Installation +To avoid conflicts with other Python projects, it is recommended to create a virtual environment for this project. To do this, run: + + +```console +python -m venv venv +source venv/bin/activate +``` + Install required modules: ```console diff --git a/requirements.txt b/requirements.txt index 0648cf9..0e57943 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ pandas geopandas python-dotenv colorama -samgeo +segment-geospatial cdsapi shapely -bisect From a168298be13b30f16f8207b40a632d693b3051eb Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 15 Apr 2024 14:10:29 +1000 Subject: [PATCH 02/25] chore: Ignore venv --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index dd5392b..4459324 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ experiments .vscode/launch.json .vscode/settings.json domain.ipynb + +# Virtual environment +venv +.venv \ No newline at end of file From db37d8b3c72bf2e70b20e77e7b58583b97b2b110 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Tue, 16 Apr 2024 11:31:54 +1000 Subject: [PATCH 03/25] chore: Add license header config --- .copyright.tmpl | 15 +++++++++++++++ Makefile | 3 +++ __init__.py | 17 +++++++++++++++++ requirements-dev.txt | 1 + 4 files changed, 36 insertions(+) create mode 100644 .copyright.tmpl create mode 100644 Makefile create mode 100644 requirements-dev.txt diff --git a/.copyright.tmpl b/.copyright.tmpl new file mode 100644 index 0000000..d551594 --- /dev/null +++ b/.copyright.tmpl @@ -0,0 +1,15 @@ +Copyright ${years} ${owner}. + +This file is part of ${projectname}. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8565d99 --- /dev/null +++ b/Makefile @@ -0,0 +1,3 @@ +.PHONY: update-licenseheaders +update-licenseheaders: ## add or update license headers in all python files + licenseheaders --current-year --owner "The Superpower Institute Ltd" --projname "OpenMethane" --tmpl .copyright.tmpl --ext .py -x "venv/*" diff --git a/__init__.py b/__init__.py index e69de29..87ba6f5 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..d75fe2b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +licenseheaders \ No newline at end of file From 38519135ea883f5f845dcbafb5a7eb547d67d6f9 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Tue, 16 Apr 2024 11:43:31 +1000 Subject: [PATCH 04/25] chore: Add license header to the files --- omAgLulucfWasteEmis.py | 26 +++++++++++++++++------- omCreateDomainInfo.py | 26 +++++++++++++++++------- omDownloadInputs.py | 26 ++++++++++++++++++------ omElectricityEmis.py | 26 +++++++++++++++++------- omFugitiveEmis.py | 26 +++++++++++++++++------- omGFASEmis.py | 28 ++++++++++++++++++++------ omGeoJSON.py | 26 +++++++++++++++++------- omIndustrialStationaryTransportEmis.py | 26 +++++++++++++++++------- omInputs.py | 17 ++++++++++++++++ omOutputs.py | 17 ++++++++++++++++ omPrior.py | 26 +++++++++++++++++------- omPriorVerify.py | 26 +++++++++++++++++------- omTermiteEmis.py | 26 +++++++++++++++++------- omUtils.py | 26 +++++++++++++++++------- omWetlandEmis.py | 26 +++++++++++++++++------- 15 files changed, 285 insertions(+), 89 deletions(-) diff --git a/omAgLulucfWasteEmis.py b/omAgLulucfWasteEmis.py index 508c7cb..65e6afd 100644 --- a/omAgLulucfWasteEmis.py +++ b/omAgLulucfWasteEmis.py @@ -1,11 +1,23 @@ -""" -omAgLulucfWasteEmis.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Process livestock methane emissions """ import csv diff --git a/omCreateDomainInfo.py b/omCreateDomainInfo.py index 601177e..8ad058c 100644 --- a/omCreateDomainInfo.py +++ b/omCreateDomainInfo.py @@ -1,11 +1,23 @@ -""" -omCreateDomainInfo.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Generate domain file from example domain """ from omInputs import domainPath, geomFilePath, croFilePath, dotFilePath diff --git a/omDownloadInputs.py b/omDownloadInputs.py index 5f8adf4..372c384 100644 --- a/omDownloadInputs.py +++ b/omDownloadInputs.py @@ -1,11 +1,25 @@ +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """ -omDownloadInputs.py +Download required input files -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +This downloads the input files that rarely change and can be cached between runs. """ from omInputs import electricityPath, fugitivesPath, landUsePath, sectoralEmissionsPath, sectoralMappingsPath, ntlPath, auShapefilePath, livestockDataPath, termitePath, wetlandPath, coalPath, oilGasPath diff --git a/omElectricityEmis.py b/omElectricityEmis.py index f4ebe72..b2dc14a 100644 --- a/omElectricityEmis.py +++ b/omElectricityEmis.py @@ -1,11 +1,23 @@ -""" -omEelctricityEmis.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Process emissions from the electricity sector """ import numpy as np diff --git a/omFugitiveEmis.py b/omFugitiveEmis.py index e4f0ce2..9744c73 100644 --- a/omFugitiveEmis.py +++ b/omFugitiveEmis.py @@ -1,11 +1,23 @@ -""" -omFugitiveEmis.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Process fugitive Methane emissions """ import numpy as np diff --git a/omGFASEmis.py b/omGFASEmis.py index ad1b932..6f090b5 100644 --- a/omGFASEmis.py +++ b/omGFASEmis.py @@ -1,11 +1,27 @@ +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """ -omGFASEmis.py +Download and process GFAS data -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +This downloads files from [ADS](https://atmosphere.copernicus.eu/data). +See the project readme for more information about configuring +the required credentials. """ import numpy as np diff --git a/omGeoJSON.py b/omGeoJSON.py index e7fd47e..c1431a3 100644 --- a/omGeoJSON.py +++ b/omGeoJSON.py @@ -1,11 +1,23 @@ -""" -omGeoJSON.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Utilities related to GEOJSON files """ import numpy as np diff --git a/omIndustrialStationaryTransportEmis.py b/omIndustrialStationaryTransportEmis.py index a31b44e..8998d9f 100644 --- a/omIndustrialStationaryTransportEmis.py +++ b/omIndustrialStationaryTransportEmis.py @@ -1,11 +1,23 @@ -""" -omIndustrialStationaryTransportEmis.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Processing industrual stationary transport emissions """ import numpy as np diff --git a/omInputs.py b/omInputs.py index e181fcd..35601e4 100644 --- a/omInputs.py +++ b/omInputs.py @@ -1,3 +1,20 @@ +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# """ omInputs.py diff --git a/omOutputs.py b/omOutputs.py index a29c90c..ec3a6ee 100644 --- a/omOutputs.py +++ b/omOutputs.py @@ -1,3 +1,20 @@ +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# """ omOutputs.py diff --git a/omPrior.py b/omPrior.py index 6c193b7..ffcf4cc 100644 --- a/omPrior.py +++ b/omPrior.py @@ -1,11 +1,23 @@ -""" -omPrior.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Main entry point for running the openmethane-prior """ import argparse diff --git a/omPriorVerify.py b/omPriorVerify.py index bdffd40..ec36480 100644 --- a/omPriorVerify.py +++ b/omPriorVerify.py @@ -1,11 +1,23 @@ -""" -omPriorVerify.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Utilities for verifying the generated output file """ from omInputs import sectoralEmissionsPath, livestockDataPath, domainXr as ds diff --git a/omTermiteEmis.py b/omTermiteEmis.py index b1b8ca8..98ec63f 100644 --- a/omTermiteEmis.py +++ b/omTermiteEmis.py @@ -1,11 +1,23 @@ -""" -omTermiteEmis.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Processing termite Methane emissions """ import numpy as np diff --git a/omUtils.py b/omUtils.py index e66c59c..a696e9c 100644 --- a/omUtils.py +++ b/omUtils.py @@ -1,11 +1,23 @@ -""" -omUtils.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +General utilties """ import numpy as np diff --git a/omWetlandEmis.py b/omWetlandEmis.py index d1270fc..7826665 100644 --- a/omWetlandEmis.py +++ b/omWetlandEmis.py @@ -1,11 +1,23 @@ -""" -omWetlandEmis.py +# +# Copyright 2024 The Superpower Institute Ltd. +# +# This file is part of OpenMethane. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -Copyright 2023 The Superpower Institute Ltd -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. +""" +Processing wetland emissions """ import numpy as np From ea9cf7e8633453884e5875c5c3ebf45260e5ee61 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Mon, 6 May 2024 16:24:25 +0200 Subject: [PATCH 05/25] github setup and CI --- .github/actions/setup/action.yaml | 47 +++++++++++++++++ .github/workflows/bump.yaml | 60 ++++++++++++++++++++++ .github/workflows/ci.yaml | 84 +++++++++++++++++++++++++++++++ .github/workflows/deploy.yaml | 33 ++++++++++++ .github/workflows/install.yaml | 30 +++++++++++ .github/workflows/release.yaml | 47 +++++++++++++++++ .gitignore | 1 + omDownloadInputs.py | 5 +- tests/test_download_data.py | 5 ++ 9 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 .github/actions/setup/action.yaml create mode 100644 .github/workflows/bump.yaml create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/deploy.yaml create mode 100644 .github/workflows/install.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 tests/test_download_data.py diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml new file mode 100644 index 0000000..e0cfdfa --- /dev/null +++ b/.github/actions/setup/action.yaml @@ -0,0 +1,47 @@ +name: 'venv setup' +description: 'configures venv, python and caches' + +inputs: + python-version: + description: "Python version to use" + required: true + cache-dependency-path: + description: "Requirement files to install. Can be a glob pattern." + default: '**/requirements*.txt' + venv-dir: + default: '.venv' + install-cmd: + description: 'Command to install python dependencies.' + +outputs: + cache-hit: + description: "A boolean value to indicate if a cache was restored" + value: ${{ steps.cache-venv.outputs.cache-hit }} + +runs: + using: 'composite' + steps: + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + id: setup-python + with: + python-version: ${{ inputs.python-version }} + + - run: echo '::remove-matcher owner=python::' + shell: bash + + - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + id: cache-venv + with: + path: ${{ inputs.venv-dir }} + key: setup-venv-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ steps.setup-python.outputs.python-path }}-${{ hashFiles(inputs.cache-dependency-path) }}-${{ inputs.install-cmd }} + + - run: python3 -m venv ${{ inputs.venv-dir }} + if: steps.cache-venv.outputs.cache-hit != 'true' + shell: bash + + - run: | + source ${{ inputs.venv-dir }}/bin/activate + python -m pip install -r requirements.txt + echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV + echo "${VIRTUAL_ENV}/bin" >> $GITHUB_PATH + shell: bash diff --git a/.github/workflows/bump.yaml b/.github/workflows/bump.yaml new file mode 100644 index 0000000..84d287a --- /dev/null +++ b/.github/workflows/bump.yaml @@ -0,0 +1,60 @@ +name: Bump version + +on: + workflow_dispatch: + inputs: + bump_rule: + type: choice + description: How to bump the project's version (see https://python-poetry.org/docs/cli/#version) + options: + - patch + - minor + - major + - prepatch + - preminor + - premajor + - prerelease + required: true + +jobs: + bump_version: + name: "Bump version and create changelog" + if: "!startsWith(github.event.head_commit.message, 'bump:')" + runs-on: ubuntu-latest + env: + CI_COMMIT_EMAIL: "ci-runner@primap-visualisation-tool.invalid" + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" + + # towncrier imports the package as part of building so we have to + # install the pacakage (to keep things slim, we only install the main + # dependencies, which also means that we get a test that we can import + # the package with only the compulsory dependencies installed for free) + - uses: ./.github/actions/setup + with: + python-version: "3.11" + venv-id: "bump" + poetry-dependency-install-flags: "--only main" + + - name: Install towncrier + run: | + poetry run pip install towncrier + + - name: Create bump and changelog + + run: | + BASE_VERSION=`poetry version -s` + NEW_VERSION=`poetry version -s ${{ github.event.inputs.bump_rule }}` + poetry run towncrier build --yes --version v$NEW_VERSION + + git config --global user.name "$GITHUB_ACTOR" + git config --global user.email "$CI_COMMIT_EMAIL" + + git commit -a -m "bump: version $BASE_VERSION -> $NEW_VERSION" + git tag v$NEW_VERSION + git push && git push --tags + echo "Bumped to version $NEW_VERSION" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..40fa9ee --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,84 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + tags: ['v*'] + +jobs: + + tests: + strategy: + fail-fast: false + matrix: + os: [ "ubuntu-latest" ] + python-version: [ "3.10" ] + runs-on: "${{ matrix.os }}" + steps: + - name: Check out repository + uses: actions/checkout@v3 + - uses: ./.github/actions/setup + with: + python-version: "${{ matrix.python-version }}" + venv-id: "tests-${{ runner.os }}" + poetry-dependency-install-flags: "--all-extras" + - name: Run tests + run: | + python pytest -r a -v tests --doctest-modules --cov=src --cov-report=term-missing --cov-report=xml + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + + imports-without-extras: + strategy: + fail-fast: false + matrix: + os: [ "ubuntu-latest" ] + python-version: [ "3.10" ] + runs-on: "${{ matrix.os }}" + steps: + - name: Check out repository + uses: actions/checkout@v3 + - uses: ./.github/actions/setup + with: + python-version: "${{ matrix.python-version }}" + venv-id: "imports-without-extras-${{ runner.os }}" + poetry-dependency-install-flags: "--only main" + - name: Check importable without extras + run: poetry run python scripts/test-install.py + + check-build: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + - uses: ./.github/actions/setup + with: + python-version: "3.10" + venv-id: "check-build-${{ runner.os }}" + run-poetry-install: false + poetry-dependency-install-flags: "not used" + - name: Build package + run: | + poetry build --no-interaction + - name: Check build + run: | + tar -tvf dist/primap_visualisation_tool-*.tar.gz --wildcards '*primap_visualisation_tool/py.typed' + + check-dependency-licences: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + - uses: ./.github/actions/setup + with: + python-version: "3.10" + venv-id: "licence-check" + poetry-dependency-install-flags: "--all-extras" + - name: Check licences of dependencies + shell: bash + run: | + TEMP_FILE=$(mktemp) + poetry export --without=tests --without=docs --without=dev > $TEMP_FILE + poetry run liccheck -r $TEMP_FILE -R licence-check.txt + cat licence-check.txt diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..2b8e1aa --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,33 @@ +name: Deploy + +on: + release: + types: [published] + +defaults: + run: + shell: bash + +jobs: + deploy-pypi: + name: Deploy to PyPI + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup + with: + python-version: "3.9" + venv-id: "deploy" + poetry-dependency-install-flags: "--all-extras" + - name: Run tests + run: | + poetry run pytest -r a src tests --doctest-modules + - name: Publish to PyPI + env: + PYPI_TOKEN: "${{ secrets.PYPI_TOKEN }}" + run: | + poetry config pypi-token.pypi $PYPI_TOKEN + poetry publish --build --no-interaction diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml new file mode 100644 index 0000000..785018f --- /dev/null +++ b/.github/workflows/install.yaml @@ -0,0 +1,30 @@ +name: Install + +on: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * 3' + +jobs: + test-pypi-install: + runs-on: "${{ matrix.os }}" + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: [ "3.9", "3.10", "3.11" ] + steps: + - name: Set up Python "${{ matrix.python-version }}" + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + - name: Install + run: | + pip install --upgrade pip + pip install primap-visualisation-tool + - name: Checkout repository + uses: actions/checkout@v3 + - name: Test installation (${{ matrix.os }} Python ${{ matrix.python-version }}) + run: | + python scripts/test-install.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..06efc2c --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,47 @@ +name: Release + +on: + push: + tags: ['v*'] + +defaults: + run: + shell: bash + +jobs: + draft-release: + name: Create draft release + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup + with: + python-version: "3.9" + venv-id: "release" + poetry-dependency-install-flags: "--all-extras" + - name: Add version to environment + run: | + PROJECT_VERSION=$(poetry version --short) + echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV + - name: Run tests + run: | + poetry run pytest -r a -v src tests --doctest-modules + - name: Build package + run: | + poetry build --no-interaction + - name: Generate Release Notes + run: | + git log $(git describe --tags --abbrev=0 HEAD^)..HEAD --pretty='format:* %h %s' --no-merges >> ".github/release_template.md" + - name: Create Release Draft + uses: softprops/action-gh-release@v1 + with: + body_path: ".github/release_template.md" + draft: true + files: | + dist/primap_visualisation_tool-${{ env.PROJECT_VERSION }}-py3-none-any.whl + dist/primap_visualisation_tool-${{ env.PROJECT_VERSION }}.tar.gz + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index 4459324..7f189ce 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ inputs intermediates experiments .env +.idea !outputs/README.md !inputs/README.md diff --git a/omDownloadInputs.py b/omDownloadInputs.py index 372c384..6f20bed 100644 --- a/omDownloadInputs.py +++ b/omDownloadInputs.py @@ -22,7 +22,9 @@ This downloads the input files that rarely change and can be cached between runs. """ -from omInputs import electricityPath, fugitivesPath, landUsePath, sectoralEmissionsPath, sectoralMappingsPath, ntlPath, auShapefilePath, livestockDataPath, termitePath, wetlandPath, coalPath, oilGasPath +from omInputs import (electricityPath, + # fugitivesPath, + landUsePath, sectoralEmissionsPath, sectoralMappingsPath, ntlPath, auShapefilePath, livestockDataPath, termitePath, wetlandPath, coalPath, oilGasPath) import requests import os from omUtils import getenv @@ -41,6 +43,7 @@ wetlandFile = getenv("WETLANDS") coalFile = getenv("COAL") oilGasFile = getenv("OILGAS") + downloads = [ [electricityFile, electricityPath], [coalFile, coalPath], diff --git a/tests/test_download_data.py b/tests/test_download_data.py new file mode 100644 index 0000000..0960152 --- /dev/null +++ b/tests/test_download_data.py @@ -0,0 +1,5 @@ +import pytest + +@pytest.mark.xfail +def test_download_input_data(): + assert False \ No newline at end of file From 7351800114b537111ca6216e9999276168df053f Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Mon, 6 May 2024 16:33:03 +0200 Subject: [PATCH 06/25] reduce CI to tests only --- .github/workflows/ci.yaml | 58 ++------------------------------------- 1 file changed, 2 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 40fa9ee..9930f29 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,60 +25,6 @@ jobs: poetry-dependency-install-flags: "--all-extras" - name: Run tests run: | - python pytest -r a -v tests --doctest-modules --cov=src --cov-report=term-missing --cov-report=xml + pytest -r a -v tests --doctest-modules --cov=src --cov-report=term-missing --cov-report=xml - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - - imports-without-extras: - strategy: - fail-fast: false - matrix: - os: [ "ubuntu-latest" ] - python-version: [ "3.10" ] - runs-on: "${{ matrix.os }}" - steps: - - name: Check out repository - uses: actions/checkout@v3 - - uses: ./.github/actions/setup - with: - python-version: "${{ matrix.python-version }}" - venv-id: "imports-without-extras-${{ runner.os }}" - poetry-dependency-install-flags: "--only main" - - name: Check importable without extras - run: poetry run python scripts/test-install.py - - check-build: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - - uses: ./.github/actions/setup - with: - python-version: "3.10" - venv-id: "check-build-${{ runner.os }}" - run-poetry-install: false - poetry-dependency-install-flags: "not used" - - name: Build package - run: | - poetry build --no-interaction - - name: Check build - run: | - tar -tvf dist/primap_visualisation_tool-*.tar.gz --wildcards '*primap_visualisation_tool/py.typed' - - check-dependency-licences: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - - uses: ./.github/actions/setup - with: - python-version: "3.10" - venv-id: "licence-check" - poetry-dependency-install-flags: "--all-extras" - - name: Check licences of dependencies - shell: bash - run: | - TEMP_FILE=$(mktemp) - poetry export --without=tests --without=docs --without=dev > $TEMP_FILE - poetry run liccheck -r $TEMP_FILE -R licence-check.txt - cat licence-check.txt + uses: codecov/codecov-action@v3 \ No newline at end of file From 976c92b64cd4dd0dcdacde8688999b2fce685870 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Mon, 6 May 2024 16:40:52 +0200 Subject: [PATCH 07/25] add pytest installation to action --- .github/actions/setup/action.yaml | 1 + requirements-dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index e0cfdfa..508c048 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -42,6 +42,7 @@ runs: - run: | source ${{ inputs.venv-dir }}/bin/activate python -m pip install -r requirements.txt + python -m pip install -r requirements-dev.txt echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV echo "${VIRTUAL_ENV}/bin" >> $GITHUB_PATH shell: bash diff --git a/requirements-dev.txt b/requirements-dev.txt index ce5bd93..43c9212 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ licenseheaders ruff=0.3.7 +pytest=8.2.0 From a2d0ed492ca08be7f5a6894d4b190b6d3066769f Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Mon, 6 May 2024 16:44:28 +0200 Subject: [PATCH 08/25] equal operator in dev requirements --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 43c9212..a0869c4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ licenseheaders -ruff=0.3.7 -pytest=8.2.0 +ruff==0.3.7 +pytest==8.2.0 From b07e0319b9195eb7025189b142a8e35efb0b4bdb Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Tue, 7 May 2024 09:23:16 +0200 Subject: [PATCH 09/25] pytest command --- .github/workflows/ci.yaml | 2 +- omDownloadInputs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9930f29..946ea8b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,6 +25,6 @@ jobs: poetry-dependency-install-flags: "--all-extras" - name: Run tests run: | - pytest -r a -v tests --doctest-modules --cov=src --cov-report=term-missing --cov-report=xml + pytest -r a -v tests - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 \ No newline at end of file diff --git a/omDownloadInputs.py b/omDownloadInputs.py index 6f20bed..7608ed5 100644 --- a/omDownloadInputs.py +++ b/omDownloadInputs.py @@ -46,8 +46,8 @@ downloads = [ [electricityFile, electricityPath], - [coalFile, coalPath], - [oilGasFile, oilGasPath], +# [coalFile, coalPath], +# [oilGasFile, oilGasPath], [landUseFile, landUsePath], [sectoralEmissionsFile, sectoralEmissionsPath], [sectoralMappingsFile, sectoralMappingsPath], From adad2cee0c6cdb702b568d270018adee0b1c1e88 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Tue, 7 May 2024 11:42:06 +0200 Subject: [PATCH 10/25] test for url response --- omDownloads.py | 0 temporary_file_for_tests.py | 51 +++++++++++++++++++++++++++++++++++++ tests/test_download_data.py | 16 ++++++++++-- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 omDownloads.py create mode 100644 temporary_file_for_tests.py diff --git a/omDownloads.py b/omDownloads.py new file mode 100644 index 0000000..e69de29 diff --git a/temporary_file_for_tests.py b/temporary_file_for_tests.py new file mode 100644 index 0000000..f3b13e6 --- /dev/null +++ b/temporary_file_for_tests.py @@ -0,0 +1,51 @@ +import os +import dotenv + +dotenv.load_dotenv() +getenv = os.environ.get + +remote = getenv("PRIOR_REMOTE") + +electricityFile = getenv("CH4_ELECTRICITY") +fugitivesFile = getenv("CH4_FUGITIVES") +landUseFile = getenv("LAND_USE") +sectoralEmissionsFile = getenv("SECTORAL_EMISSIONS") +sectoralMappingsFile = getenv("SECTORAL_MAPPING") +ntlFile = getenv("NTL") +auShapefileFile = getenv("AUSF") +livestockDataFile = getenv("LIVESTOCK_DATA") +termiteFile = getenv("TERMITES") +wetlandFile = getenv("WETLANDS") +coalFile = getenv("COAL") +oilGasFile = getenv("OILGAS") + +inputsPath = getenv("INPUTS") +cmaqExamplePath = getenv("CMAQ_EXAMPLE") +climateTracePath = os.path.join( inputsPath, getenv("CLIMATETRACE")) +fossilPath = os.path.join( climateTracePath, getenv("FOSSIL")) + +electricityPath = os.path.join(inputsPath, getenv("CH4_ELECTRICITY")) +oilGasPath = os.path.join( fossilPath, getenv("OILGAS")) +coalPath = os.path.join( fossilPath, getenv("COAL")) +landUsePath = os.path.join(inputsPath, getenv("LAND_USE")) +sectoralEmissionsPath = os.path.join(inputsPath, getenv("SECTORAL_EMISSIONS")) +sectoralMappingsPath = os.path.join(inputsPath, getenv("SECTORAL_MAPPING")) +ntlPath = os.path.join(inputsPath, getenv("NTL")) +auShapefilePath = os.path.join(inputsPath, getenv("AUSF")) +livestockDataPath = os.path.join(inputsPath, getenv("LIVESTOCK_DATA")) +termitePath = os.path.join(inputsPath, getenv("TERMITES")) +wetlandPath = os.path.join(inputsPath, getenv("WETLANDS")) + +downloads = [ + [electricityFile, electricityPath], + [coalFile, coalPath], + [oilGasFile, oilGasPath], + [landUseFile, landUsePath], + [sectoralEmissionsFile, sectoralEmissionsPath], + [sectoralMappingsFile, sectoralMappingsPath], + [ntlFile, ntlPath], + [auShapefileFile, auShapefilePath], + [livestockDataFile, livestockDataPath], + [termiteFile, termitePath], + [wetlandFile, wetlandPath] +] \ No newline at end of file diff --git a/tests/test_download_data.py b/tests/test_download_data.py index 0960152..a0d283e 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -1,5 +1,17 @@ +# work around until folder structure is updated^ +import os +import sys +# insert root directory into python module search path +sys.path.insert(1, os.getcwd()) + +from temporary_file_for_tests import downloads, remote import pytest +import requests @pytest.mark.xfail -def test_download_input_data(): - assert False \ No newline at end of file +def test_response_for_download_links() : + for filename, filepath in downloads : + url = f"{remote}{filename}" + with requests.get(url, stream=True) as response : + print(f"Response code for {url}: {response.status_code}") + assert response.status_code == 200 From 30420e802b96f6ffbefe51eeff521614f7a11457 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Tue, 7 May 2024 14:18:49 +0200 Subject: [PATCH 11/25] .env for github action --- .github/actions/setup/action.yaml | 1 + .github/workflows/ci.yaml | 2 ++ omDownloadInputs.py | 4 ++-- omDownloads.py | 0 tests/test_download_data.py | 20 ++++++++++++++++++++ 5 files changed, 25 insertions(+), 2 deletions(-) delete mode 100644 omDownloads.py diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index 508c048..34a93ab 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -45,4 +45,5 @@ runs: python -m pip install -r requirements-dev.txt echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV echo "${VIRTUAL_ENV}/bin" >> $GITHUB_PATH + cp .env.example .env shell: bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 946ea8b..765424d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,6 +11,8 @@ jobs: tests: strategy: fail-fast: false + env: + INPUTS: inputs matrix: os: [ "ubuntu-latest" ] python-version: [ "3.10" ] diff --git a/omDownloadInputs.py b/omDownloadInputs.py index 7608ed5..6f20bed 100644 --- a/omDownloadInputs.py +++ b/omDownloadInputs.py @@ -46,8 +46,8 @@ downloads = [ [electricityFile, electricityPath], -# [coalFile, coalPath], -# [oilGasFile, oilGasPath], + [coalFile, coalPath], + [oilGasFile, oilGasPath], [landUseFile, landUsePath], [sectoralEmissionsFile, sectoralEmissionsPath], [sectoralMappingsFile, sectoralMappingsPath], diff --git a/omDownloads.py b/omDownloads.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_download_data.py b/tests/test_download_data.py index a0d283e..c99207b 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -15,3 +15,23 @@ def test_response_for_download_links() : with requests.get(url, stream=True) as response : print(f"Response code for {url}: {response.status_code}") assert response.status_code == 200 + +# @pytest.mark.xfail +# def test_download_input_files(): +# assert False + +# EXPECTED_FILE_NAMES = ["file1.txt", "file2.txt", "file3.txt"] +# +# def test_empty_folder(): +# # Check if the folder is empty before running the script +# assert not os.listdir(TEST_FOLDER), f"Folder '{TEST_FOLDER}' is not empty" +# +# def test_execute_script(): +# # Execute the script +# subprocess.run(SCRIPT_COMMAND, shell=True, cwd=TEST_FOLDER) +# +# def test_generated_files(): +# # Check if the generated files have the correct names +# generated_files = os.listdir(TEST_FOLDER) +# assert sorted(generated_files) == sorted(EXPECTED_FILE_NAMES), \ +# f"Generated files {generated_files} do not match expected names {EXPECTED_FILE_NAMES}" \ No newline at end of file From 39f3beafedd2802c57041ddb64ee11b7c53ce6c0 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Tue, 7 May 2024 14:20:08 +0200 Subject: [PATCH 12/25] fix yaml --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 765424d..946ea8b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,8 +11,6 @@ jobs: tests: strategy: fail-fast: false - env: - INPUTS: inputs matrix: os: [ "ubuntu-latest" ] python-version: [ "3.10" ] From 21b4db68c52a10289165692299a51ccb4312c038 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 10:48:01 +0200 Subject: [PATCH 13/25] test for downloads --- tests/test_download_data.py | 58 +++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/tests/test_download_data.py b/tests/test_download_data.py index c99207b..eabedb4 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -7,31 +7,47 @@ from temporary_file_for_tests import downloads, remote import pytest import requests +from pathlib import Path +import subprocess -@pytest.mark.xfail -def test_response_for_download_links() : +def test_001_response_for_download_links() : for filename, filepath in downloads : url = f"{remote}{filename}" with requests.get(url, stream=True) as response : print(f"Response code for {url}: {response.status_code}") assert response.status_code == 200 -# @pytest.mark.xfail -# def test_download_input_files(): -# assert False - -# EXPECTED_FILE_NAMES = ["file1.txt", "file2.txt", "file3.txt"] -# -# def test_empty_folder(): -# # Check if the folder is empty before running the script -# assert not os.listdir(TEST_FOLDER), f"Folder '{TEST_FOLDER}' is not empty" -# -# def test_execute_script(): -# # Execute the script -# subprocess.run(SCRIPT_COMMAND, shell=True, cwd=TEST_FOLDER) -# -# def test_generated_files(): -# # Check if the generated files have the correct names -# generated_files = os.listdir(TEST_FOLDER) -# assert sorted(generated_files) == sorted(EXPECTED_FILE_NAMES), \ -# f"Generated files {generated_files} do not match expected names {EXPECTED_FILE_NAMES}" \ No newline at end of file + +def test_002_omDownloadInputs(): + # Check if the folder is empty before running the script + input_folder = Path(__file__).parent.parent / "inputs" + + EXPECTED_FILES_START = ['README.md'] + EXPECTED_FILES_END = [ + "ch4-electricity.csv", + "coal-mining_emissions-sources.csv", + "oil-and-gas-production-and-transport_emissions-sources.csv", + "NLUM_ALUMV8_250m_2015_16_alb.tif", + "ch4-sectoral-emissions.csv", + "landuse-sector-map.csv", + "nasa-nighttime-lights.tiff", + "AUS_2021_AUST_SHP_GDA2020.zip", + "EntericFermentation.nc", + "termite_emissions_2010-2016.nc", + "DLEM_totflux_CRU_diagnostic.nc", + 'README.md', + ] + + assert os.listdir(input_folder) == EXPECTED_FILES_START, f"Folder '{input_folder}' is not empty" + + subprocess.run(["python", "../omDownloadInputs.py"], capture_output=True, timeout=60*10, check=True) + + downloaded_files = os.listdir(input_folder) + + # clean up folder + for file in [i for i in downloaded_files if i != 'README.md']: + filepath = os.path.join(input_folder, file) + os.remove(filepath) + + assert sorted(downloaded_files) == sorted(EXPECTED_FILES_END) + From 519d3deaf325e384d0db05c785f858ce14ae3700 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 11:26:30 +0200 Subject: [PATCH 14/25] resolve path --- definitions.py | 0 tests/test_download_data.py | 7 +++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 definitions.py diff --git a/definitions.py b/definitions.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_download_data.py b/tests/test_download_data.py index eabedb4..50b1215 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -5,6 +5,7 @@ sys.path.insert(1, os.getcwd()) from temporary_file_for_tests import downloads, remote +from definitions import ROOT_DIRECTORY import pytest import requests from pathlib import Path @@ -20,7 +21,7 @@ def test_001_response_for_download_links() : def test_002_omDownloadInputs(): # Check if the folder is empty before running the script - input_folder = Path(__file__).parent.parent / "inputs" + input_folder = os.path.join(ROOT_DIRECTORY, "inputs") EXPECTED_FILES_START = ['README.md'] EXPECTED_FILES_END = [ @@ -40,7 +41,9 @@ def test_002_omDownloadInputs(): assert os.listdir(input_folder) == EXPECTED_FILES_START, f"Folder '{input_folder}' is not empty" - subprocess.run(["python", "../omDownloadInputs.py"], capture_output=True, timeout=60*10, check=True) + script_path = os.path.join(ROOT_DIRECTORY, "omDownloadInputs.py") + + subprocess.run(["python", script_path], capture_output=True, timeout=60*10, check=True) downloaded_files = os.listdir(input_folder) From 4b095ee62191a60c68c2095d9f26a853ea743bcd Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 11:41:40 +0200 Subject: [PATCH 15/25] resolve paths for github actions --- definitions.py | 0 omDownloadInputs.py | 4 ++++ omInputs.py | 9 +++++++-- tests/test_download_data.py | 7 ++++++- 4 files changed, 17 insertions(+), 3 deletions(-) delete mode 100644 definitions.py diff --git a/definitions.py b/definitions.py deleted file mode 100644 index e69de29..0000000 diff --git a/omDownloadInputs.py b/omDownloadInputs.py index 6f20bed..f77e433 100644 --- a/omDownloadInputs.py +++ b/omDownloadInputs.py @@ -29,6 +29,8 @@ import os from omUtils import getenv +root_path = os.path.dirname(os.path.realpath(__file__)) + remote = getenv("PRIOR_REMOTE") electricityFile = getenv("CH4_ELECTRICITY") @@ -59,6 +61,8 @@ ] for filename, filepath in downloads: + filepath = os.path.join(root_path, filepath) + print(filepath) url = f"{remote}{filename}" if not os.path.exists(filepath): diff --git a/omInputs.py b/omInputs.py index 35601e4..d661563 100644 --- a/omInputs.py +++ b/omInputs.py @@ -37,8 +37,13 @@ domainFilename = getenv("DOMAIN") domainPath = os.path.join(inputsPath, domainFilename) electricityPath = os.path.join(inputsPath, getenv("CH4_ELECTRICITY")) -oilGasPath = os.path.join( fossilPath, getenv("OILGAS")) -coalPath = os.path.join( fossilPath, getenv("COAL")) +# TODO: Changing this to match with the rest of the download file paths. +# The originally specified directory does not exist and stops the download. +# Maybe needs to be changed back later in the process. +# oilGasPath = os.path.join( fossilPath, getenv("OILGAS")) +oilGasPath = os.path.join( inputsPath, getenv("OILGAS")) +# coalPath = os.path.join( fossilPath, getenv("COAL")) +coalPath = os.path.join( inputsPath, getenv("COAL")) landUsePath = os.path.join(inputsPath, getenv("LAND_USE")) sectoralEmissionsPath = os.path.join(inputsPath, getenv("SECTORAL_EMISSIONS")) sectoralMappingsPath = os.path.join(inputsPath, getenv("SECTORAL_MAPPING")) diff --git a/tests/test_download_data.py b/tests/test_download_data.py index 50b1215..7931bcb 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -5,12 +5,17 @@ sys.path.insert(1, os.getcwd()) from temporary_file_for_tests import downloads, remote -from definitions import ROOT_DIRECTORY import pytest import requests from pathlib import Path import subprocess +# Are we in the root directory (for github actions) +if os.path.exists("tests"): + ROOT_DIRECTORY = Path(__file__).parent +else: + ROOT_DIRECTORY = Path(__file__).parent.parent + def test_001_response_for_download_links() : for filename, filepath in downloads : url = f"{remote}{filename}" From bc4b8d063f6544655cf6b2c57ffb13ba74cfa171 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 11:52:02 +0200 Subject: [PATCH 16/25] paths --- tests/test_download_data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_download_data.py b/tests/test_download_data.py index 7931bcb..ddc9e37 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -4,15 +4,18 @@ # insert root directory into python module search path sys.path.insert(1, os.getcwd()) +# At this point there is no file that contains only the definitions needed here +# so I copied it to a separate file. Should be changed after refactoring. from temporary_file_for_tests import downloads, remote import pytest import requests from pathlib import Path import subprocess -# Are we in the root directory (for github actions) +# TODO maybe there's a better way to do this +# Are we in the root directory (github will execute this file from the root directory) if os.path.exists("tests"): - ROOT_DIRECTORY = Path(__file__).parent + ROOT_DIRECTORY = Path(__file__).parent.parent else: ROOT_DIRECTORY = Path(__file__).parent.parent From fa0700486655ccea6d93cb4a63074ca7c95d9109 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 11:55:17 +0200 Subject: [PATCH 17/25] minor changes --- tests/test_download_data.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_download_data.py b/tests/test_download_data.py index ddc9e37..0911790 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -4,20 +4,16 @@ # insert root directory into python module search path sys.path.insert(1, os.getcwd()) -# At this point there is no file that contains only the definitions needed here -# so I copied it to a separate file. Should be changed after refactoring. +# TODO: At this point there is no file that contains only the definitions needed here +# so I copied it into a separate file. Should be changed after refactoring. from temporary_file_for_tests import downloads, remote + import pytest import requests from pathlib import Path import subprocess -# TODO maybe there's a better way to do this -# Are we in the root directory (github will execute this file from the root directory) -if os.path.exists("tests"): - ROOT_DIRECTORY = Path(__file__).parent.parent -else: - ROOT_DIRECTORY = Path(__file__).parent.parent +ROOT_DIRECTORY = Path(__file__).parent.parent def test_001_response_for_download_links() : for filename, filepath in downloads : From 5585d08dc7e1e58222bb76a73dbe450ba0c8b72e Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 13:18:16 +0200 Subject: [PATCH 18/25] minor changes --- tests/test_download_data.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_download_data.py b/tests/test_download_data.py index 0911790..16fcf65 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -1,14 +1,11 @@ -# work around until folder structure is updated^ +# work around until folder structure is updated import os import sys # insert root directory into python module search path sys.path.insert(1, os.getcwd()) -# TODO: At this point there is no file that contains only the definitions needed here -# so I copied it into a separate file. Should be changed after refactoring. from temporary_file_for_tests import downloads, remote -import pytest import requests from pathlib import Path import subprocess @@ -24,7 +21,7 @@ def test_001_response_for_download_links() : def test_002_omDownloadInputs(): - # Check if the folder is empty before running the script + input_folder = os.path.join(ROOT_DIRECTORY, "inputs") EXPECTED_FILES_START = ['README.md'] @@ -51,7 +48,6 @@ def test_002_omDownloadInputs(): downloaded_files = os.listdir(input_folder) - # clean up folder for file in [i for i in downloaded_files if i != 'README.md']: filepath = os.path.join(input_folder, file) os.remove(filepath) From 2257e85cde0902a847681c25045d72e44aad0069 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 14:49:01 +0200 Subject: [PATCH 19/25] subprocess to function --- omDownloadInputs.py | 30 ++++++++++++++++++------------ tests/test_download_data.py | 9 ++++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/omDownloadInputs.py b/omDownloadInputs.py index f77e433..23db7a2 100644 --- a/omDownloadInputs.py +++ b/omDownloadInputs.py @@ -60,17 +60,23 @@ [wetlandFile, wetlandPath] ] -for filename, filepath in downloads: - filepath = os.path.join(root_path, filepath) - print(filepath) - url = f"{remote}{filename}" +def download_input_files(root_path, downloads, remote): + for filename, filepath in downloads: + filepath = os.path.join(root_path, filepath) + print(filepath) + url = f"{remote}{filename}" - if not os.path.exists(filepath): - print(f"Downloading {filename} to {filepath} from {url}") + if not os.path.exists(filepath): + print(f"Downloading {filename} to {filepath} from {url}") - with requests.get(url, stream=True) as response: - with open(filepath, mode="wb") as file: - for chunk in response.iter_content(chunk_size=10 * 1024): - file.write(chunk) - else: - print(f"Skipping {filename} beacuse it already exists at {filepath}") + with requests.get(url, stream=True) as response: + with open(filepath, mode="wb") as file: + for chunk in response.iter_content(chunk_size=10 * 1024): + file.write(chunk) + else: + print(f"Skipping {filename} beacuse it already exists at {filepath}") + +if __name__ == '__main__': + download_input_files(root_path=root_path, + downloads=downloads, + remote=remote) \ No newline at end of file diff --git a/tests/test_download_data.py b/tests/test_download_data.py index 16fcf65..fbe4d40 100644 --- a/tests/test_download_data.py +++ b/tests/test_download_data.py @@ -4,11 +4,10 @@ # insert root directory into python module search path sys.path.insert(1, os.getcwd()) -from temporary_file_for_tests import downloads, remote +from omDownloadInputs import root_path, downloads, remote, download_input_files import requests from pathlib import Path -import subprocess ROOT_DIRECTORY = Path(__file__).parent.parent @@ -42,9 +41,9 @@ def test_002_omDownloadInputs(): assert os.listdir(input_folder) == EXPECTED_FILES_START, f"Folder '{input_folder}' is not empty" - script_path = os.path.join(ROOT_DIRECTORY, "omDownloadInputs.py") - - subprocess.run(["python", script_path], capture_output=True, timeout=60*10, check=True) + download_input_files(root_path=root_path, + downloads=downloads, + remote=remote) downloaded_files = os.listdir(input_folder) From 8e9987f79b3ac48f4f5b58f4e589fc0a889665aa Mon Sep 17 00:00:00 2001 From: crdanielbusch <150670891+crdanielbusch@users.noreply.github.com> Date: Wed, 8 May 2024 14:50:38 +0200 Subject: [PATCH 20/25] Update .github/workflows/ci.yaml Co-authored-by: Jared Lewis --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 946ea8b..51c6df1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,7 +22,6 @@ jobs: with: python-version: "${{ matrix.python-version }}" venv-id: "tests-${{ runner.os }}" - poetry-dependency-install-flags: "--all-extras" - name: Run tests run: | pytest -r a -v tests From f49052c3acf784070ae4631754bc80821954c6f3 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 17:08:17 +0200 Subject: [PATCH 21/25] github action --- .github/actions/setup/action.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index 34a93ab..a707c53 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -10,8 +10,6 @@ inputs: default: '**/requirements*.txt' venv-dir: default: '.venv' - install-cmd: - description: 'Command to install python dependencies.' outputs: cache-hit: From 696ddd4225b2a965e1f573ba1d2790ec0943daf1 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 17:16:15 +0200 Subject: [PATCH 22/25] separate step for copying .env --- .github/actions/setup/action.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index a707c53..944b430 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -19,6 +19,9 @@ outputs: runs: using: 'composite' steps: + - run: cp .env.example .env + shell: bash + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 id: setup-python with: @@ -43,5 +46,5 @@ runs: python -m pip install -r requirements-dev.txt echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV echo "${VIRTUAL_ENV}/bin" >> $GITHUB_PATH - cp .env.example .env + shell: bash From fa1bc654ec0cfb95a1a46d911506f5ba44993d5e Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 17:23:21 +0200 Subject: [PATCH 23/25] remove unused workflows --- .github/workflows/bump.yaml | 60 ---------------------------------- .github/workflows/deploy.yaml | 33 ------------------- .github/workflows/install.yaml | 30 ----------------- .github/workflows/release.yaml | 47 -------------------------- 4 files changed, 170 deletions(-) delete mode 100644 .github/workflows/bump.yaml delete mode 100644 .github/workflows/deploy.yaml delete mode 100644 .github/workflows/install.yaml delete mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/bump.yaml b/.github/workflows/bump.yaml deleted file mode 100644 index 84d287a..0000000 --- a/.github/workflows/bump.yaml +++ /dev/null @@ -1,60 +0,0 @@ -name: Bump version - -on: - workflow_dispatch: - inputs: - bump_rule: - type: choice - description: How to bump the project's version (see https://python-poetry.org/docs/cli/#version) - options: - - patch - - minor - - major - - prepatch - - preminor - - premajor - - prerelease - required: true - -jobs: - bump_version: - name: "Bump version and create changelog" - if: "!startsWith(github.event.head_commit.message, 'bump:')" - runs-on: ubuntu-latest - env: - CI_COMMIT_EMAIL: "ci-runner@primap-visualisation-tool.invalid" - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - token: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" - - # towncrier imports the package as part of building so we have to - # install the pacakage (to keep things slim, we only install the main - # dependencies, which also means that we get a test that we can import - # the package with only the compulsory dependencies installed for free) - - uses: ./.github/actions/setup - with: - python-version: "3.11" - venv-id: "bump" - poetry-dependency-install-flags: "--only main" - - - name: Install towncrier - run: | - poetry run pip install towncrier - - - name: Create bump and changelog - - run: | - BASE_VERSION=`poetry version -s` - NEW_VERSION=`poetry version -s ${{ github.event.inputs.bump_rule }}` - poetry run towncrier build --yes --version v$NEW_VERSION - - git config --global user.name "$GITHUB_ACTOR" - git config --global user.email "$CI_COMMIT_EMAIL" - - git commit -a -m "bump: version $BASE_VERSION -> $NEW_VERSION" - git tag v$NEW_VERSION - git push && git push --tags - echo "Bumped to version $NEW_VERSION" diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml deleted file mode 100644 index 2b8e1aa..0000000 --- a/.github/workflows/deploy.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Deploy - -on: - release: - types: [published] - -defaults: - run: - shell: bash - -jobs: - deploy-pypi: - name: Deploy to PyPI - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: ./.github/actions/setup - with: - python-version: "3.9" - venv-id: "deploy" - poetry-dependency-install-flags: "--all-extras" - - name: Run tests - run: | - poetry run pytest -r a src tests --doctest-modules - - name: Publish to PyPI - env: - PYPI_TOKEN: "${{ secrets.PYPI_TOKEN }}" - run: | - poetry config pypi-token.pypi $PYPI_TOKEN - poetry publish --build --no-interaction diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml deleted file mode 100644 index 785018f..0000000 --- a/.github/workflows/install.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Install - -on: - schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 0 * * 3' - -jobs: - test-pypi-install: - runs-on: "${{ matrix.os }}" - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: [ "3.9", "3.10", "3.11" ] - steps: - - name: Set up Python "${{ matrix.python-version }}" - id: setup-python - uses: actions/setup-python@v4 - with: - python-version: "${{ matrix.python-version }}" - - name: Install - run: | - pip install --upgrade pip - pip install primap-visualisation-tool - - name: Checkout repository - uses: actions/checkout@v3 - - name: Test installation (${{ matrix.os }} Python ${{ matrix.python-version }}) - run: | - python scripts/test-install.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 06efc2c..0000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Release - -on: - push: - tags: ['v*'] - -defaults: - run: - shell: bash - -jobs: - draft-release: - name: Create draft release - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: ./.github/actions/setup - with: - python-version: "3.9" - venv-id: "release" - poetry-dependency-install-flags: "--all-extras" - - name: Add version to environment - run: | - PROJECT_VERSION=$(poetry version --short) - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV - - name: Run tests - run: | - poetry run pytest -r a -v src tests --doctest-modules - - name: Build package - run: | - poetry build --no-interaction - - name: Generate Release Notes - run: | - git log $(git describe --tags --abbrev=0 HEAD^)..HEAD --pretty='format:* %h %s' --no-merges >> ".github/release_template.md" - - name: Create Release Draft - uses: softprops/action-gh-release@v1 - with: - body_path: ".github/release_template.md" - draft: true - files: | - dist/primap_visualisation_tool-${{ env.PROJECT_VERSION }}-py3-none-any.whl - dist/primap_visualisation_tool-${{ env.PROJECT_VERSION }}.tar.gz - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From 4292068a31e12e91f3d2ecb5b20d9f8527653c37 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 17:26:23 +0200 Subject: [PATCH 24/25] fix workflow --- .github/workflows/install.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/install.yaml diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml new file mode 100644 index 0000000..785018f --- /dev/null +++ b/.github/workflows/install.yaml @@ -0,0 +1,30 @@ +name: Install + +on: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * 3' + +jobs: + test-pypi-install: + runs-on: "${{ matrix.os }}" + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + python-version: [ "3.9", "3.10", "3.11" ] + steps: + - name: Set up Python "${{ matrix.python-version }}" + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + - name: Install + run: | + pip install --upgrade pip + pip install primap-visualisation-tool + - name: Checkout repository + uses: actions/checkout@v3 + - name: Test installation (${{ matrix.os }} Python ${{ matrix.python-version }}) + run: | + python scripts/test-install.py From b613de7d7c025191f974cb77b474c558217b0d30 Mon Sep 17 00:00:00 2001 From: Daniel Busch Date: Wed, 8 May 2024 17:38:45 +0200 Subject: [PATCH 25/25] remove install workflow --- .github/workflows/install.yaml | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/install.yaml diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml deleted file mode 100644 index 785018f..0000000 --- a/.github/workflows/install.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Install - -on: - schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 0 * * 3' - -jobs: - test-pypi-install: - runs-on: "${{ matrix.os }}" - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: [ "3.9", "3.10", "3.11" ] - steps: - - name: Set up Python "${{ matrix.python-version }}" - id: setup-python - uses: actions/setup-python@v4 - with: - python-version: "${{ matrix.python-version }}" - - name: Install - run: | - pip install --upgrade pip - pip install primap-visualisation-tool - - name: Checkout repository - uses: actions/checkout@v3 - - name: Test installation (${{ matrix.os }} Python ${{ matrix.python-version }}) - run: | - python scripts/test-install.py