add devtool utilities

.github/workflows/tests.yml

@@ -0,0 +1,155 @@
+# run test suites
+
+name: Tests
+on:
+  - pull_request
+  - push
+  - release
+  - workflow_dispatch
+
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # see: https://github.com/fkirc/skip-duplicate-actions
+  skip_duplicate:
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip && ! contains(github.ref, 'refs/tags') }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          concurrent_skipping: "same_content"
+          skip_after_successful_duplicate: "true"
+          do_not_skip: '["pull_request", "workflow_dispatch", "schedule", "release"]'
+
+  # see: https://github.com/actions/setup-python
+  tests:
+    needs: skip_duplicate
+    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.allow-failure }}
+    env:
+      # override make command to install directly in active python
+      CONDA_CMD: ""
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10", "3.11", "3.12"]
+        allow-failure: [false]
+        test-case: [test-unit]
+#        include:
+#          # experimental python
+#          - os: ubuntu-latest
+#            python-version: "3.13"
+#            allow-failure: true
+#            test-case: test-unit-only
+#          - os: ubuntu-latest
+#            python-version: "3.13"
+#            allow-failure: true
+#            test-case: test-func-only
+#          # linter tests
+#          - os: ubuntu-latest
+#            python-version: "3.10"
+#            allow-failure: false
+#            test-case: check-all
+#          # documentation build
+#          - os: ubuntu-latest
+#            python-version: "3.10"
+#            allow-failure: false
+#            test-case: docs
+#          # coverage test
+#          - os: ubuntu-latest
+#            python-version: "3.10"
+#            allow-failure: false
+#            test-case: test-coverage-only
+#          # smoke test of Docker image
+#          - os: ubuntu-latest
+#            python-version: "3.10"  # doesn't matter which one (in docker), but match default of repo
+#            allow-failure: false
+#            test-case: test-docker
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Setup Python
+        # skip python setup if running with docker
+        if: ${{ matrix.test-case != 'test-docker' }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: "${{ matrix.python-version }}"
+      - name: Parse Python Version
+        id: python-semver
+        run: |
+          echo "::set-output name=major:$(echo ${{ matrix.python-version }} | cut -d '.' -f 1)"
+          echo "::set-output name=minor:$(echo ${{ matrix.python-version }} | cut -d '.' -f 2)"
+      - uses: actions/cache@v3
+        name: Check Proj Lib Pre-Built in Cache
+        id: cache-proj
+        with:
+          # note: '22' is v8, '21' is v7
+          path: /tmp/proj-8.2.1/install
+          key: ${{ runner.os }}-python${{ matrix.python-version }}-proj
+      - name: Install Dependencies
+        # skip python setup if running with docker
+        if: ${{ matrix.test-case != 'test-docker' }}
+        # install package and dependencies directly,
+        # skip sys/conda setup to use active python
+        run: make install-sys install-pkg install-pip install-raw install-dev version
+      - name: Display Packages
+        # skip python setup if running with docker
+        if: ${{ matrix.test-case != 'test-docker' }}
+        run: pip freeze
+      #- name: Setup Environment Variables
+      #  uses: c-py/action-dotenv-to-setenv@v2
+      #  with:
+      #    env-file: ./ci/weaver.env
+      - name: Display Environment Variables
+        run: |
+          hash -r
+          env | sort
+      - name: Run Tests
+        run: make ${{ matrix.test-case }}
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v2
+        if: ${{ success() && matrix.test-case == 'test-coverage-only' }}
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./reports/coverage.xml
+          fail_ci_if_error: true
+          verbose: true
+
+#  deploy-docker:
+#    needs: tests
+#    if: ${{ success() && (contains(github.ref, 'refs/tags') || github.ref == 'refs/heads/master') }}
+#    runs-on: ubuntu-latest
+#    steps:
+#      - uses: actions/checkout@v2
+#        with:
+#          fetch-depth: "0"
+#      - name: Get Tag Version
+#        id: version
+#        shell: bash
+#        run: |
+#          if [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then
+#            echo "::set-output name=TAG_VERSION::latest"
+#          else
+#            echo "::set-output name=TAG_VERSION::${GITHUB_REF##*/}"
+#          fi
+#      - name: Build Docker
+#        run: |
+#          make DOCKER_REPO=pavics/weaver APP_VERSION=${{ steps.version.outputs.TAG_VERSION }} docker-info docker-build
+#      - name: Login to DockerHub
+#        uses: docker/login-action@v1
+#        with:
+#          username: ${{ secrets.DOCKERHUB_USERNAME }}
+#          password: ${{ secrets.DOCKERHUB_TOKEN }}
+#      - name: Push to DockerHub
+#        run: |
+#          make DOCKER_REPO=pavics/weaver APP_VERSION=${{ steps.version.outputs.TAG_VERSION }} docker-push

.gitignore

@@ -1,3 +1,6 @@
+.coverage
+.pytest_cache
+reports
 *.pyc
 STACpopulator.egg-info/
 .vscode/

CHANGES.md

@@ -0,0 +1,17 @@
+# Changes
+
+## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)
+
+* Add `LICENSE` file.
+* Add `bump-my-version` with `make version` and `make VERSION=<...> bump` utilities to self-update release versions.
+* Add more metadata to `pyproject.toml`.
+* Adjust `README.md` with updated references and release version indicators.
+* Add `CHANGES.md` to record version updates.
+* Add `dev` dependencies to `pyproject.toml` for testing the package (install with `pip install ".[dev]"`).
+* Add GitHub CI tests.
+* Remove `requirements.txt` in favor of all dependencies combined in `pyproject.toml`.
+* Refactor of `CMIP6_UofT` with more robust parsing strategies and STAC Item generation from THREDDS NCML metadata.
+
+## [0.0.1](https://github.com/crim-ca/stac-populator/tree/0.0.1) (2023-08-22)
+
+* Initial release with implementation of `CMIP6_UofT`.

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2013-2014 Computer Research Institute of Montreal (CRIM)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -1,20 +1,64 @@
-IMP_DIR = STACpopulator/implementations
-STAC_HOST = http://localhost:8880/stac
+MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
+# Include custom config if it is available
+-include Makefile.config
+APP_ROOT    := $(abspath $(lastword $(MAKEFILE_NAME))/..)
+APP_NAME    := $(shell basename $(APP_ROOT))
+APP_VERSION ?= 0.0.1
 
-testcmip6:
+
+IMP_DIR := STACpopulator/implementations
+STAC_HOST ?= http://localhost:8880/stac
+
+## -- Testing targets -------------------------------------------------------------------------------------------- ##
+
+test-cmip6:
 	python $(IMP_DIR)/CMIP6_UofT/add_CMIP6.py $(STAC_HOST) https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html
 
-delcmip6:
+del-cmip6:
 	curl --location --request DELETE '$(STAC_HOST)/collections/CMIP6_UofT'
 	@echo ""
 
-starthost:
+docker-start:
 	docker compose up
+starthost: docker-start
 
-stophost:
+docker-stop:
 	docker compose down
+stophost: docker-stop
 
 del_docker_volume: stophost
 	docker volume rm stac-populator_stac-db
 
 resethost: del_docker_volume starthost
+
+install:
+	pip install "$(APP_ROOT)"
+
+install-dev:
+	pip install "$(APP_ROOT)[dev]"
+
+test-unit:
+	pytest "$(APP_ROOT)"
+
+## -- Versioning targets -------------------------------------------------------------------------------------------- ##
+
+# Bumpversion 'dry' config
+# if 'dry' is specified as target, any bumpversion call using 'BUMP_XARGS' will not apply changes
+BUMP_TOOL := bump-my-version
+BUMP_XARGS ?= --verbose --allow-dirty
+ifeq ($(filter dry, $(MAKECMDGOALS)), dry)
+	BUMP_XARGS := $(BUMP_XARGS) --dry-run
+endif
+.PHONY: dry
+dry: pyproject.toml		## run 'bump' target without applying changes (dry-run) [make VERSION=<x.y.z> bump dry]
+	@-echo > /dev/null
+
+.PHONY: bump
+bump:  ## bump version using VERSION specified as user input [make VERSION=<x.y.z> bump]
+	@-echo "Updating package version ..."
+	@[ "${VERSION}" ] || ( echo ">> 'VERSION' is not set"; exit 1 )
+	@-bash -c '$(CONDA_CMD) $(BUMP_TOOL) $(BUMP_XARGS) --new-version "${VERSION}" patch;'
+
+.PHONY: version
+version:	## display current version
+	@-echo "$(APP_NAME) version: $(APP_VERSION)"

README.md

@@ -1,21 +1,38 @@
 # STAC Catalog Populator
 
+![Latest Version](https://img.shields.io/badge/latest%20version-0.0.1-blue?logo=github)
+![Commits Since Latest](https://img.shields.io/github/commits-since/crim-ca/stac-populator/0.0.1.svg?logo=github)
 
-This repository contains a framework [STACpopulator](STACpopulator) that can be used to implement concrete populators (see [implementations](implementations)) for populating the STAC catalog on a DACCS node.
+This repository contains a framework [STACpopulator](STACpopulator)
+that can be used to implement concrete populators (see [implementations](STACpopulator/implementations))
+for populating the STAC Catalog, Collections and Items from various dataset/catalog sources, and pushed using
+STAC API on a server node.
 
 ## Framework
 
-The framwork is centered around a Python Abstract Base Class: `STACpopulatorBase` that implements all the logic for populating a STAC catalog. This class implements an abstract method called `process_STAC_item` that should be defined in implementations of the class and contain all the logic for constructing the STAC representation for an item in the collection that is to be processed.
+The framework is centered around a Python Abstract Base Class: `STACpopulatorBase` that implements all the logic
+for populating a STAC catalog. This class provides abstract methods that should be overridden by implementations that
+contain all the logic for constructing the STAC representation for an item in the collection that is to be processed.
 
 ## Implementations
 
-Currently, one implementation of `STACpopulatorBase` is provided in [add_CMIP6.py](implementations/add_CMIP6.py). 
+Provided implementations of `STACpopulatorBase`:
+
+- [CMIP6_UofT][CMIP6_UofT]
+
+[CMIP6_UofT]: STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py
 
 ## Testing
 
-The provided `docker-compose` file can be used to launch a test STAC server. The `add_CMIP6.py` script can be run as:
+The provided [`docker-compose`](docker-compose.yml) configuration file can be used to launch a test STAC server.
+For example, the [CMIP6_UofT][CMIP6_UofT] script can be run as:
 
+```shell
+python STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py \
+    "http://localhost:8880/stac/" \
+    "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html" \
+    "STACpopulator/implementations/CMIP6_UofT/collection_config.yml"
 ```
-python implementations/CMIP6-UofT/add_CMIP6.py http://localhost:8880/stac/ https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html implementations/CMIP6-UofT/CMIP6.yml
-```
-Note: in the script above, I am currently using a sample THREDDS catalog URL and not one relevant to the global scale CMIP6 data.
+
+*Note*:
+In the script above, a sample THREDDS catalog URL is employed and not one relevant to the global scale CMIP6 data.

STACpopulator/__init__.py

@@ -1 +1 @@
-from .populator_base import STACpopulatorBase
+__version__ = "0.0.1"