Release 2.45 (#116)

* Feature/2.45 bugfix sources (#114)

* [2023-10-31 16:18] Multiple fixes and improvements

FIXED:
 - Harmonie Arome filtering should now work as intended
 - Harmonie Arome now better handles duplicate factor values. (will still need to look into what is going on with those duplicates though)
 - OpenAPI Specification properly is now updated.
 - Several code smells have been removed / erased.

CHANGES:
 - Core API functionality has been upgraded
 - Logging handler has been replaced with loguru
 - Due to issues with the OpenAPI spec's validity, optional WeatherModel fields have been replaced with non-optional equivalents.
Signed-off-by: Raoul Linnenbank <58594297+rflinnenbank@users.noreply.github.com>

* [2023-11-01 08:33] Server setting adjustments

Signed-off-by: Raoul Linnenbank <58594297+rflinnenbank@users.noreply.github.com>

* [2023-11-01 08:53] Docker image update

* [2023-11-01 09:02] Docker image update #2

* [2023-11-01 09:04] Relative import fix

* [2023-11-01 09:16] Docker Image update

Fixes new application location

* Delete openapi.json

Signed-off-by: Raoul Linnenbank <58594297+rflinnenbank@users.noreply.github.com>

---------

Signed-off-by: Raoul Linnenbank <58594297+rflinnenbank@users.noreply.github.com>

Dockerfile

@@ -4,14 +4,14 @@
 #
 
 # *** BASE IMAGE ***
-FROM rflinnenbank/wpla-eccodes-ubuntu:latest as base-image
+FROM rflinnenbank/wpla-eccodes-ubuntu:1.0.0 AS base-image
 
     # Copy the project requirements file to the proper location
 WORKDIR $PYSETUP_PATH
 COPY pyproject.toml ./
 
     # Install the runtime environment dependencies (The $POETRY_VIRTUALENVS_IN_PROJECT value ensures an environment)
-RUN poetry install --no-interaction --no-ansi -vvv --without dev
+RUN poetry install --no-interaction --no-ansi -v --without dev
 
 WORKDIR /
 COPY ./weather_provider_api ./weather_provider_api
@@ -20,29 +20,27 @@ COPY ./pyproject.toml ./pyproject.toml
 
 # *** DEV IMAGE ***
 # The purpose of this image is to supply the project as an interpreter / testing ground
-FROM base-image as dev-image
+FROM base-image AS dev-image
 
     # Set working directory
 WORKDIR /
 
-# TODO: Add SSL interpreter interface
-
 # *** GUNICORN IMAGE ***
 # The purpose of this image is to supply the project with a gunicorn-run API
-FROM base-image as uvicorn-image
+FROM base-image AS uvicorn-image
 
 WORKDIR /
 
 EXPOSE 8000
 
-CMD ["uvicorn", "--reload", "--host", "0.0.0.0", "--port", "8000", "weather_provider_api.main:app" ]
+CMD ["uvicorn", "--reload", "--host", "0.0.0.0", "--port", "8000", "weather_provider_api.core.application:WPLA_APPLICATION" ]
 
 # *** UVICORN IMAGE ***
 # The purpose of this image is to supply the project with a uvicorn-run API
-FROM base-image as gunicorn-image
+FROM base-image AS gunicorn-image
 
 WORKDIR /
 
 EXPOSE 8000
 
-CMD ["gunicorn", "-k", "uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:8000", "weather_provider_api.main:app", "--timeout", "180"]
+CMD ["gunicorn", "-k", "uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:8000", "weather_provider_api.core.application:WPLA_APPLICATION", "--timeout", "180"]

doc/copyright_profiles/velocity_copyright_text.txt

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2019-$today.year Alliander N.V.
+SPDX-License-Identifier: MPL-2.0

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "weather_provider_api"
-version = "2.43"
+version = "2.45.0"
 description = "Weather Provider Libraries and API"
 authors = ["Verbindingsteam", "Raoul Linnenbank <58594297+rflinnenbank@users.noreply.github.com>"]
 license = "MPL-2.0"
@@ -16,7 +16,6 @@ fastapi = "^0.103.1"
 requests = "^2.28.1"
 geopy = "^2.3.0"
 numpy = "^1.24.4"
-structlog = "^23.1.0"
 gunicorn = "^21.2.0"
 eccodes = "^1.5.0"
 accept-types = "^0.4.1"
@@ -32,14 +31,14 @@ uvicorn = "^0.21.1"
 slowapi = "^0.1.7"
 ecmwflibs = "0.5.1"
 starlette = "^0.27.0"
+loguru = "^0.7.2"
+black = "^23.10.1"
+email-validator = "^2.1.0.post1"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.2.0"
 coverage = "^6.5.0"
 pytest-cov = "^4.0.0"
 isort = "^5.10.1"
-black = "^22.10.0"
-jupyter = "^1.0.0"
 pylint = "^2.15.10"
 
 
@@ -57,6 +56,9 @@ wpla_run_api = "weather_provider_api.main:main"
 [tool.pylint]
 max-line-length = 120
 
+[tool.black]
+line-length = 120
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

tests/conftest.py

@@ -122,6 +122,7 @@ def mock_dataset_arome(mock_coordinates, mock_factors):
         )
         for weather_factor in weather_factors
     }
+
     ds = xr.Dataset(
         data_vars=data_dict,
         coords={

tests/pytest.ini

@@ -2,4 +2,4 @@
 ; SPDX-License-Identifier: MPL-2.0
 
 [pytest]
-junit_family=xunit1junit_family=xunit1
+junit_family=xunit1junit_family=xunit1

weather_provider_api/__main__.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#  SPDX-FileCopyrightText: 2019-2023 Alliander N.V.
+#  SPDX-License-Identifier: MPL-2.0
+
+""" Main executable module """
+
+import uvicorn
+from loguru import logger
+
+from weather_provider_api.core.initializers.logging_handler import initialize_logging
+
+# Logging is initialized before the importing of APP_CONFIG, to ensure custom logging for APP_CONFIG initialisation.
+initialize_logging()
+
+# Import application configuration settings
+from weather_provider_api.config import APP_CONFIG
+
+
+def launch_api(run_mode: str = "uvicorn", host: str = "127.0.0.1", port: int = 8080):
+    """The main method for running this application directly.
+
+    (The Dockerfile uses the WPLA_APPLICATION object in [vbd_memo_api.core.application].)
+
+    Args:
+        run_mode (str): The run mode for the application. Accepted values are 'uvicorn' and 'gunicorn'.
+        host (str):     The host id to run this application on. Usually 'localhost', '127.0.0.1' or '0.0.0.0' in this
+                         context.
+        port (str):     The port to broadcast the application at.
+
+    Returns:
+        Nothing. Either runs successfully until stopped, or breaks from an Exception.
+
+    Notes:
+        As Gunicorn only works within the Linux OS, the 'gunicorn' run_mode setting will not work from any other OS.
+
+    """
+    project_title = (
+        APP_CONFIG["base"]["full_title"] if APP_CONFIG["base"]["full_title"] else APP_CONFIG["base"]["title"]
+    )
+    launch_string = f"Launching: {project_title}..."
+    logger.info("-" * len(launch_string))
+    logger.info(launch_string)
+    logger.info("-" * len(launch_string))
+
+    from weather_provider_api.core.application import WPLA_APPLICATION
+
+    # start application based on parameters
+    if run_mode.upper() == "UVICORN":
+        uvicorn.run(WPLA_APPLICATION, host=host, port=port)
+    elif run_mode.upper() == "GUNICORN":
+        # Error handling for problems with the Gunicorn application get handled by Gunicorn itself.
+        from weather_provider_api.core.gunicorn_application import GunicornApplication
+
+        gunicorn_app = GunicornApplication(options={"bind": f"{host}:{port}"}, fastapi_application=WPLA_APPLICATION)
+        gunicorn_app.run()
+    else:
+        raise ValueError(f"Invalid run-mode selected: {run_mode}")
+
+    shutting_down_string = f"Shutting down: {project_title}"
+    logger.info("-" * len(shutting_down_string))
+    logger.info(shutting_down_string)
+    logger.info("-" * len(shutting_down_string))
+
+
+# The main function for easy local execution
+if __name__ == "__main__":
+    # If run from main start using the defaults
+    launch_api()

weather_provider_api/app_version.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#  SPDX-FileCopyrightText: 2019-2023 Alliander N.V.
+#  SPDX-License-Identifier: MPL-2.0
+
+""" Version detection module """
+
+from importlib import metadata
+
+import tomli
+from loguru import logger
+
+
+def _get_app_version() -> str:
+    """This method tries to identify the API's main version identifier and return it.
+
+    Returns:
+        str:    The found version identifier, if found. If no version identifier was found, a warning value is returned
+                 instead.
+
+    """
+    # First attempt: Get the version number by looking for a pyproject.toml file in the working directory.
+    # Please note that this assumes that this function was called from a context that has the Project's main folder as
+    #  the working directory.
+    try:
+        with open("./pyproject.toml", mode="rb") as project_file:
+            version = tomli.load(project_file)["tool"]["poetry"]["version"]
+        logger.info(f"Retrieved the project version from the pyproject.toml file: {version}")
+        return version
+    except FileNotFoundError as fnf_error:
+        logger.debug(f"Could not retrieve the active version from the pyproject.toml file: {fnf_error}")
+
+    # Second attempt: Get the version number from the package that was used to install this component, if applicable.
+    try:
+        version = metadata.version(__package__)
+        logger.info(f"Retrieved the project version from package data: {version}")
+        return version
+    except metadata.PackageNotFoundError as pnf_error:
+        logger.debug(f"Could not retrieve the active version from package data: {pnf_error}")
+
+    # No version could be determined
+    logger.warning("No version could be found for the project!")
+    return "<< version could not be determined >>"
+
+
+APP_VERSION = _get_app_version()

weather_provider_api/config/__init__.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#  SPDX-FileCopyrightText: 2019-2023 Alliander N.V.
+#  SPDX-License-Identifier: MPL-2.0
+
+""" Configuration folder """
+
+import os
+import tempfile
+from pathlib import Path
+
+from tomli import load
+
+config_file_path = Path(__file__).parent.joinpath("config.toml")
+with config_file_path.open(mode="rb") as file_processor:
+    # TODO: Check if config.toml overwrite is set in the environment and replace accordingly!!!
+    APP_CONFIG = load(file_processor)
+
+
+# Settings taken from the environment if available.
+APP_DEBUGGING = os.environ.get("WPLA_DEBUG", "False").lower() in ("true", "1", "y", "yes")
+APP_DEPLOYED = os.environ.get("WPLA_DEPLOYED", "False").lower() in ("true", "1", "y", "yes")
+APP_STORAGE_FOLDER = os.environ.get("WPLA_STORAGE_FOLDER", f"{tempfile.gettempdir()}/Weather_Repository")
+APP_LOG_LEVEL = os.environ.get("WPLA_LOG_LEVEL", APP_CONFIG["logging"]["log_level"]).upper()
+APP_SERVER = os.environ.get("WPLA_SERVER_URL", "http://127.0.0.1:8080")  # 127.0.0.1:8080 for debugging is the default