Skip to content

Commit

Permalink
add missing dev dependencies + fix more lint checks
Browse files Browse the repository at this point in the history
  • Loading branch information
fmigneault committed Mar 27, 2024
1 parent c11cda7 commit 952dc58
Show file tree
Hide file tree
Showing 8 changed files with 706 additions and 142 deletions.
575 changes: 575 additions & 0 deletions .pylintrc

Large diffs are not rendered by default.

50 changes: 8 additions & 42 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
# Application
APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..)
APP_NAME := $(shell basename $(APP_ROOT))
APP_PKG := pystac_ml_aoi
APP_VERSION ?= 0.1.0

# guess OS (Linux, Darwin,...)
Expand Down Expand Up @@ -162,7 +163,7 @@ fixme-list-only: mkdir-reports ## list all FIXME/TODO/HACK items that require
--score n --persistent n \
--rcfile="$(REPORTS_DIR)/fixmerc" \
-f colorized \
"$(APP_ROOT)/weaver" "$(APP_ROOT)/tests" \
"$(APP_ROOT)/$(APP_PKG)" "$(APP_ROOT)/tests" \
1> >(tee "$(REPORTS_DIR)/fixme.txt")'

.PHONY: fixme-list
Expand Down Expand Up @@ -204,7 +205,6 @@ conda-install:
$(MAKE) -C "$(APP_ROOT)" conda-env \
)


.PHONY: conda-env
conda-env: conda-base conda-config ## create the conda environment
@test -d "$(CONDA_ENV_PATH)" || \
Expand Down Expand Up @@ -242,7 +242,7 @@ install-doc: install-pip ## install documentation dependencies
) || echo "No documentation requirements to install."

.PHONY: install-dev
install-dev: install-run install-npm-stylelint install-npm-remarklint ## install development and test dependencies
install-dev: install-run install-npm-remarklint ## install development and test dependencies
@test -f "$(APP_ROOT)/requirements-dev.txt" && ( \
echo "Installing development packages with pip..." && \
bash -c '$(CONDA_CMD) pip install $(PIP_XARGS) -r "$(APP_ROOT)/requirements-dev.txt"' && \
Expand Down Expand Up @@ -286,13 +286,6 @@ install-npm: ## install npm package manager and dependencies if they cannot b
apt-get install npm \
)

.PHONY: install-npm-stylelint
install-npm-stylelint: install-npm ## install stylelint dependency for 'check-css' target using npm
@[ `npm ls 2>/dev/null | grep stylelint-config-standard | wc -l` = 1 ] || ( \
echo "Install required dependencies for CSS checks." && \
npm install "stylelint@<16" "stylelint-config-standard@<35" --save-dev \
)

.PHONY: install-npm-remarklint
install-npm-remarklint: install-npm ## install remark-lint dependency for 'check-md' target using npm
@[ `npm ls 2>/dev/null | grep remark-lint | wc -l` = 1 ] || ( \
Expand Down Expand Up @@ -438,7 +431,7 @@ CHECKS := $(addprefix check-, $(CHECKS))

# items that should not install python dev packages should be added here instead
# they must provide their own target/only + with dependency install variants
CHECKS_NO_PY := css md
CHECKS_NO_PY := md
CHECKS_NO_PY := $(addprefix check-, $(CHECKS_NO_PY))
CHECKS_ALL := $(CHECKS) $(CHECKS_NO_PY)

Expand Down Expand Up @@ -473,19 +466,16 @@ check-lint-only: mkdir-reports ## check linting of code style
--load-plugins pylint_quotes \
--rcfile="$(APP_ROOT)/.pylintrc" \
--reports y \
"$(APP_ROOT)/weaver" "$(APP_ROOT)/tests" \
"$(APP_ROOT)/$(APP_PKG)" "$(APP_ROOT)/tests" \
1> >(tee "$(REPORTS_DIR)/check-lint.txt")'

.PHONY: check-security-only
check-security-only: check-security-code-only check-security-deps-only ## run security checks

# FIXME: safety ignore file (https://github.com/pyupio/safety/issues/351)
# ignored codes:
# 42194: https://github.com/kvesteri/sqlalchemy-utils/issues/166 # not fixed since 2015
# 42498: celery<5.2.0 bumps kombu>=5.2.1 with security fixes to {redis,sqs} # mongo is used by default in Weaver
# 43738: celery<5.2.2 CVE-2021-23727: trusts the messages and metadata stored in backends
# 45185: pylint<2.13.0: unrelated doc extension (https://github.com/PyCQA/pylint/issues/5322)
SAFETY_IGNORE := 42194 42498 43738 45185
SAFETY_IGNORE := 45185
SAFETY_IGNORE := $(addprefix "-i ",$(SAFETY_IGNORE))

.PHONY: check-security-deps-only
Expand All @@ -497,7 +487,6 @@ check-security-deps-only: mkdir-reports ## run security checks on package depen
--full-report \
-r "$(APP_ROOT)/requirements.txt" \
-r "$(APP_ROOT)/requirements-dev.txt" \
-r "$(APP_ROOT)/requirements-doc.txt" \
-r "$(APP_ROOT)/requirements-sys.txt" \
$(SAFETY_IGNORE) \
1> >(tee "$(REPORTS_DIR)/check-security-deps.txt")'
Expand Down Expand Up @@ -555,7 +544,7 @@ check-docstring-only: mkdir-reports ## check code docstring style and linting
.PHONY: check-links-only
check-links-only: ## check all external links in documentation for integrity
@echo "Running link checks on docs..."
@bash -c '$(CONDA_CMD) $(MAKE) -C "$(APP_ROOT)/docs" linkcheck'
@bash -c '$(CONDA_CMD) $(MAKE) -C "$(APP_ROOT)/docs" linkcheck' || true

.PHONY: check-imports-only
check-imports-only: mkdir-reports ## check imports ordering and styles
Expand All @@ -565,17 +554,6 @@ check-imports-only: mkdir-reports ## check imports ordering and styles
isort --check-only --diff --recursive $(APP_ROOT) \
1> >(tee "$(REPORTS_DIR)/check-imports.txt")'

.PHONY: check-css-only
check-css-only: mkdir-reports ## check CSS linting
@echo "Running CSS style checks..."
@npx --no-install stylelint \
--config "$(APP_ROOT)/.stylelintrc.json" \
--output-file "$(REPORTS_DIR)/check-css.txt" \
"$(APP_ROOT)/**/*.css"

.PHONY: check-css
check-css: install-npm-stylelint check-css-only ## check CSS linting after dependency installation

# must pass 2 search paths because '<dir>/.<subdir>' are somehow not correctly detected with only the top-level <dir>
.PHONY: check-md-only
check-md-only: mkdir-reports ## check Markdown linting
Expand All @@ -597,7 +575,7 @@ FIXES := imports lint docf fstring
FIXES := $(addprefix fix-, $(FIXES))
# items that should not install python dev packages should be added here instead
# they must provide their own target/only + with dependency install variants
FIXES_NO_PY := css md
FIXES_NO_PY := md
FIXES_NO_PY := $(addprefix fix-, $(FIXES_NO_PY))
FIXES_ALL := $(FIXES) $(FIXES_NO_PY)

Expand Down Expand Up @@ -656,18 +634,6 @@ fix-fstring-only: mkdir-reports
flynt $(FLYNT_FLAGS) "$(APP_ROOT)" \
1> >(tee "$(REPORTS_DIR)/fixed-fstring.txt")'

.PHONY: fix-css-only
fix-css-only: mkdir-reports ## fix CSS linting problems automatically
@echo "Fixing CSS style problems..."
@npx stylelint \
--fix \
--config "$(APP_ROOT)/.stylelintrc.json" \
--output-file "$(REPORTS_DIR)/fixed-css.txt" \
"$(APP_ROOT)/**/*.css"

.PHONY: fix-css
fix-css: install-npm-stylelint fix-css-only ## fix CSS linting problems after dependency installation

# must pass 2 search paths because '<dir>/.<subdir>' are somehow not correctly detected with only the top-level <dir>
.PHONY: fix-md-only
fix-md-only: mkdir-reports ## fix Markdown linting problems automatically
Expand Down
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
"remark-lint-maximum-line-length": "^3.1.3",
"remark-message-control": "^8.0.0",
"remark-preset-lint-markdown-style-guide": "^5.1.3",
"remark-preset-lint-recommended": "^6.1.3"
"remark-preset-lint-recommended": "^6.1.3",
"stylelint": "^15.11.0",
"stylelint-config-standard": "^34.0.0"
}
}
83 changes: 46 additions & 37 deletions pystac_ml_aoi/extensions/ml_aoi.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
Utilities to extend :mod:`pystac` objects with STAC ML-AOI extension.
"""
import abc
import os
import json
import os
from datetime import datetime
from typing import (
Any,
Annotated,
Any,
Generic,
Iterable,
List,
Expand All @@ -22,31 +22,24 @@
TypeVar,
Union,
cast,
get_args,
get_args
)
import shapely.geometry.polygon

import pyessv
import pystac
from pydantic import (
AnyHttpUrl,
BaseModel,
ConfigDict,
Field,
FieldValidationInfo,
field_validator,
model_validator,
)
import shapely.geometry.polygon
from pydantic import AnyHttpUrl, BaseModel, ConfigDict, Field, FieldValidationInfo, field_validator, model_validator
from pydantic.fields import FieldInfo
from pystac.extensions import item_assets
from pystac.extensions.base import S, P # generic pystac.STACObject
from pystac.extensions.base import (
from pystac.extensions.base import ( # generic pystac.STACObject
ExtensionManagementMixin,
P,
PropertiesExtension,
SummariesExtension,
S,
SummariesExtension
)
from pystac.extensions.label import LabelRelType
from pystac.extensions.hooks import ExtensionHooks
from pystac.extensions.label import LabelRelType
from pystac.utils import StringEnum

T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition)
Expand Down Expand Up @@ -149,14 +142,17 @@ def add_ml_aoi_prefix(name: str) -> str:


class ML_AOI_BaseFields(BaseModel, validate_assignment=True):
"""ML-AOI base definition to validate fields and properties."""
"""
ML-AOI base definition to validate fields and properties.
"""

model_config = ConfigDict(alias_generator=add_ml_aoi_prefix, populate_by_name=True, extra="ignore")

@model_validator(mode="after")
def validate_one_of(self) -> "ML_AOI_BaseFields":
"""
All fields are optional, but at least one is required.
Additional ``ml-aoi:`` prefixed properties are allowed as well, but no additional validation is performed.
"""
# note:
Expand All @@ -173,13 +169,17 @@ def validate_one_of(self) -> "ML_AOI_BaseFields":


class ML_AOI_ItemProperties(ML_AOI_BaseFields, validate_assignment=True):
"""ML-AOI properties for STAC Items."""
"""
ML-AOI properties for STAC Items.
"""

split: Optional[ML_AOI_Split] # split is required since it is the only available field


class ML_AOI_CollectionFields(ML_AOI_BaseFields, validate_assignment=True):
"""ML-AOI properties for STAC Collections."""
"""
ML-AOI properties for STAC Collections.
"""

split: Optional[List[ML_AOI_Split]] # split is required since it is the only available field

Expand Down Expand Up @@ -273,8 +273,8 @@ def apply(
] = None,
**extra_fields: Any,
) -> None:
"""Applies ML-AOI Extension properties to the extended
:class:`~pystac.Item` or :class:`~pystac.Asset`.
"""
Applies ML-AOI Extension properties to the extended :class:`~pystac.Item` or :class:`~pystac.Asset`.
"""
if not fields:
fields = {}
Expand Down Expand Up @@ -326,14 +326,13 @@ def has_extension(cls, obj: S):

@classmethod
def ext(cls, obj: T, add_if_missing: bool = False) -> "ML_AOI_Extension[T]":
"""Extends the given STAC Object with properties from the
:stac-ext:`ML-AOI Extension <ml-aoi>`.
"""
Extends the given STAC Object with properties from the :stac-ext:`ML-AOI Extension <ml-aoi>`.
This extension can be applied to instances of :class:`~pystac.Item` or
:class:`~pystac.Asset`.
Raises:
pystac.ExtensionTypeError : If an invalid object type is passed.
"""
if isinstance(obj, pystac.Collection):
Expand All @@ -353,7 +352,9 @@ def ext(cls, obj: T, add_if_missing: bool = False) -> "ML_AOI_Extension[T]":

@classmethod
def summaries(cls, obj: pystac.Collection, add_if_missing: bool = False) -> "ML_AOI_SummariesExtension":
"""Returns the extended summaries object for the given collection."""
"""
Returns the extended summaries object for the given collection.
"""
cls.ensure_has_extension(obj, add_if_missing)
return ML_AOI_SummariesExtension(obj)

Expand Down Expand Up @@ -399,9 +400,11 @@ class ML_AOI_ItemExtension(
ML_AOI_PropertiesExtension,
ML_AOI_Extension[pystac.Item],
):
"""A concrete implementation of :class:`ML_AOI_Extension` on an :class:`~pystac.Item`
that extends the properties of the Item to include properties defined in the
:stac-ext:`ML-AOI Extension <ml-aoi>`.
"""
STAC Item extension for ML-AOI.
A concrete implementation of :class:`ML_AOI_Extension` on an :class:`~pystac.Item` that extends the properties of
the Item to include properties defined in the :stac-ext:`ML-AOI Extension <ml-aoi>`.
This class should generally not be instantiated directly. Instead, call
:meth:`ML_AOI_Extension.ext` on an :class:`~pystac.Item` to extend it.
Expand All @@ -422,7 +425,8 @@ def get_assets(
media_type: Optional[Union[pystac.MediaType, str]] = None,
asset_role: Optional[str] = None,
) -> dict[str, pystac.Asset]:
"""Get the item's assets where ``ml-aoi`` fields are matched.
"""
Get the item's assets where ``ml-aoi`` fields are matched.
Args:
role: The ML-AOI role, or a list of roles to filter Assets.
Expand Down Expand Up @@ -484,9 +488,11 @@ class ML_AOI_AssetExtension(
ML_AOI_PropertiesExtension,
ML_AOI_Extension[pystac.Asset],
):
"""A concrete implementation of :class:`ML_AOI_Extension` on an :class:`~pystac.Asset`
that extends the Asset fields to include properties defined in the
:stac-ext:`ML-AOI Extension <ml-aoi>`.
"""
Asset extension for ML-AOI.
A concrete implementation of :class:`ML_AOI_Extension` on an :class:`~pystac.Asset` that extends the Asset fields to
include properties defined in the :stac-ext:`ML-AOI Extension <ml-aoi>`.
This class should generally not be instantiated directly. Instead, call
:meth:`ML_AOI_Extension.ext` on an :class:`~pystac.Asset` to extend it.
Expand Down Expand Up @@ -533,9 +539,11 @@ class ML_AOI_SummariesExtension(
SummariesExtension,
ML_AOI_Extension[pystac.Collection],
):
"""A concrete implementation of :class:`~SummariesExtension` that extends
the ``summaries`` field of a :class:`~pystac.Collection` to include properties
defined in the :stac-ext:`ML-AOI <ml-aoi>`.
"""
Summaries extension for ML-AOI.
A concrete implementation of :class:`~SummariesExtension` that extends the ``summaries`` field of a
:class:`~pystac.Collection` to include properties defined in the :stac-ext:`ML-AOI <ml-aoi>`.
"""
model = ML_AOI_CollectionFields

Expand Down Expand Up @@ -610,7 +618,8 @@ def __repr__(self) -> str:
return f"<ML_AOI_CollectionExtension Collection id={self.collection.id}>"

def set_self_href(self, href: Optional[str]) -> None:
"""Sets the absolute HREF that is represented by the ``rel == 'self'`` :class:`~pystac.Link`.
"""
Sets the absolute HREF that is represented by the ``rel == 'self'`` :class:`~pystac.Link`.
Adds the relevant ML-AOI role applicable for the Collection.
"""
Expand Down
8 changes: 8 additions & 0 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,10 @@
bandit
bump2version
doc8
docformatter
flake8
flynt
isort
pylint
pytest
safety
2 changes: 1 addition & 1 deletion setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ exclude = *.egg-info,./build,./dist,./env,./tests,test_*
targets = .

[flake8]
ignore = E126,E226,E402,F401,W503,W504,B007,B009,B010,B023
ignore = E126,E226,E402,E501,F401,W503,W504,B007,B009,B010,B023
max-line-length = 120
exclude =
src,
Expand Down
Loading

0 comments on commit 952dc58

Please sign in to comment.