Skip to content

Commit

Permalink
Merge pull request #75 from OCHA-DAP/add-hapi-errors
Browse files Browse the repository at this point in the history
Create add_hapi_error function
  • Loading branch information
b-j-mills authored Nov 11, 2024
2 parents fdfb767 + 31574c8 commit c13ff6c
Show file tree
Hide file tree
Showing 7 changed files with 939 additions and 814 deletions.
6 changes: 3 additions & 3 deletions .config/pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ default_language_version:
python: python3.12
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.6.0
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-ast
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.6.4
rev: v0.7.0
hooks:
# Run the linter.
- id: ruff
Expand All @@ -17,7 +17,7 @@ repos:
- id: ruff-format
args: [--config, .config/ruff.toml]
- repo: https://github.com/astral-sh/uv-pre-commit
rev: 0.4.7
rev: 0.4.24
hooks:
# Run the pip compile
- id: pip-compile
Expand Down
9 changes: 6 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ dependencies = [
"ckanapi>=4.8",
"defopt>=6.4.0",
"email_validator",
"hdx-python-country>=3.7.8",
"hdx-python-utilities>=3.7.3",
"libhxl>=5.2.1",
"hdx-python-country>=3.8.3",
"hdx-python-utilities>=3.7.4",
"libhxl>=5.2.2",
"makefun",
"quantulum3",
"requests",
Expand Down Expand Up @@ -94,6 +94,9 @@ run = """
--cov-report=lcov --cov-report=term-missing
"""

[tool.hatch.envs.hatch-static-analysis]
dependencies = ["ruff==0.7.0"]

[tool.hatch.envs.hatch-static-analysis.scripts]
format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
format-fix = ["ruff format --config .config/ruff.toml {args:.}",]
Expand Down
77 changes: 39 additions & 38 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,19 @@ cfgv==3.4.0
# via pre-commit
chardet==5.2.0
# via frictionless
charset-normalizer==3.3.2
charset-normalizer==3.4.0
# via requests
ckanapi==4.8
# via hdx-python-api (pyproject.toml)
click==8.1.7
# via typer
coverage==7.6.1
coverage==7.6.4
# via pytest-cov
defopt==6.4.0
# via hdx-python-api (pyproject.toml)
distlib==0.3.8
distlib==0.3.9
# via virtualenv
dnspython==2.6.1
dnspython==2.7.0
# via email-validator
docopt==0.6.2
# via
Expand All @@ -38,31 +38,31 @@ docutils==0.21.2
# via defopt
email-validator==2.2.0
# via hdx-python-api (pyproject.toml)
et-xmlfile==1.1.0
et-xmlfile==2.0.0
# via openpyxl
filelock==3.16.0
filelock==3.16.1
# via virtualenv
frictionless==5.17.1
frictionless==5.18.0
# via hdx-python-utilities
google-auth==2.34.0
google-auth==2.36.0
# via
# google-auth-oauthlib
# gspread
google-auth-oauthlib==1.2.1
# via gspread
gspread==6.1.2
gspread==6.1.4
# via hdx-python-api (pyproject.toml)
hdx-python-country==3.7.8
hdx-python-country==3.8.3
# via hdx-python-api (pyproject.toml)
hdx-python-utilities==3.7.3
hdx-python-utilities==3.7.4
# via
# hdx-python-api (pyproject.toml)
# hdx-python-country
humanize==4.10.0
humanize==4.11.0
# via frictionless
identify==2.6.0
identify==2.6.1
# via pre-commit
idna==3.8
idna==3.10
# via
# email-validator
# requests
Expand All @@ -72,33 +72,33 @@ inflect==7.4.0
# via quantulum3
iniconfig==2.0.0
# via pytest
isodate==0.6.1
isodate==0.7.2
# via frictionless
jinja2==3.1.4
# via frictionless
jsonlines==4.0.0
# via hdx-python-utilities
jsonpath-ng==1.6.1
jsonpath-ng==1.7.0
# via libhxl
jsonschema==4.23.0
# via
# frictionless
# tableschema-to-template
jsonschema-specifications==2023.12.1
jsonschema-specifications==2024.10.1
# via jsonschema
libhxl==5.2.1
libhxl==5.2.2
# via
# hdx-python-api (pyproject.toml)
# hdx-python-country
loguru==0.7.2
# via hdx-python-utilities
makefun==1.15.4
makefun==1.15.6
# via hdx-python-api (pyproject.toml)
markdown-it-py==3.0.0
# via rich
marko==2.1.2
# via frictionless
markupsafe==2.1.5
markupsafe==3.0.2
# via jinja2
mdurl==0.1.2
# via markdown-it-py
Expand All @@ -116,7 +116,7 @@ packaging==24.1
# via pytest
petl==1.7.15
# via frictionless
platformdirs==4.3.2
platformdirs==4.3.6
# via virtualenv
pluggy==1.5.0
# via pytest
Expand All @@ -126,27 +126,27 @@ ply==3.11
# libhxl
pockets==0.9.1
# via sphinxcontrib-napoleon
pre-commit==3.8.0
pre-commit==4.0.1
# via hdx-python-api (pyproject.toml)
pyasn1==0.6.0
pyasn1==0.6.1
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.4.0
pyasn1-modules==0.4.1
# via google-auth
pydantic==2.9.1
pydantic==2.9.2
# via frictionless
pydantic-core==2.23.3
pydantic-core==2.23.4
# via pydantic
pygments==2.18.0
# via rich
pyphonetics==0.5.3
# via hdx-python-country
pytest==8.3.2
pytest==8.3.3
# via
# hdx-python-api (pyproject.toml)
# pytest-cov
pytest-cov==5.0.0
pytest-cov==6.0.0
# via hdx-python-api (pyproject.toml)
python-dateutil==2.9.0.post0
# via
Expand Down Expand Up @@ -186,30 +186,29 @@ requests-oauthlib==2.0.0
# via google-auth-oauthlib
rfc3986==2.0.0
# via frictionless
rich==13.8.0
rich==13.9.4
# via typer
rpds-py==0.20.0
rpds-py==0.21.0
# via
# jsonschema
# referencing
rsa==4.9
# via google-auth
ruamel-yaml==0.18.6
# via hdx-python-utilities
ruamel-yaml-clib==0.2.8
ruamel-yaml-clib==0.2.12
# via ruamel-yaml
setuptools==74.1.2
setuptools==75.3.0
# via ckanapi
shellingham==1.5.4
# via typer
simpleeval==0.9.13
simpleeval==1.0.3
# via frictionless
simplejson==3.19.3
# via ckanapi
six==1.16.0
# via
# ckanapi
# isodate
# pockets
# python-dateutil
# sphinxcontrib-napoleon
Expand All @@ -223,11 +222,13 @@ tableschema-to-template==0.0.13
# via hdx-python-utilities
tabulate==0.9.0
# via frictionless
tenacity==9.0.0
# via hdx-python-country
text-unidecode==1.3
# via python-slugify
typeguard==4.3.0
typeguard==4.4.1
# via inflect
typer==0.12.5
typer==0.13.0
# via frictionless
typing-extensions==4.12.2
# via
Expand All @@ -240,13 +241,13 @@ unidecode==1.3.8
# via
# libhxl
# pyphonetics
urllib3==2.2.2
urllib3==2.2.3
# via
# libhxl
# requests
validators==0.34.0
# via frictionless
virtualenv==20.26.4
virtualenv==20.27.1
# via pre-commit
wheel==0.44.0
# via libhxl
Expand Down
35 changes: 35 additions & 0 deletions src/hdx/data/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -2960,3 +2960,38 @@ def download_and_generate_resource(
quickcharts=quickcharts,
encoding=kwargs.get("encoding", None),
)

def add_hapi_error(
self,
error_message: str,
resource_name: Optional[str] = None,
resource_id: Optional[str] = None,
) -> bool:
"""Writes error messages that were uncovered while processing data for
the HAPI database to a resource's metadata on HDX. If the resource
already has an error message, it is only overwritten if the two
messages are different.
Args:
error_message (str): Error(s) uncovered
resource_name (Optional[str]): Resource name. Defaults to None
resource_id (Optional[str]): Resource id. Defaults to None
Returns:
bool: True if a message was added, False if not
"""
if resource_name is None and resource_id is None:
return False
resource = None
for res in self.get_resources():
if res["name"] == resource_name or res["id"] == resource_id:
resource = res
break
if not resource:
return False
resource_error = resource.get("qa_hapi_report")
if resource_error and resource_error == error_message:
return False
resource["qa_hapi_report"] = error_message
resource.update_in_hdx(operation="patch")
return True
71 changes: 71 additions & 0 deletions tests/hdx/data/test_dataset_add_hapi_error.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import copy
import json

import pytest

from . import MockResponse, dataset_data, dataset_resultdict, resources_data
from hdx.api.configuration import Configuration
from hdx.data.dataset import Dataset


class TestDatasetAddHAPIError:
@pytest.fixture(scope="function")
def hapi_resource_update(self):
class MockSession:
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode("utf-8"))
if "show" in url:
resource_id = datadict["id"]
resources_json = dataset_resultdict["resources"]
if resource_id == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5":
resource_json = resources_json[0]
elif resource_id == "3d777226-96aa-4239-860a-703389d16d1f":
resource_json = resources_json[1]
elif resource_id == "3d777226-96aa-4239-860a-703389d16d1g":
resource_json = resources_json[2]
else:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Invalid id", "__type": "TEST ERROR: Invalid id Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_patch"}',
)
result = json.dumps(resource_json)
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_create"}'
% result,
)
if "patch" not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not patch", "__type": "TEST ERROR: Not Patch Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_patch"}',
)
result = json.dumps(datadict)
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_patch"}'
% result,
)

Configuration.read().remoteckan().session = MockSession()

def test_add_hapi_error(self, configuration, hapi_resource_update):
datasetdata = copy.deepcopy(dataset_data)
resourcesdata = copy.deepcopy(resources_data)
datasetdata["resources"] = resourcesdata
dataset = Dataset(datasetdata)
success = dataset.add_hapi_error(
error_message="test message",
resource_name="Resource1",
)
assert success is True
success = dataset.add_hapi_error(
error_message="test message",
resource_id="de6549d8-268b-4dfe-adaf-a4ae5c8510d5",
)
assert success is False
success = dataset.add_hapi_error(
error_message="test message 1",
resource_name="Resource1",
)
assert success is True
Loading

0 comments on commit c13ff6c

Please sign in to comment.