From c34556f2f6245854010af363e8d71cac6ba380f5 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sat, 6 Feb 2021 22:16:35 -0800 Subject: [PATCH 1/2] Build flexible reporting endpoint Fixes #931 --- .../code/lacity_data_api/routers/reports.py | 64 +++++++++++++++++-- .../api/code/lacity_data_api/routers/shim.py | 10 +-- .../{reports.py => visualizations.py} | 0 .../api/tests/integration/test_api_reports.py | 20 ++++++ 4 files changed, 82 insertions(+), 12 deletions(-) rename server/api/code/lacity_data_api/services/{reports.py => visualizations.py} (100%) create mode 100644 server/api/tests/integration/test_api_reports.py diff --git a/server/api/code/lacity_data_api/routers/reports.py b/server/api/code/lacity_data_api/routers/reports.py index cf5027dac..194bfb998 100644 --- a/server/api/code/lacity_data_api/routers/reports.py +++ b/server/api/code/lacity_data_api/routers/reports.py @@ -1,20 +1,70 @@ import datetime +import re +from typing import List, Optional -from fastapi import APIRouter +from fastapi import APIRouter, Query +from sqlalchemy import text +from ..models import db from ..models.service_request import ServiceRequest +from ..models.request_type import RequestType +from ..models.council import Council router = APIRouter() +# maps keywords to sqlalchemy columns or functions +field_dict = { + "created_year": db.extract( + 'year', + ServiceRequest.created_date + ).label('created_year'), + "created_month": db.extract( + 'month', + ServiceRequest.created_date + ).label('created_month'), + "created_date": ServiceRequest.created_date, + "council_name": Council.council_name, + "type_name": RequestType.type_name +} -@router.get("/", description="*BETA: Rudimentary reporting endpoint") +filter_regex = "^(\w+)([>=<]+)([\w-]+)$" + + +@router.get("") async def run_report( - start_date: datetime.date = datetime.date.today() - datetime.timedelta(days=7), - end_date: datetime.date = datetime.date.today() + field: Optional[List[str]] = Query( + ["type_name", "created_date"], + description="ex. created_date", + regex="(created_year|created_month|created_date|council_name|type_name)" + ), + filter: Optional[List[str]] = Query( + [f"created_date>={str(datetime.date.today() - datetime.timedelta(days=7))}"], + description="Field then operator then value (ex. created_date>=2021-01-01 or council_name=Arleta", + regex=filter_regex + ) ): - result = await ServiceRequest.get_request_reports( - start_date, - end_date + # set up the fields for select and group by + fields = [field_dict[i] for i in field] + group_by = fields.copy() + fields.append(db.func.count().label("counts")) + + # set up filters for where clause + filters = [] + for f in filter: + match = re.search(filter_regex, f) + filters.append(f"{match.group(1)} {match.group(2)} '{match.group(3)}'") + + result = await ( + db.select( + fields + ).select_from( + ServiceRequest.join(RequestType).join(Council) + ).where( + text(' AND '.join(filters)) + ).group_by( + *group_by + ).gino.all() ) + return result diff --git a/server/api/code/lacity_data_api/routers/shim.py b/server/api/code/lacity_data_api/routers/shim.py index 2093459f5..06ee9c519 100644 --- a/server/api/code/lacity_data_api/routers/shim.py +++ b/server/api/code/lacity_data_api/routers/shim.py @@ -7,7 +7,7 @@ request_type, service_request, council ) from ..services import ( - email, github, reports + email, github, visualizations ) router = APIRouter() @@ -92,7 +92,7 @@ async def get_pins(filter: Filter): @router.post("/visualizations") async def get_visualizations(filter: Filter): - result = await reports.get_visualization( + result = await visualizations.get_visualization( filter.startDate, filter.endDate, filter.requestTypes, @@ -103,19 +103,19 @@ async def get_visualizations(filter: Filter): @router.post("/comparison/frequency") async def get_comparison_frequency(comp_filter: Comparison): - result = await reports.freq_comparison(**dict(comp_filter)) + result = await visualizations.freq_comparison(**dict(comp_filter)) return result @router.post("/comparison/timetoclose") async def get_comparison_time_to_close(comp_filter: Comparison): - result = await reports.ttc_comparison(**dict(comp_filter)) + result = await visualizations.ttc_comparison(**dict(comp_filter)) return result @router.post("/comparison/counts") async def get_comparison_counts(comp_filter: Comparison): - result = await reports.counts_comparison(**dict(comp_filter)) + result = await visualizations.counts_comparison(**dict(comp_filter)) return result diff --git a/server/api/code/lacity_data_api/services/reports.py b/server/api/code/lacity_data_api/services/visualizations.py similarity index 100% rename from server/api/code/lacity_data_api/services/reports.py rename to server/api/code/lacity_data_api/services/visualizations.py diff --git a/server/api/tests/integration/test_api_reports.py b/server/api/tests/integration/test_api_reports.py new file mode 100644 index 000000000..88c640ea3 --- /dev/null +++ b/server/api/tests/integration/test_api_reports.py @@ -0,0 +1,20 @@ + +def test_api_report(client): + url = "/reports?filter=created_date>=2020-01-01" + response = client.get(url) + assert response.status_code == 200 + assert len(response.json()) == 30 + + +def test_api_report_type_name(client): + url = "/reports?filter=created_date>=2020-01-01&field=type_name" + response = client.get(url) + assert response.status_code == 200 + assert len(response.json()) == 10 + + +def test_api_report_council_name(client): + url = "/reports?filter=created_date>=2020-01-01&field=council_name" + response = client.get(url) + assert response.status_code == 200 + assert len(response.json()) == 99 From 7a58e96c1841a7866ef95fd342a8fc5cef252078 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sat, 6 Feb 2021 22:20:25 -0800 Subject: [PATCH 2/2] flake8 --- server/api/code/lacity_data_api/routers/reports.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/api/code/lacity_data_api/routers/reports.py b/server/api/code/lacity_data_api/routers/reports.py index 194bfb998..ada066ccf 100644 --- a/server/api/code/lacity_data_api/routers/reports.py +++ b/server/api/code/lacity_data_api/routers/reports.py @@ -27,7 +27,7 @@ "type_name": RequestType.type_name } -filter_regex = "^(\w+)([>=<]+)([\w-]+)$" +filter_regex = "^(\w+)([>=<]+)([\w-]+)$" # noqa @router.get("") @@ -39,7 +39,10 @@ async def run_report( ), filter: Optional[List[str]] = Query( [f"created_date>={str(datetime.date.today() - datetime.timedelta(days=7))}"], - description="Field then operator then value (ex. created_date>=2021-01-01 or council_name=Arleta", + description=""" + Field then operator then value + (ex. created_date>=2021-01-01 or council_name=Arleta + """, regex=filter_regex ) ):