diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7687425..c0f436b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r space2stats_api/requirements.txt - pip install pre-commit + pip install -r space2stats_api/src/requirements.txt + pip install pre-commit pytest - name: Set PYTHONPATH run: echo "PYTHONPATH=$(pwd)/space2stats_api" >> $GITHUB_ENV diff --git a/.gitignore b/.gitignore index 32a76a8..4edf48e 100644 --- a/.gitignore +++ b/.gitignore @@ -100,4 +100,8 @@ db.env # data *.parquet *.duckdb -.pgdata \ No newline at end of file +.pgdata +space2stats_api/space2stats_env +*.env +cdk.out +lambda_layer \ No newline at end of file diff --git a/notebooks/space2stats_api_demo.ipynb b/notebooks/space2stats_api_demo.ipynb index 62dec03..971688a 100644 --- a/notebooks/space2stats_api_demo.ipynb +++ b/notebooks/space2stats_api_demo.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 8, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -21,18 +21,18 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ - "BASE_URL = \"http://localhost:8000\"\n", + "BASE_URL = \"https://space2stats.ds.io\"\n", "FIELDS_ENDPOINT = f\"{BASE_URL}/fields\"\n", "SUMMARY_ENDPOINT = f\"{BASE_URL}/summary\"" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -123,7 +123,7 @@ "request_payload = {\n", " \"aoi\": aoi,\n", " \"spatial_join_method\": \"centroid\",\n", - " \"fields\": [\"sum_pop_2020\", \"sum_pop_f_2020\", \"sum_pop_m_2020\"], \n", + " \"fields\": [\"sum_pop_2020\"], \n", " \"geometry\": \"point\"\n", "}\n", "\n", @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -165,135 +165,98 @@ " hex_id\n", " geometry\n", " sum_pop_2020\n", - " sum_pop_f_2020\n", - " sum_pop_m_2020\n", " \n", " \n", " \n", " \n", " 0\n", " 866a4a48fffffff\n", - " POINT (36.31771 2.23633)\n", + " POINT (35.76352 2.99589)\n", " 399.860905\n", - " 189.675539\n", - " 210.185366\n", " \n", " \n", " 1\n", " 866a4a497ffffff\n", - " POINT (40.18159 0.05763)\n", + " POINT (40.58048 -3.79365)\n", " 582.555159\n", - " 276.337255\n", - " 306.217904\n", " \n", " \n", " 2\n", " 866a4a49fffffff\n", - " POINT (38.59096 0.13944)\n", + " POINT (41.10421 3.37873)\n", " 749.911237\n", - " 355.723245\n", - " 394.187992\n", " \n", " \n", " 3\n", " 866a4a4d7ffffff\n", - " POINT (35.07124 0.80971)\n", + " POINT (37.26153 3.74581)\n", " 863.888290\n", - " 418.309236\n", - " 445.579054\n", " \n", " \n", " 4\n", " 866a5820fffffff\n", - " POINT (37.4356 3.35699)\n", + " POINT (40.01148 1.53124)\n", " 525.085147\n", - " 249.076134\n", - " 276.009012\n", " \n", " \n", " ...\n", " ...\n", " ...\n", " ...\n", - " ...\n", - " ...\n", " \n", " \n", " 16212\n", " 867b5dd77ffffff\n", - " POINT (39.15438 -1.51437)\n", + " POINT (34.94474 1.24558)\n", " -36.000000\n", - " -18.000000\n", - " -18.000000\n", " \n", " \n", " 16213\n", " 867b5dd87ffffff\n", - " POINT (35.80252 0.90823)\n", + " POINT (40.95343 -1.83280)\n", " -36.000000\n", - " -18.000000\n", - " -18.000000\n", " \n", " \n", " 16214\n", " 867b5dd8fffffff\n", - " POINT (37.93845 0.83454)\n", + " POINT (35.20290 -0.29666)\n", " -36.000000\n", - " -18.000000\n", - " -18.000000\n", " \n", " \n", " 16215\n", " 867b5dd9fffffff\n", - " POINT (38.65824 -2.60028)\n", + " POINT (41.28333 -1.08552)\n", " -36.000000\n", - " -18.000000\n", - " -18.000000\n", " \n", " \n", " 16216\n", " 867b5ddafffffff\n", - " POINT (36.6641 2.37083)\n", + " POINT (36.63048 1.35038)\n", " -36.000000\n", - " -18.000000\n", - " -18.000000\n", " \n", " \n", "\n", - "

16217 rows × 5 columns

\n", + "

16217 rows × 3 columns

\n", "" ], "text/plain": [ - " hex_id geometry sum_pop_2020 \\\n", - "0 866a4a48fffffff POINT (36.31771 2.23633) 399.860905 \n", - "1 866a4a497ffffff POINT (40.18159 0.05763) 582.555159 \n", - "2 866a4a49fffffff POINT (38.59096 0.13944) 749.911237 \n", - "3 866a4a4d7ffffff POINT (35.07124 0.80971) 863.888290 \n", - "4 866a5820fffffff POINT (37.4356 3.35699) 525.085147 \n", - "... ... ... ... \n", - "16212 867b5dd77ffffff POINT (39.15438 -1.51437) -36.000000 \n", - "16213 867b5dd87ffffff POINT (35.80252 0.90823) -36.000000 \n", - "16214 867b5dd8fffffff POINT (37.93845 0.83454) -36.000000 \n", - "16215 867b5dd9fffffff POINT (38.65824 -2.60028) -36.000000 \n", - "16216 867b5ddafffffff POINT (36.6641 2.37083) -36.000000 \n", - "\n", - " sum_pop_f_2020 sum_pop_m_2020 \n", - "0 189.675539 210.185366 \n", - "1 276.337255 306.217904 \n", - "2 355.723245 394.187992 \n", - "3 418.309236 445.579054 \n", - "4 249.076134 276.009012 \n", - "... ... ... \n", - "16212 -18.000000 -18.000000 \n", - "16213 -18.000000 -18.000000 \n", - "16214 -18.000000 -18.000000 \n", - "16215 -18.000000 -18.000000 \n", - "16216 -18.000000 -18.000000 \n", + " hex_id geometry sum_pop_2020\n", + "0 866a4a48fffffff POINT (35.76352 2.99589) 399.860905\n", + "1 866a4a497ffffff POINT (40.58048 -3.79365) 582.555159\n", + "2 866a4a49fffffff POINT (41.10421 3.37873) 749.911237\n", + "3 866a4a4d7ffffff POINT (37.26153 3.74581) 863.888290\n", + "4 866a5820fffffff POINT (40.01148 1.53124) 525.085147\n", + "... ... ... ...\n", + "16212 867b5dd77ffffff POINT (34.94474 1.24558) -36.000000\n", + "16213 867b5dd87ffffff POINT (40.95343 -1.83280) -36.000000\n", + "16214 867b5dd8fffffff POINT (35.20290 -0.29666) -36.000000\n", + "16215 867b5dd9fffffff POINT (41.28333 -1.08552) -36.000000\n", + "16216 867b5ddafffffff POINT (36.63048 1.35038) -36.000000\n", "\n", - "[16217 rows x 5 columns]" + "[16217 rows x 3 columns]" ] }, - "execution_count": 17, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -306,24 +269,24 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f9d4a524b5bf4d1a950f1cb8cc8d5b54", + "model_id": "00bbfea95ae440d3a73ebb161e3142ab", "version_major": 2, - "version_minor": 1 + "version_minor": 0 }, "text/plain": [ - "Map(layers=[ScatterplotLayer(get_fill_color=\n", + "Map(layers=[ScatterplotLayer(get_fill_color=\n", "[\n", " [\n", " 2…" ] }, - "execution_count": 18, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -338,20 +301,6 @@ "m = Map(layer)\n", "m\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/postgres/deploy.md b/postgres/deploy.md new file mode 100644 index 0000000..ffa9b2b --- /dev/null +++ b/postgres/deploy.md @@ -0,0 +1,7 @@ +## Deployment Notes + +- Create database instance +- Update configuration in `db.env` +- Ingest parquet file with `load_to_prod.sh` (may require `chmod +x load_to_prod.sh`) +- Create index on hex_id (for performance):`CREATE INDEX idx_hex_id ON space2stats (hex_id)` - critical for performance of our queries +- Test with the [example notebook](notebooks/space2stats_api_demo.ipynb) \ No newline at end of file diff --git a/postgres/load_nyc_sample.sh b/postgres/load_nyc_sample.sh index cb73169..276956c 100755 --- a/postgres/load_nyc_sample.sh +++ b/postgres/load_nyc_sample.sh @@ -1,11 +1,15 @@ #!/bin/bash -# Database connection details -DB_HOST="localhost" -DB_PORT="5439" -DB_NAME="postgis" -DB_USER="username" -DB_PASSWORD="password" +# Load environment variables from db.env file +if [ -f db.env ]; then + export $(cat db.env | grep -v '#' | awk '/=/ {print $1}') +fi + +# Check if required environment variables are set +if [ -z "$DB_HOST" ] || [ -z "$DB_PORT" ] || [ -z "$DB_NAME" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ]; then + echo "One or more required environment variables are missing." + exit 1 +fi # Path to the sample Parquet file PARQUET_FILE="nyc_sample.parquet" diff --git a/postgres/load_parquet_chunks.sh b/postgres/load_parquet_chunks.sh index 18c1dd4..8f79d8b 100755 --- a/postgres/load_parquet_chunks.sh +++ b/postgres/load_parquet_chunks.sh @@ -1,11 +1,16 @@ #!/bin/bash -# Database connection details -DB_HOST="localhost" -DB_PORT="5439" -DB_NAME="postgis" -DB_USER="username" -DB_PASSWORD="password" + +# Load environment variables from db.env file +if [ -f db.env ]; then + export $(cat db.env | grep -v '#' | awk '/=/ {print $1}') +fi + +# Check if required environment variables are set +if [ -z "$DB_HOST" ] || [ -z "$DB_PORT" ] || [ -z "$DB_NAME" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ]; then + echo "One or more required environment variables are missing." + exit 1 +fi # Directory containing the Parquet chunks CHUNKS_DIR="parquet_chunks" diff --git a/postgres/load_to_prod.sh b/postgres/load_to_prod.sh new file mode 100755 index 0000000..3149e7b --- /dev/null +++ b/postgres/load_to_prod.sh @@ -0,0 +1,30 @@ +#!/bin/bash + + +# Load environment variables from db.env file +if [ -f db.env ]; then + export $(cat db.env | grep -v '#' | awk '/=/ {print $1}') +fi + +# Check if required environment variables are set +if [ -z "$DB_HOST" ] || [ -z "$DB_PORT" ] || [ -z "$DB_NAME" ] || [ -z "$DB_USER" ] || [ -z "$DB_PASSWORD" ]; then + echo "One or more required environment variables are missing." + exit 1 +fi + +# Directory containing the Parquet chunks +CHUNKS_DIR="parquet_chunks" + +# Name of the target table +TABLE_NAME="space2stats" +PARQUET_FILE=space2stats_updated.parquet + +echo "Starting" + +ogr2ogr -progress -f "PostgreSQL" \ + PG:"host=$DB_HOST port=$DB_PORT dbname=$DB_NAME user=$DB_USER password=$DB_PASSWORD" \ + "$PARQUET_FILE" \ + -nln $TABLE_NAME \ + -append \ + -lco SPATIAL_INDEX=NONE + diff --git a/space2stats_api/app/main.py b/space2stats_api/app/main.py deleted file mode 100644 index b28e3f0..0000000 --- a/space2stats_api/app/main.py +++ /dev/null @@ -1,13 +0,0 @@ -from fastapi import FastAPI - -from .routers import api - - -app = FastAPI() - -app.include_router(api.router) - - -@app.get("/") -def read_root(): - return {"message": "Welcome to Space2Stats!"} diff --git a/space2stats_api/app/models/summary.py b/space2stats_api/app/models/summary.py deleted file mode 100644 index e69de29..0000000 diff --git a/space2stats_api/app/settings.py b/space2stats_api/app/settings.py deleted file mode 100644 index e6f48eb..0000000 --- a/space2stats_api/app/settings.py +++ /dev/null @@ -1,15 +0,0 @@ -from pydantic_settings import BaseSettings, SettingsConfigDict - - -class Settings(BaseSettings): - DB_HOST: str - DB_PORT: int - DB_NAME: str - DB_USER: str - DB_PASSWORD: str - DB_TABLE_NAME: str - - model_config = SettingsConfigDict(env_file='db.env') - - -settings = Settings() diff --git a/space2stats_api/cdk/app.py b/space2stats_api/cdk/app.py new file mode 100644 index 0000000..2b9f419 --- /dev/null +++ b/space2stats_api/cdk/app.py @@ -0,0 +1,17 @@ +from aws_cdk import App, Environment +from aws_stack import Space2StatsStack +from settings import DeploymentSettings + + +settings = DeploymentSettings(_env_file="aws_deployment.env") + +env = Environment( + account=settings.CDK_DEFAULT_ACCOUNT, + region=settings.CDK_DEFAULT_REGION +) + +app = App() + +Space2StatsStack(app, "Space2StatsStack", env=env) + +app.synth() \ No newline at end of file diff --git a/space2stats_api/cdk/aws_stack.py b/space2stats_api/cdk/aws_stack.py new file mode 100644 index 0000000..50d08c8 --- /dev/null +++ b/space2stats_api/cdk/aws_stack.py @@ -0,0 +1,52 @@ +from aws_cdk import ( + Stack, + aws_apigatewayv2 as apigatewayv2, + aws_apigatewayv2_integrations as integrations, + aws_lambda as _lambda, + aws_certificatemanager as acm, + Duration +) +from aws_cdk.aws_lambda_python_alpha import PythonFunction +from constructs import Construct +from settings import AppSettings, DeploymentSettings + +class Space2StatsStack(Stack): + def __init__(self, scope: Construct, id: str, **kwargs) -> None: + super().__init__(scope, id, **kwargs) + + app_settings = AppSettings(_env_file="./aws_app.env") + deployment_settings = DeploymentSettings(_env_file="./aws_deployment.env") + + lambda_function = PythonFunction( + self, "Space2StatsFunction", + entry="../src", + runtime=_lambda.Runtime.PYTHON_3_11, + index="app/main.py", + timeout=Duration.seconds(120), + handler="handler", + environment=app_settings.model_dump(), + memory_size=1024 + ) + + certificate = acm.Certificate.from_certificate_arn(self, "Certificate", deployment_settings.CDK_CERTIFICATE_ARN) + + domain_name = apigatewayv2.DomainName( + self, "DomainName", + domain_name=deployment_settings.CDK_DOMAIN_NAME, + certificate=certificate + ) + + http_api = apigatewayv2.HttpApi( + self, "Space2StatsHttpApi", + default_integration=integrations.HttpLambdaIntegration( + "LambdaIntegration", + handler=lambda_function + ) + ) + + apigatewayv2.ApiMapping( + self, "ApiMapping", + api=http_api, + domain_name=domain_name, + stage=http_api.default_stage + ) \ No newline at end of file diff --git a/space2stats_api/cdk/cdk.json b/space2stats_api/cdk/cdk.json new file mode 100644 index 0000000..5ad42b5 --- /dev/null +++ b/space2stats_api/cdk/cdk.json @@ -0,0 +1,4 @@ +{ + "app": "python app.py", + "watch": { "include": ["../src/**", "."], "exclude": ["**/*.pyc"] } +} diff --git a/space2stats_api/cdk/settings.py b/space2stats_api/cdk/settings.py new file mode 100644 index 0000000..c8903e4 --- /dev/null +++ b/space2stats_api/cdk/settings.py @@ -0,0 +1,16 @@ +from pydantic_settings import BaseSettings + + +class AppSettings(BaseSettings): + DB_HOST: str + DB_PORT: str + DB_NAME: str + DB_USER: str + DB_PASSWORD: str + DB_TABLE_NAME: str + +class DeploymentSettings(BaseSettings): + CDK_DEFAULT_ACCOUNT: str + CDK_DEFAULT_REGION: str + CDK_CERTIFICATE_ARN: str + CDK_DOMAIN_NAME: str \ No newline at end of file diff --git a/space2stats_api/pytest.ini b/space2stats_api/pytest.ini new file mode 100644 index 0000000..343729c --- /dev/null +++ b/space2stats_api/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +env_files = db.env \ No newline at end of file diff --git a/space2stats_api/app/models/__init__.py b/space2stats_api/src/app/__init__.py similarity index 100% rename from space2stats_api/app/models/__init__.py rename to space2stats_api/src/app/__init__.py diff --git a/space2stats_api/src/app/main.py b/space2stats_api/src/app/main.py new file mode 100644 index 0000000..3e02954 --- /dev/null +++ b/space2stats_api/src/app/main.py @@ -0,0 +1,23 @@ +from fastapi import FastAPI +from mangum import Mangum +from fastapi.middleware.cors import CORSMiddleware + +from .routers import api + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(api.router) + +@app.get("/") +def read_root(): + return {"message": "Welcome to Space2Stats!"} + +handler = Mangum(app) \ No newline at end of file diff --git a/space2stats_api/app/routers/__init__.py b/space2stats_api/src/app/routers/__init__.py similarity index 100% rename from space2stats_api/app/routers/__init__.py rename to space2stats_api/src/app/routers/__init__.py diff --git a/space2stats_api/app/routers/api.py b/space2stats_api/src/app/routers/api.py similarity index 91% rename from space2stats_api/app/routers/api.py rename to space2stats_api/src/app/routers/api.py index 9dabf67..54da23e 100644 --- a/space2stats_api/app/routers/api.py +++ b/space2stats_api/src/app/routers/api.py @@ -4,8 +4,8 @@ from pydantic import BaseModel from geojson_pydantic import Feature, Polygon -from app.utils.h3_utils import generate_h3_ids, generate_h3_geometries -from app.utils.db_utils import get_available_fields, get_summaries +from ..utils.h3_utils import generate_h3_ids, generate_h3_geometries +from ..utils.db_utils import get_available_fields, get_summaries router = APIRouter() diff --git a/space2stats_api/src/app/settings.py b/space2stats_api/src/app/settings.py new file mode 100644 index 0000000..77b0403 --- /dev/null +++ b/space2stats_api/src/app/settings.py @@ -0,0 +1,10 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + DB_HOST: str + DB_PORT: int + DB_NAME: str + DB_USER: str + DB_PASSWORD: str + DB_TABLE_NAME: str diff --git a/space2stats_api/app/utils/__init__.py b/space2stats_api/src/app/utils/__init__.py similarity index 100% rename from space2stats_api/app/utils/__init__.py rename to space2stats_api/src/app/utils/__init__.py diff --git a/space2stats_api/app/utils/db_utils.py b/space2stats_api/src/app/utils/db_utils.py similarity index 96% rename from space2stats_api/app/utils/db_utils.py rename to space2stats_api/src/app/utils/db_utils.py index c95ef82..a565efd 100644 --- a/space2stats_api/app/utils/db_utils.py +++ b/space2stats_api/src/app/utils/db_utils.py @@ -1,5 +1,7 @@ import psycopg as pg -from ..settings import settings +from ..settings import Settings + +settings = Settings() DB_HOST = settings.DB_HOST DB_PORT = settings.DB_PORT diff --git a/space2stats_api/app/utils/h3_utils.py b/space2stats_api/src/app/utils/h3_utils.py similarity index 100% rename from space2stats_api/app/utils/h3_utils.py rename to space2stats_api/src/app/utils/h3_utils.py diff --git a/space2stats_api/requirements.txt b/space2stats_api/src/requirements.txt similarity index 75% rename from space2stats_api/requirements.txt rename to space2stats_api/src/requirements.txt index 1f46cef..a064fb7 100644 --- a/space2stats_api/requirements.txt +++ b/space2stats_api/src/requirements.txt @@ -1,12 +1,12 @@ fastapi uvicorn +mangum pandas python-dotenv shapely h3 -pytest psycopg[binary] httpx geojson-pydantic shapely -pydantic-settings>=2.0.0 \ No newline at end of file +pydantic-settings>=2.0.0 diff --git a/space2stats_api/tests/test_api.py b/space2stats_api/tests/test_api.py index 649f12b..f5ec0c7 100644 --- a/space2stats_api/tests/test_api.py +++ b/space2stats_api/tests/test_api.py @@ -2,7 +2,7 @@ import pytest from unittest.mock import patch -from app.main import app +from src.app.main import app client = TestClient(app) diff --git a/space2stats_api/tests/test_db_utils.py b/space2stats_api/tests/test_db_utils.py index a320156..61a661f 100644 --- a/space2stats_api/tests/test_db_utils.py +++ b/space2stats_api/tests/test_db_utils.py @@ -1,6 +1,6 @@ import unittest from unittest.mock import patch, Mock -from app.utils.db_utils import get_summaries, get_available_fields +from src.app.utils.db_utils import get_summaries, get_available_fields from psycopg.sql import SQL, Identifier diff --git a/space2stats_api/tests/test_h3_utils.py b/space2stats_api/tests/test_h3_utils.py index 8813581..7813dda 100644 --- a/space2stats_api/tests/test_h3_utils.py +++ b/space2stats_api/tests/test_h3_utils.py @@ -1,6 +1,6 @@ import pytest from shapely.geometry import Polygon, mapping -from app.utils.h3_utils import generate_h3_ids, generate_h3_geometries +from src.app.utils.h3_utils import generate_h3_ids, generate_h3_geometries polygon_coords = [ [-74.3, 40.5],