Merge pull request #59 from joegasewicz/server-should-create-tables-a…

…nd-dump-csv-#56

server should create tables and dump csv #56

README.md

@@ -5,9 +5,8 @@ REST API & Apache Airflow project that returns location data.
 ### Version 2
 The latest version for geolocations-api is available here:
 ```
-docker pull bandnoticeboard/geolocations-api:latest
+docker pull bandnoticeboard/geolocations-api:2.1.0-alpha
 ```
-
 ### ETL
 ETL pipelines with [Apache Airflow](https://airflow.apache.org/). See [etl](etl)
 
@@ -18,23 +17,12 @@ ETL pipelines with [Apache Airflow](https://airflow.apache.org/). See [etl](etl)
 A tornado REST api. See [server](server)
 
 ## Quick Start
-1. Run docker compose `make docker-compose-local`
-2. Clone & cd into the root folder of this repo
-3. Run Airflow to provision the Postgres database with the geolocation data.
-```bash
-# This will create a `locations` table in your db from the `TODO` dump
-TODO
-# OR run the following cmd:
-TODO
-```
-4. Run the latest version of geolocations-api
-```bash
-docker run bandnoticeboard/geolocations-api
-```
+1. Run docker compose `docker run bandnoticeboard/geolocations-api:2.1.0-alpha`
+
 # Example Queries
 Currently, the api returns 5 entrees per query
 
-Use the `name` query param to fetch the first 5 similar results:
+Use the `town` query param to fetch the first 5 similar results:
 ```bash
 curl http://localhost:8000/towns?name=col
 ```
@@ -57,20 +45,20 @@ Will return
 ## Docker Compose Example
 See [docker-compose.example.yml](https://github.com/joegasewicz/geolocations-api/docker-compose.example.yml)
 ```bash
-version: "3"
-
 services:
 
   geolocations_api:
     image: "bandnoticeboard/geolocations-api:latest"
     ports:
       - "8000:8000"
     environment:
-      API_DB_NAME: towns_db
-      API_DB_USERNAME: admin
-      API_DB_PASSWORD: admin
-      API_DB_HOST: host.docker.internal
-      API_DB_PORT: 5433
+      PGPORT: 5433
+      PGDATABASE: location_etl_db
+      PGUSER: admin
+      PGPASSWORD: admin
+      PGHOST: host.docker.internal
+      SERVER_PORT: 8000
+      SERVER_HOST: 0.0.0.0
 ```
 ## Contributing
 PR's are welcome for bug fixes or open an issue.

docker-compose.example.yml

@@ -8,14 +8,14 @@ services:
 #      - "host.docker.internal:host-gateway"
     image: "bandnoticeboard/geolocations-api:latest"
     ports:
-      - "7000:7000"
+      - "8000:8000"
     environment:
-      API_DB_NAME: towns_db
-      API_DB_USERNAME: admin
-      API_DB_PASSWORD: admin
-      API_DB_HOST: host.docker.internal
-      API_DB_PORT: 27017
-      SERVER_PORT: 6000
+      PGPORT: 5433
+      PGDATABASE: location_etl_db
+      PGUSER: admin
+      PGPASSWORD: admin
+      PGHOST: host.docker.internal
+      SERVER_PORT: 8000
       SERVER_HOST: 0.0.0.0
 
   postgres_etl:

server/Makefile

@@ -1,3 +1,7 @@
+# Docker variables
+DOCKER_TAG=2.1.0-alpha
+DOCKER_NAME=bandnoticeboard/geolocations-api
+
 # Server Variables
 PGPORT=5433
 PGDATABASE=location_etl_db
@@ -6,9 +10,6 @@ PGPASSWORD=admin
 PGHOST=host.docker.internal
 SERVER_PORT=8000
 SERVER_HOST=0.0.0.0
-# Docker variables
-DOCKER_TAG=2.0.0
-DOCKER_NAME=bandnoticeboard/geolocations-api
 
 locations-api-server:
 	pipenv run python prod.py

server/Pipfile

@@ -9,6 +9,7 @@ sqlalchemy = "*"
 marshmallow = "*"
 psycopg2-binary = "*"
 alembic = "*"
+pandas = "*"
 
 [dev-packages]
 pytest = "*"

server/app/__init__.py

@@ -1,14 +1,63 @@
 import tornado
+import pandas as pd
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy import (
+    MetaData,
+    Table,
+    Column,
+    Integer,
+    String,
+    Float,
+)
 
 from app.handlers import (
     HealthHandler,
     LocationHandler,
 )
+from app.models import (
+    Location as LocationModel,
+)
 from app.config import Config
 from app.utils.server import get_version
+from app.utils.database import engine, Session, Model
+from app.utils.logger import logger
 
 
 def make_app(config: Config) -> tornado.web.Application:
+    """
+    :param config:
+    :return:
+    """
+    # Only create tables that don't already exist
+    metadata = MetaData()
+    metadata.reflect(bind=engine)
+    locations_table = metadata.tables.get("locations")
+    if locations_table is None:
+        logger.info(f"Loading data from {config.LOCATION_DATA_FILE}")
+        df = pd.read_csv(config.LOCATION_DATA_FILE, index_col=0)
+        session = Session()
+        try:
+            Model.metadata.create_all(engine)
+            df_dict =df.to_dict("records")
+            for record in df_dict:
+                location = LocationModel(
+                    town=record["town"],
+                    latitude=record["latitude"],
+                    longitude=record["longitude"],
+                    iso_3166_1=record["iso_3166_1"],
+                    country=record["country"],
+                )
+                session.add(location)
+            session.commit()
+            session.close()
+        except SQLAlchemyError as err:
+            session.rollback()
+            logger.error(f"Error dumping {config.LOCATION_DATA_FILE}", exc_info=err)
+        finally:
+           engine.dispose()
+    else:
+        logger.info("Skipping dumping locations data")
+
     version = get_version(config)
     return tornado.web.Application([
         (fr"{version}/health", HealthHandler),

server/app/config.py

@@ -12,6 +12,9 @@ class Config:
     VERSION = "v0.1"
     SERVER_PORT = os.getenv("SERVER_PORT", 8888)
     SERVER_HOST = os.getenv("SERVER_HOST", "0.0.0.0")
+    # Location Data
+    LOCATION_DATA_FILE = "data/uk_ons_locations_transform.csv"
+    SQLALCHEMY_LOGGING = False
 
 class TestConfig(Config):
     # Postgres

server/app/handlers/locations.py

@@ -8,7 +8,7 @@
 from app.schemas import (
     Location as LocationSchema,
 )
-from app.utils.database import get_session
+from app.utils.database import Session, engine
 from app.utils.logger import logger
 
 
@@ -27,7 +27,7 @@ def get(self) -> None:
             return
         location_list = []
         try:
-            session = get_session()
+            session = Session()
             q = select(LocationModel).where(LocationModel.town.ilike(f"%{town_query}%"))
             q = q.limit(5)
             locations = session.execute(q).all()

server/app/utils/database.py

@@ -18,9 +18,8 @@ def get_engine(*, config: Config):
         database=config.PGDATABASE,
         port=config.PGPORT,
     )
-    return create_engine(url, echo=True)
+    return create_engine(url, echo=config.SQLALCHEMY_LOGGING)
 
 
-def get_session():
-    engine = get_engine(config=Config())
-    return sessionmaker(engine)()
+engine = get_engine(config=Config())
+Session = sessionmaker(engine)