geocoding: remove pymongo-inmemory, lru-dict

* Replace lru-dict with functools.lru_cache
* Run tests in docker using mongo image, removing pymongo-inmemory

.github/workflows/geocoding-service-python.yml

@@ -32,7 +32,5 @@ jobs:
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Run python tests
-      run: |
-        pip install poetry
-        ./run_tests.sh
+      run: ./test_docker.sh
 

.github/workflows/suggest-python.yml

@@ -20,7 +20,7 @@ jobs:
         working-directory: suggest/acronyms
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3.10
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'

geocoding/location-service/Dockerfile-test

@@ -0,0 +1,63 @@
+# `python-base` sets up all our shared environment variables
+FROM python:3.10-slim as python-base
+
+ENV PYTHONUNBUFFERED=1 \
+    # prevents python creating .pyc files
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.2.2 \
+    # make poetry install to this location
+    POETRY_HOME="/opt/poetry" \
+    # make poetry create the virtual environment in the project's root
+    # it gets named `.venv`
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    \
+    # this is where our requirements + virtual environment will live
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+# `builder-base` stage is used to build deps + create our virtual environment
+FROM python-base as builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y curl
+
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+# copy project requirement files here to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+COPY poetry.lock pyproject.toml ./
+
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN poetry install --no-dev
+
+# `development` image is used during development / testing
+FROM python-base as development
+
+RUN apt-get update && apt-get upgrade -y curl
+
+WORKDIR $PYSETUP_PATH
+
+# copy in our built poetry + venv
+COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+# quicker install as runtime deps are already installed
+RUN poetry install
+
+# will become mountpoint of our code
+WORKDIR /app
+
+COPY ./ ./
+
+CMD ["./test.sh"]

geocoding/location-service/docker-compose-test.yml

@@ -0,0 +1,11 @@
+version: "3.7"
+
+services:
+  test:
+    build:
+      context: .
+      dockerfile: Dockerfile-test
+  mongo:
+    image: mongo:5.0
+    ports:
+      - "27017:27017"

geocoding/location-service/pyproject.toml

@@ -25,7 +25,6 @@ itsdangerous = "1.1.0"
 Jinja2 = "2.11.3"
 jmespath = "0.10.0"
 json_schema = "0.3"
-lru-dict = "1.1.7"
 mapbox = "0.18.1"
 MarkupSafe = "1.1.1"
 mock = "4.0.3"
@@ -50,7 +49,6 @@ ratelimiter = "1.2.0"
 dnspython = "2.1.0"
 
 [tool.poetry.dev-dependencies]
-pymongo-inmemory = "0.2.0"
 pytest = "7.1.3"
 
 [build-system]

geocoding/location-service/src/app/admins_fetcher.py

@@ -1,8 +1,8 @@
 import json
 import logging
+import functools
 
 import ratelimiter
-from lru import LRU
 
 from src.integration.mapbox_client import mapbox_tile_query
 
@@ -12,18 +12,16 @@ def __init__(self, access_token, db, rate_limit=600):
         self.rate_limit = ratelimiter.RateLimiter(max_calls=rate_limit, period=60)
         self.access_token = access_token
         self.admins = db.get_collection('admins')
-        self.cache = LRU(500)
+
+    @functools.lru_cache(maxsize=500)
+    def cached_mapbox_tile_query(self, geocode_geometry):
+        return mapbox_tile_query(self.access_token, json.loads(geocode_geometry), rate_limit=self.rate_limit)
 
     def fill_admins(self, geocode):
         if 'administrativeAreaLevel1' in geocode and \
             'administrativeAreaLevel2' in geocode and 'administrativeAreaLevel3' in geocode:
             return geocode
-        cacheKey = json.dumps(geocode)
-        if cacheKey in self.cache:
-            response = self.cache[cacheKey]
-        else:
-            response = mapbox_tile_query(self.access_token, geocode['geometry'], rate_limit=self.rate_limit)
-            self.cache[cacheKey] = response
+        response = self.cached_mapbox_tile_query(json.dumps(geocode['geometry']))
         if 'features' not in response:
             # probably your API key doesn't support the premium APIs, skip this step
             return geocode

geocoding/location-service/src/app/geocoder.py

@@ -3,8 +3,8 @@
 import logging
 import ratelimiter
 import sys
+import functools
 
-from lru import LRU
 
 from src.integration.mapbox_client import mapbox_geocode
 
@@ -25,7 +25,6 @@ def __init__(self, api_token, admins_fetcher, rate_limit=600):
         """Needs a mapbox API token."""
         self.rate_limit = ratelimiter.RateLimiter(max_calls=rate_limit, period=60)
         self.api_token = api_token
-        self.cache = LRU(500)
         self.admins_fetcher = admins_fetcher
 
     def resolutionToMapboxType(self, resolution):
@@ -114,17 +113,21 @@ def unpackGeoJson(self, feature):
         self.admins_fetcher.fill_admins(res)
         return res
 
+    @functools.lru_cache(500)
+    def cached_mapbox_geocode(self, query: str, options: str):
+        options = json.loads(options)
+        limit_resolution = options.get("limitToResolution", [])
+        limit_country = options.get("limitToCountry")
+        types = [self.resolutionToMapboxType(i) for i in limit_resolution] if limit_resolution else None
+        geoResult = mapbox_geocode(
+            self.api_token,
+            json.loads(query),
+            types=types, limit=5,
+            languages=['en'],
+            country=limit_country,
+            rate_limit=self.rate_limit
+        )
+        return [self.unpackGeoJson(feature) for feature in geoResult['features']]
+
     def geocode(self, query, options={}):
-        cacheKey = json.dumps({
-            'query': query.lower(),
-            'options': options
-        })
-        if cacheKey in self.cache:
-            return self.cache[cacheKey]
-        resolutions = options.get('limitToResolution', [])
-        types = [self.resolutionToMapboxType(i) for i in resolutions] if resolutions else None
-        countries = options.get('limitToCountry')
-        geoResult = mapbox_geocode(self.api_token, query, types=types, limit=5, languages=['en'], country=countries, rate_limit=self.rate_limit)
-        response = [self.unpackGeoJson(feature) for feature in geoResult['features']]
-        self.cache[cacheKey] = response
-        return response
+        return self.cached_mapbox_geocode(json.dumps(query), json.dumps(options))

geocoding/location-service/test.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -eou pipefail
+
+DOCKERIZED=1 poetry run pytest .
+echo "Tests and code quality checks passed"

geocoding/location-service/test/test_admins_fetcher.py

@@ -1,15 +1,17 @@
+import os
 import unittest
 from mock import patch
-from pymongo_inmemory import MongoClient
+from pymongo import MongoClient
 
 from src.app.admins_fetcher import AdminsFetcher
 from src.app.geocoder import Geocoder
 
-
+@unittest.skipIf(not os.environ.get("DOCKERIZED", False),
+                 "Skipping outside dockerized environment")
 class TestAdminsFetcher(unittest.TestCase):
 
     def setUp(self):
-        self.mongo = MongoClient()
+        self.mongo = MongoClient(host="mongo")
         self.db = self.mongo['testdb']
         self.fetcher = AdminsFetcher('token', self.db)
 

geocoding/location-service/test_docker.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eo pipefail
+
+function cleanup() {
+  docker compose -f docker-compose-test.yml stop
+  docker compose -f docker-compose-test.yml down -v --remove-orphans
+}
+
+trap cleanup EXIT
+
+docker compose -f docker-compose-test.yml up --build --exit-code-from test