Change how the backend and database migrations are structured (Fixes #…

…119)

- BREAKING: Added `appointment` module folder in `src`.
- BREAKING: alembic.ini should now be placed in the backend root folder. You will need to grab a fresh copy as there's been additional changes.

- Added a super simple cli interface handled by main.py
- Adjusted main.py to bootstrap either a fast api server or a cli interface.
- Added a `update-db` command that is installed when pip install this module.
- The `update-db` command will initialize a fresh database or run migrations, resolving most of our database woes.
- New folder structure more closely matches the deployed folder structure for easier deployments.
- Local docker now only mounts the `src` folder
- Commented out some non-existent columns/constraints in a recent migration
- Added missing trailing slash to the frontend for `schedule/`
- Added sentry support to migrations
- Adjusted code-referenced folder locations as required

.gitignore

@@ -9,3 +9,4 @@ dist
 *.pickle
 .env
 venv
+.idea

backend/Dockerfile

@@ -6,10 +6,18 @@ WORKDIR /app
 ENV PATH="${PATH}:/root/.local/bin"
 ENV PYTHONPATH=.
 
+RUN mkdir scripts
+
 COPY requirements.txt .
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
+COPY pyproject.toml .
+COPY alembic.ini.example alembic.ini
+COPY scripts/dev-entry.sh scripts/dev-entry.sh
 
-COPY src/ .
+# Dev only
+COPY .env .
+
+RUN pip install --upgrade pip
+RUN pip install .
 
+EXPOSE 5000
 CMD ["/bin/sh", "./scripts/dev-entry.sh"]

backend/src/alembic.ini.example → backend/alembic.ini.example

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = migrations
+script_location = src/appointment/migrations
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
@@ -12,7 +12,7 @@ file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(re
 
 # sys.path path, will be prepended to sys.path if present.
 # defaults to the current working directory.
-prepend_sys_path = .
+prepend_sys_path = src/appointment
 
 # timezone to use when rendering the date within the migration file
 # as well as the filename.

backend/deploy.dockerfile

@@ -6,14 +6,24 @@ WORKDIR /app
 ENV PATH="${PATH}:/root/.local/bin"
 ENV PYTHONPATH=.
 
+RUN mkdir scripts
+
 COPY requirements.txt .
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
+COPY pyproject.toml .
+COPY alembic.ini.example alembic.ini
+COPY scripts/entry.sh scripts/entry.sh
 
-COPY src/ ./src
-COPY scripts/ ./scripts
+# Needed for deploy, we don't have a volume attached
+COPY src .
+
+RUN pip install --upgrade pip
+RUN pip install .
 
-RUN cp ./src/alembic.ini.example ./src/alembic.ini
+# install removes the src file and installs the application as /app/appointment
+# that's fine, but uhh let's add this hack to line it up with our dev environment.
+# I'll buy whoever fixes this a coffee.
+RUN mkdir src
+RUN ln -s /app/appointment src/appointment
 
 EXPOSE 5000
 CMD ["/bin/sh", "./scripts/entry.sh"]

backend/pyproject.toml

@@ -5,6 +5,9 @@ description = "Backend component to Thunderbird Appointment"
 requires-python = ">3.10"
 dynamic = ["dependencies"]
 
+[project.scripts]
+run-command = "src.appointment.main:cli"
+
 [project.urls]
 homepage = "https://appointment.day"
 repository = "https://github.com/thundernest/appointment.git"

backend/scripts/dev-entry.sh

@@ -1,10 +1,5 @@
 #!/bin/sh
 
-cd src
-echo 'Starting migrations...'
-alembic current
-alembic upgrade head
-echo 'Finished migrations!'
-cd ../
+run-command update-db
 
-uvicorn src.main:app --reload --host 0.0.0.0 --port 8090
+uvicorn --factory src.appointment.main:server --reload --host 0.0.0.0 --port 8090

backend/scripts/entry.sh

@@ -1,10 +1,5 @@
 #!/bin/sh
 
-cd src
-echo 'Starting migrations...'
-alembic current
-alembic upgrade head
-echo 'Finished migrations!'
-cd ../
+run-command update-db
 
-uvicorn src.main:app --host 0.0.0.0 --port 5000
+uvicorn --factory src.appointment.main:server --host 0.0.0.0 --port 5000

backend/src/appointment/commands/update_db.py

@@ -0,0 +1,37 @@
+import os
+
+from ..database import models
+from ..database.database import engine
+from alembic.runtime import migration
+
+
+def run():
+    print("Checking if we have a fresh database...")
+
+    # then, load the Alembic configuration and generate the
+    # version table, "stamping" it with the most recent rev:
+    from alembic import command
+    from alembic.config import Config
+
+    # TODO: Does this work on stage?
+    alembic_cfg = Config("./alembic.ini")
+
+    # If we have our database url env variable set, use that instead!
+    if os.getenv("DATABASE_URL"):
+        alembic_cfg.set_main_option("sqlalchemy.url", os.getenv("DATABASE_URL"))
+
+    with engine.begin() as connection:
+        context = migration.MigrationContext.configure(connection)
+        # Returns a tuple, empty if there's no revisions saved
+        revisions = context.get_current_heads()
+
+        # If we have no revisions, then fully create the database from the model metadata,
+        # and set our revision number to the latest revision. Otherwise run any new migrations
+        if len(revisions) == 0:
+            print("Initializing database, and setting it to the latest revision")
+            models.Base.metadata.create_all(bind=engine)
+            command.stamp(alembic_cfg, "head")
+        else:
+            print("Database already initialized, running migrations")
+            command.upgrade(alembic_cfg, 'head')
+

backend/src/controller/mailer.py → backend/src/appointment/controller/mailer.py

@@ -17,7 +17,7 @@
 from email.mime.text import MIMEText
 from fastapi.templating import Jinja2Templates
 
-templates = Jinja2Templates("src/templates/email")
+templates = Jinja2Templates("src/appointment/templates/email")
 
 
 def get_template(template_name) -> "jinja2.Template":

backend/src/main.py → backend/src/appointment/main.py

@@ -48,21 +48,6 @@
 
 logging.debug("Logger started!")
 
-# database
-from .database import models
-from .database.database import engine
-
-models.Base.metadata.create_all(bind=engine)
-
-# extra routes
-from .routes import api
-from .routes import account
-from .routes import google
-from .routes import schedule
-
-# init app
-app = FastAPI()
-
 if os.getenv("SENTRY_DSN") != "" or os.getenv("SENTRY_DSN") is not None:
     sentry_sdk.init(
         dsn=os.getenv("SENTRY_DSN"),
@@ -73,29 +58,58 @@
         environment=os.getenv("APP_ENV", "dev"),
     )
 
-# allow requests from own frontend running on a different port
-app.add_middleware(
-    CORSMiddleware,
-    # Work around for now :)
-    allow_origins=[
-        os.getenv("FRONTEND_URL", "http://localhost:8080"),
-        "https://accounts.google.com",
-        "https://www.googleapis.com/auth/calendar",
-    ],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
+
+def server():
+    """
+    Main function for the fast api server
+    """
+    # extra routes
+    from .routes import api
+    from .routes import account
+    from .routes import google
+    from .routes import schedule
+
+    # init app
+    app = FastAPI()
+
+    # allow requests from own frontend running on a different port
+    app.add_middleware(
+        CORSMiddleware,
+        # Work around for now :)
+        allow_origins=[
+            os.getenv("FRONTEND_URL", "http://localhost:8080"),
+            "https://accounts.google.com",
+            "https://www.googleapis.com/auth/calendar",
+        ],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    @app.exception_handler(RefreshError)
+    async def catch_google_refresh_errors(request, exc):
+        """Catch google refresh errors, and use our error instead."""
+        return await http_exception_handler(request, APIGoogleRefreshError())
+
+    # Mix in our extra routes
+    app.include_router(api.router)
+    app.include_router(account.router, prefix="/account")
+    app.include_router(google.router, prefix="/google")
+    app.include_router(schedule.router, prefix="/schedule")
+
+    return app
 
 
-@app.exception_handler(RefreshError)
-async def catch_google_refresh_errors(request, exc):
-    """Catch google refresh errors, and use our error instead."""
-    return await http_exception_handler(request, APIGoogleRefreshError())
+def cli():
+    """
+    A very simple cli handler
+    """
+    if len(sys.argv) < 2:
+        print("No command specified")
+        return
 
+    command = sys.argv[1:]
 
-# Mix in our extra routes
-app.include_router(api.router)
-app.include_router(account.router, prefix="/account")
-app.include_router(google.router, prefix="/google")
-app.include_router(schedule.router, prefix="/schedule")
+    if command[0] == 'update-db':
+        from .commands import update_db
+        update_db.run()

backend/src/migrations/env.py → backend/src/appointment/migrations/env.py

@@ -9,6 +9,8 @@
 # This is ran from src/ so ignore the errors
 from secrets import normalize_secrets
 
+import sentry_sdk
+
 # Normalize any AWS secrets
 normalize_secrets()
 
@@ -32,6 +34,17 @@
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
+# Catch any errors that may run during migrations
+if os.getenv("SENTRY_DSN") != "" or os.getenv("SENTRY_DSN") is not None:
+    sentry_sdk.init(
+        dsn=os.getenv("SENTRY_DSN"),
+        # Set traces_sample_rate to 1.0 to capture 100%
+        # of transactions for performance monitoring.
+        # We recommend adjusting this value in production,
+        traces_sample_rate=1.0,
+        environment=os.getenv("APP_ENV", "dev"),
+    )
+
 
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.

backend/src/migrations/versions/2023_07_27_1102-f9c5471478d0_modify_schedules_table.py → backend/src/appointment/migrations/versions/2023_07_27_1102-f9c5471478d0_modify_schedules_table.py

@@ -26,8 +26,8 @@ def secret():
 
 
 def upgrade() -> None:
-    op.drop_column("appointments", "appointment_type")
-    op.drop_constraint("schedules_ibfk_1", "schedules", type_="foreignkey")
+    # op.drop_column("appointments", "appointment_type")
+    #op.drop_constraint("schedules_ibfk_1", "schedules", type_="foreignkey")
     op.drop_column("schedules", "appointment_id")
     op.add_column("schedules", sa.Column("calendar_id", sa.Integer, sa.ForeignKey("calendars.id")))
     op.add_column("schedules", sa.Column("location_type", sa.Enum(LocationType), default=LocationType.online))

docker-compose.yml

@@ -1,11 +1,13 @@
 services:
 
   backend:
-    build: ./backend
+    build:
+      context: ./backend
+      dockerfile: ./Dockerfile
     ports:
       - 8090:8090
     volumes:
-      - ./backend:/app
+      - ./backend/src:/app/src
     environment:
       - DATABASE_URL=mysql+mysqldb://tba:tba@mysql:3306/appointment
     depends_on:

frontend/docker/etc/nginx/conf.d/appointments.conf

@@ -6,7 +6,7 @@ server {
     #access_log  /var/log/nginx/host.access.log  main;
 
     # Backend API proxy
-    location /api/v1 {
+    location ^~ /api/v1/ {
         # Remove our fake /api/v1/ prefix for FastAPI
         rewrite ^/api/v1/(.*)$ /$1 break;
         proxy_pass http://127.0.0.1:5000;

frontend/src/components/ScheduleCreation.vue

@@ -480,7 +480,7 @@ const saveSchedule = async (withConfirmation = true) => {
   // save schedule data
   const { data, error } = props.schedule
     ? await call(`schedule/${props.schedule.id}`).put(obj).json()
-    : await call("schedule").post(obj).json();
+    : await call("schedule/").post(obj).json();
 
   if (error.value) {
     // error message is in data

frontend/src/views/ScheduleView.vue

@@ -165,7 +165,8 @@ const firstSchedule = computed(() => schedules.value.length > 0
 const schedulesReady = ref(false);
 const getFirstSchedule = async () => {
   calendarEvents.value = [];
-  const { data } = await call('schedule').get().json();
+  // trailing slash to prevent fast api redirect which doesn't work great on our container setup
+  const { data } = await call('schedule/').get().json();
   schedules.value = data.value;
 };