fix: migrate ormconfig/connection to datasource

TODO: test setup is still broken
virtualcommons · Feb 2, 2024 · 4c5ce1c · 4c5ce1c
1 parent 3593add
commit 4c5ce1c
Show file tree

Hide file tree

Showing 25 changed files with 149 additions and 152 deletions.
diff --git a/Makefile b/Makefile
@@ -7,15 +7,14 @@ DATA_DUMP_PATH=docker/dump
 LOG_DATA_PATH=docker/logs
 DB_PASSWORD_PATH=keys/pom_db_password
 REDIS_SETTINGS_PATH=keys/settings.json
-ORMCONFIG_PATH=keys/ormconfig.json
 SERVER_ENV_TEMPLATE=server/.env.template
 SERVER_ENV=server/.env
 PGPASS_PATH=keys/.pgpass
 SECRET_KEY_PATH=keys/secret_key
 SENTRY_DSN_PATH=keys/sentry_dsn
 SENTRY_DSN=$(shell cat $(SENTRY_DSN_PATH))
 MAIL_API_KEY_PATH=keys/mail_api_key
-SECRETS=$(MAIL_API_KEY_PATH) $(DB_PASSWORD_PATH) $(ORMCONFIG_PATH) $(PGPASS_PATH) $(SENTRY_DSN_PATH) $(SECRET_KEY_PATH)
+SECRETS=$(MAIL_API_KEY_PATH) $(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SENTRY_DSN_PATH) $(SECRET_KEY_PATH)
 SHARED_CONFIG_PATH=shared/src/assets/config.ts
 BUILD_ID=$(shell git describe --tags --abbrev=1)
 GA_TAG_PATH=keys/ga_tag
@@ -68,12 +67,9 @@ $(DATA_DUMP_PATH):
 $(REDIS_SETTINGS_PATH): server/deploy/settings.template.json | keys
 	cp server/deploy/settings.template.json $(REDIS_SETTINGS_PATH)
 
-$(ORMCONFIG_PATH): server/ormconfig.template.json $(DB_PASSWORD_PATH)
-	DB_PASSWORD=$$(cat $(DB_PASSWORD_PATH)); \
-	sed "s|DB_PASSWORD|$$DB_PASSWORD|g" server/ormconfig.template.json > $(ORMCONFIG_PATH)
-
 $(SERVER_ENV): $(SERVER_ENV_TEMPLATE) $(SECRETS)
 	POM_BASE_URL=${POM_BASE_URL} \
+	DB_PASSWORD=$$(cat $(DB_PASSWORD_PATH)); \
 		envsubst < $(SERVER_ENV_TEMPLATE) > $(SERVER_ENV)
 
 $(PGPASS_PATH): $(DB_PASSWORD_PATH) server/deploy/pgpass.template | keys
@@ -110,15 +106,15 @@ settings: $(SENTRY_DSN_PATH) $(SECRET_KEY_PATH) | keys
 initialize: build
 	docker compose run --rm server npm run initdb
 
-docker-compose.yml: base.yml $(ENVIR).yml config.mk $(DB_DATA_PATH) $(DATA_DUMP_PATH) $(LOG_DATA_PATH) $(REDIS_SETTINGS_PATH) $(ORMCONFIG_PATH) $(NUXT_ORMCONFIG_PATH) $(PGPASS_PATH) $(SERVER_ENV) settings
+docker-compose.yml: base.yml $(ENVIR).yml config.mk $(DB_DATA_PATH) $(DATA_DUMP_PATH) $(LOG_DATA_PATH) $(REDIS_SETTINGS_PATH) $(PGPASS_PATH) $(SERVER_ENV) settings
 	case "$(ENVIR)" in \
 	  dev|staging|prod) docker compose -f base.yml -f "$(ENVIR).yml" config > docker-compose.yml;; \
 	  *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \
 	esac
 
 .PHONY: test-setup
 test-setup: docker-compose.yml
-	docker compose run --rm server bash -c "dropdb --if-exists -h db -U ${DB_USER} ${TEST_DB_NAME} && createdb -h db -U ${DB_USER} ${TEST_DB_NAME} && npm run typeorm -- schema:sync -c test && npm run load-fixtures -- ./fixtures/sologame -cn test"
+	docker compose run --rm server bash -c "dropdb --if-exists -h db -U ${DB_USER} ${TEST_DB_NAME} && createdb -h db -U ${DB_USER} ${TEST_DB_NAME} && npm run typeorm -- schema:sync && npm run load-fixtures ./fixtures/sologame"
 
 .PHONY: test
 test: test-setup

diff --git a/base.yml b/base.yml
@@ -12,7 +12,6 @@ services:
     volumes:
       - ./docker/dump:/dump
       - ./docker/logs:/var/log/port-of-mars
-      - ./keys/ormconfig.json:/code/server/ormconfig.json
       - ./keys/.pgpass:/root/.pgpass
       - ./keys:/run/secrets
       - ./scripts:/scripts

diff --git a/server/.env.template b/server/.env.template
@@ -1,5 +1,7 @@
 BASE_URL=${POM_BASE_URL}
 
+DB_PASSWORD=${DB_PASSWORD}
+
 GOOGLE_CLIENT_ID=changeme
 GOOGLE_CLIENT_SECRET=changeme
 

diff --git a/server/ormconfig.template.json b/server/ormconfig.template.json
diff --git a/server/package.json b/server/package.json
@@ -12,7 +12,7 @@
     "cli:prod": "node -r tsconfig-paths/register src/cli.js",
     "start:prod": "node -r tsconfig-paths/register src/index.js | tee -a /var/log/port-of-mars/index.log",
     "start": "ts-node-dev -r tsconfig-paths/register src/index.ts | tee -a /var/log/port-of-mars/index.log",
-    "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js",
+    "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js --dataSource src/datasource.ts",
     "dangerously-dropdb": "dropdb -U marsmadness -h db port_of_mars",
     "initdb": "createdb -U marsmadness -h db port_of_mars ; npm run typeorm migration:run && /scripts/freeplay/setup.sh ; npm run load-fixtures ./fixtures/sologame",
     "dangerously-loaddb": "psql -h db -U marsmadness port_of_mars < pom-db.sql",
@@ -25,7 +25,7 @@
     "lint:fix": "eslint --fix -c .eslintrc.js ./ ../shared --ext .ts",
     "style": "prettier --config ../.prettierrc --check './**/*.ts' '../shared/**/*.ts'",
     "style:fix": "prettier --config ../.prettierrc --write './**/*.ts' '../shared/**/*.ts'",
-    "load-fixtures": "fixtures --config ormconfig.json --require ts-node/register --require tsconfig-paths/register"
+    "load-fixtures": "fixtures load  --require ts-node/register --require tsconfig-paths/register -d src/datasource.ts"
   },
   "author": "Center for Behavior, Institutions, and the Environment (https://cbie.asu.edu)",
   "license": "MIT",

diff --git a/server/src/cli.ts b/server/src/cli.ts
@@ -25,10 +25,11 @@ import {
 } from "@port-of-mars/server/entity";
 import { DYNAMIC_SETTINGS_PATH, RedisSettings } from "@port-of-mars/server/services/settings";
 import { generateUsername } from "@port-of-mars/server/util";
+import appDataSource from "@port-of-mars/server/datasource";
 
 import { program } from "commander";
 import { mkdir, readFile, writeFile } from "fs/promises";
-import { createConnection, EntityManager } from "typeorm";
+import { EntityManager } from "typeorm";
 /*
 import { promisify } from "util";
 
@@ -43,16 +44,6 @@ function getRandomInt(min: number, max: number) {
   return Math.floor(Math.random() * (max - min) + min);
 }
 
-async function withConnection<T>(f: (em: EntityManager) => Promise<T>): Promise<void> {
-  const conn = await createConnection("default");
-  const em = conn.createEntityManager();
-  try {
-    await f(em);
-  } finally {
-    await conn.close();
-  }
-}
-
 async function exportSoloData(em: EntityManager, start?: string, end?: string) {
   const soloGameService = getServices().sologame;
   await mkdir("/dump/solo", { recursive: true });
@@ -483,7 +474,9 @@ program
               .option("--tournamentId <tournamentId>", "ID of the tournament", customParseInt)
               .description("link existing Treatments to a given Tournament")
               .action(async cmd => {
-                await withConnection(em => addTreatments(em, cmd.treatmentIds, cmd.tournamentId));
+                await appDataSource.transaction(async em =>
+                  addTreatments(em, cmd.treatmentIds, cmd.tournamentId)
+                );
               })
           )
           .addCommand(
@@ -498,7 +491,7 @@ program
               .option("--tournamentId <tournamentId>", "ID of the tournament", customParseInt)
               .description("add a Treatment (set of mars event overrides) to a Tournament")
               .action(async cmd => {
-                await withConnection(em =>
+                await appDataSource.transaction(async em =>
                   createTournamentTreatment(
                     em,
                     cmd.name,
@@ -529,7 +522,7 @@ program
               )
               .description("add a TournamentRoundDate for the given date")
               .action(async cmd => {
-                await withConnection(em =>
+                await appDataSource.transaction(async em =>
                   createTournamentRoundDate(em, cmd.date, cmd.tournamentRoundId)
                 );
               })
@@ -544,7 +537,9 @@ program
               )
               .description("report emails for all users in the given tournament round")
               .action(async cmd => {
-                await withConnection(em => exportTournamentRoundEmails(em, cmd.tournamentRoundId));
+                await appDataSource.transaction(async em =>
+                  exportTournamentRoundEmails(em, cmd.tournamentRoundId)
+                );
               })
           )
           .addCommand(
@@ -567,7 +562,7 @@ program
               )
               .description("create invitations for the given users in the given tournament round")
               .action(async cmd => {
-                await withConnection(em =>
+                await appDataSource.transaction(async em =>
                   createTournamentRoundInvites(
                     em,
                     cmd.tournamentRoundId,
@@ -597,7 +592,7 @@ program
               .option("--announcement <announcement>", "Tournament Round announcement message", "")
               .description("create a tournament round")
               .action(async cmd => {
-                await withConnection(em =>
+                await appDataSource.transaction(async em =>
                   createRound(
                     em,
                     cmd.open,
@@ -627,7 +622,7 @@ program
           .option("--description <description>", "Description of the tournament")
           .description("create a tournament")
           .action(async cmd => {
-            await withConnection(em =>
+            await appDataSource.transaction(async em =>
               createTournament(
                 em,
                 cmd.tournamentName,
@@ -650,7 +645,7 @@ program
           .description("set a user as an administrator")
           .requiredOption("--username <username>", "username of the user")
           .action(async cmd => {
-            await withConnection(em => setAdminUser(em, cmd.username));
+            await appDataSource.transaction(async em => setAdminUser(em, cmd.username));
           })
       )
       .addCommand(
@@ -660,7 +655,9 @@ program
           .requiredOption("--startId <startUserId>", "initial user ID in range", customParseInt, 1)
           .requiredOption("--endId <endUserId>", "end user ID in range", customParseInt, 1942)
           .action(async cmd => {
-            await withConnection(em => anonymizeUsernames(em, cmd.startId, cmd.endId));
+            await appDataSource.transaction(async em =>
+              anonymizeUsernames(em, cmd.startId, cmd.endId)
+            );
           })
       )
   )
@@ -674,7 +671,7 @@ program
           .description("finalize a game that wasn't finalized properly")
           .requiredOption("--gameId <gameId>", "id of game", customParseInt)
           .action(async cmd => {
-            await withConnection(em => finalize(em, cmd.gameId));
+            await appDataSource.transaction(async em => finalize(em, cmd.gameId));
           })
       )
       .addCommand(
@@ -685,7 +682,7 @@ program
           )
           .requiredOption("--gameId <gameId>", "id of game", customParseInt)
           .action(async cmd => {
-            await withConnection(em => validate(em, cmd.gameId));
+            await appDataSource.transaction(async em => validate(em, cmd.gameId));
           })
       )
   )
@@ -701,7 +698,7 @@ program
           .description("dump game data for a given tournament round id to a pile of CSV files")
           .requiredOption("--tournamentId <tournamentId>", "tournament id", customParseInt)
           .action(async cmd => {
-            await withConnection(em => exportTournament(em, cmd.tournamentId));
+            await appDataSource.transaction(async em => exportTournament(em, cmd.tournamentId));
           })
       )
       .addCommand(
@@ -720,7 +717,9 @@ program
             [] as Array<number>
           )
           .action(async cmd => {
-            await withConnection(em => exportTournamentRound(em, cmd.tournamentId, cmd.gids));
+            await appDataSource.transaction(async em =>
+              exportTournamentRound(em, cmd.tournamentId, cmd.gids)
+            );
           })
       )
       .addCommand(
@@ -730,7 +729,7 @@ program
           .option("-s, --start <date>", "Start date (YYYY-MM-DD)")
           .option("-e, --end <date>", "End date (YYYY-MM-DD)")
           .action(async cmd => {
-            await withConnection(em => exportSoloData(em, cmd.start, cmd.end));
+            await appDataSource.transaction(async em => exportSoloData(em, cmd.start, cmd.end));
           })
       )
   )
@@ -745,7 +744,7 @@ program
         [] as Array<number>
       )
       .action(async cmd => {
-        await withConnection(em => checkQuizCompletion(em, cmd.ids));
+        await appDataSource.transaction(async em => checkQuizCompletion(em, cmd.ids));
       })
   )
   .addCommand(
@@ -759,7 +758,7 @@ program
         [] as Array<number>
       )
       .action(async cmd => {
-        await withConnection(em => completeQuizCompletion(em, cmd.ids));
+        await appDataSource.transaction(async em => completeQuizCompletion(em, cmd.ids));
       })
   )
   .addCommand(
@@ -776,7 +775,9 @@ program
         "generate a CSV for mailchimp import of all active users with a valid email address"
       )
       .action(async cmd => {
-        await withConnection(em => exportActiveEmails(em, cmd.after, cmd.enableAmdf));
+        await appDataSource.transaction(async em =>
+          exportActiveEmails(em, cmd.after, cmd.enableAmdf)
+        );
       })
   )
   .addCommand(
@@ -789,7 +790,7 @@ program
       )
       .description("Deactivate users who have unsubscribed from emails (currently from mailchimp).")
       .action(async cmd => {
-        await withConnection(em => deactivateUsers(em, cmd.filename));
+        await appDataSource.transaction(async em => deactivateUsers(em, cmd.filename));
       })
   )
   .addCommand(

diff --git a/server/src/datasource.ts b/server/src/datasource.ts
@@ -0,0 +1,29 @@
+import { DataSourceOptions, DataSource } from "typeorm";
+import * as dotenv from "dotenv";
+
+dotenv.config();
+
+const postgresConnectionOptions: DataSourceOptions = {
+  type: "postgres",
+  host: "db",
+  port: 5432,
+  username: "marsmadness",
+  password: process.env.DB_PASSWORD,
+  synchronize: false,
+  logging: false,
+  entities: ["src/entity/**/*.{js,ts}"],
+  migrations: ["src/migration/**/*.{js,ts}"],
+};
+
+const appDataSource = new DataSource({
+  ...postgresConnectionOptions,
+  database: "port_of_mars",
+});
+
+const testDataSource = new DataSource({
+  ...postgresConnectionOptions,
+  database: "pom_testing",
+});
+
+const dataSource = process.env.NODE_ENV === "test" ? testDataSource : appDataSource;
+export default dataSource;