From ae538314b47850ff02a043ef1dbcd0557eeaac7a Mon Sep 17 00:00:00 2001 From: Gustl22 Date: Sun, 1 Sep 2024 21:55:23 +0200 Subject: [PATCH] feat: Database migration (closes #29) --- .../PostgreSQL-wrestling_scoreboard-dump.sql | 24 +++++++- .../v0.0.1-beta.14_Initial-migration.sql | 59 +++++++++++++++++++ .../lib/controllers/auth_controller.dart | 8 +-- .../lib/controllers/database_controller.dart | 48 ++++++++++++--- .../lib/controllers/websocket_handler.dart | 4 +- .../lib/middleware/cors.dart | 4 +- wrestling_scoreboard_server/lib/server.dart | 31 ++-------- .../lib/services/auth.dart | 9 ++- .../lib/services/environment.dart | 38 ++++++++++++ .../lib/services/postgres_db.dart | 13 ++-- .../lib/services/pubspec.dart | 17 ++++++ wrestling_scoreboard_server/pubspec.lock | 2 +- wrestling_scoreboard_server/pubspec.yaml | 1 + 13 files changed, 204 insertions(+), 54 deletions(-) create mode 100644 wrestling_scoreboard_server/database/migration/v0.0.1-beta.14_Initial-migration.sql create mode 100644 wrestling_scoreboard_server/lib/services/environment.dart create mode 100644 wrestling_scoreboard_server/lib/services/pubspec.dart diff --git a/wrestling_scoreboard_server/database/dump/PostgreSQL-wrestling_scoreboard-dump.sql b/wrestling_scoreboard_server/database/dump/PostgreSQL-wrestling_scoreboard-dump.sql index 62e7ef44..85dfae93 100644 --- a/wrestling_scoreboard_server/database/dump/PostgreSQL-wrestling_scoreboard-dump.sql +++ b/wrestling_scoreboard_server/database/dump/PostgreSQL-wrestling_scoreboard-dump.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 13.6 (Ubuntu 13.6-0ubuntu0.21.10.1) --- Dumped by pg_dump version 16.4 (Ubuntu 16.4-0ubuntu0.24.04.1) +-- Dumped from database version 16.1 +-- Dumped by pg_dump version 16.1 SET statement_timeout = 0; SET lock_timeout = 0; @@ -679,6 +679,17 @@ ALTER SEQUENCE public.membership_id_seq OWNER TO wrestling; ALTER SEQUENCE public.membership_id_seq OWNED BY public.membership.id; +-- +-- Name: migration; Type: TABLE; Schema: public; Owner: wrestling +-- + +CREATE TABLE public.migration ( + semver character varying(127) DEFAULT '0.0.0'::character varying NOT NULL +); + + +ALTER TABLE public.migration OWNER TO wrestling; + -- -- Name: organization; Type: TABLE; Schema: public; Owner: wrestling -- @@ -1397,6 +1408,15 @@ COPY public.membership (id, person_id, club_id, no, org_sync_id, organization_id \. +-- +-- Data for Name: migration; Type: TABLE DATA; Schema: public; Owner: wrestling +-- + +COPY public.migration (semver) FROM stdin; +0.0.1-beta.14 +\. + + -- -- Data for Name: organization; Type: TABLE DATA; Schema: public; Owner: wrestling -- diff --git a/wrestling_scoreboard_server/database/migration/v0.0.1-beta.14_Initial-migration.sql b/wrestling_scoreboard_server/database/migration/v0.0.1-beta.14_Initial-migration.sql new file mode 100644 index 00000000..56850d7d --- /dev/null +++ b/wrestling_scoreboard_server/database/migration/v0.0.1-beta.14_Initial-migration.sql @@ -0,0 +1,59 @@ +create table migration +( + semver varchar(127) default '0.0.0'::character varying not null +); +alter table migration owner to wrestling; +INSERT INTO public.migration (semver) VALUES ('0.0.0'); + +CREATE TYPE public.user_privilege AS ENUM ( + 'none', + 'read', + 'write', + 'admin' +); +ALTER TYPE public.user_privilege OWNER TO wrestling; + +alter table public.bout add organization_id integer; + +alter table public.division_weight_class add organization_id integer; +alter table public.division_weight_class add org_sync_id varchar(127); + +alter table public.league add bout_days integer DEFAULT 14 NOT NULL; + +CREATE TABLE public.secured_user ( + id integer NOT NULL, + username character varying(127) NOT NULL, + password_hash bytea NOT NULL, + email character varying(127), + person_id integer, + salt character varying(127) NOT NULL, + created_at date NOT NULL, + privilege public.user_privilege DEFAULT 'none'::public.user_privilege NOT NULL +); +ALTER TABLE public.secured_user OWNER TO wrestling; + +CREATE SEQUENCE public.user_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE public.user_id_seq OWNER TO wrestling; +ALTER SEQUENCE public.user_id_seq OWNED BY public.secured_user.id; +ALTER TABLE ONLY public.secured_user ALTER COLUMN id SET DEFAULT nextval('public.user_id_seq'::regclass); + +COPY public.secured_user (id, username, password_hash, email, person_id, salt, created_at, privilege) FROM stdin; +1 admin \\xb2950268d52c1d17f1b35edd35c071be3d320b488c81425c6b144340963e524a \N 924VOg== 2024-08-25 admin +\. + +SELECT pg_catalog.setval('public.user_id_seq', 1, true); + +ALTER TABLE ONLY public.secured_user ADD CONSTRAINT user_pk PRIMARY KEY (id); +ALTER TABLE ONLY public.secured_user ADD CONSTRAINT user_pk_2 UNIQUE (username); + +ALTER TABLE ONLY public.bout ADD CONSTRAINT bout_organization_id_fk FOREIGN KEY (organization_id) REFERENCES public.organization(id); +ALTER TABLE ONLY public.division_weight_class ADD CONSTRAINT division_weight_class_organization_id_fk FOREIGN KEY (organization_id) REFERENCES public.organization(id); + +ALTER TABLE ONLY public.secured_user ADD CONSTRAINT user_person_id_fk FOREIGN KEY (person_id) REFERENCES public.person(id) ON DELETE CASCADE; + diff --git a/wrestling_scoreboard_server/lib/controllers/auth_controller.dart b/wrestling_scoreboard_server/lib/controllers/auth_controller.dart index 219f793c..3263c424 100644 --- a/wrestling_scoreboard_server/lib/controllers/auth_controller.dart +++ b/wrestling_scoreboard_server/lib/controllers/auth_controller.dart @@ -4,13 +4,13 @@ import 'package:dart_jsonwebtoken/dart_jsonwebtoken.dart'; import 'package:shelf/shelf.dart'; import 'package:wrestling_scoreboard_common/common.dart'; import 'package:wrestling_scoreboard_server/controllers/user_controller.dart'; -import 'package:wrestling_scoreboard_server/server.dart'; +import 'package:wrestling_scoreboard_server/services/environment.dart'; class AuthController { late final String jwtSecret; AuthController() { - final jwtSecret = env['JWT_SECRET']; + final jwtSecret = env.jwtSecret; if (jwtSecret == null) { throw Exception('JWT_SECRET not specified!'); } @@ -66,8 +66,8 @@ class AuthController { } String _signToken(SecuredUser user) { - final int jwtExpiresInDays = int.parse(env['JWT_EXPIRES_IN_DAYS'] ?? '90'); - final jwtIssuer = env['JWT_ISSUER'] ?? 'Wrestling Scoreboard (oberhauser.dev)'; + final int jwtExpiresInDays = env.jwtExpiresInDays ?? 90; + final jwtIssuer = env.jwtIssuer ?? 'Wrestling Scoreboard (oberhauser.dev)'; final jwt = JWT( // Payload diff --git a/wrestling_scoreboard_server/lib/controllers/database_controller.dart b/wrestling_scoreboard_server/lib/controllers/database_controller.dart index 9e8abe22..d62e2337 100644 --- a/wrestling_scoreboard_server/lib/controllers/database_controller.dart +++ b/wrestling_scoreboard_server/lib/controllers/database_controller.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:pub_semver/pub_semver.dart'; import 'package:shelf/shelf.dart'; import 'package:wrestling_scoreboard_common/common.dart'; import 'package:wrestling_scoreboard_server/controllers/user_controller.dart'; @@ -8,7 +9,34 @@ import 'package:wrestling_scoreboard_server/services/postgres_db.dart'; import 'entity_controller.dart'; class DatabaseController { - // TODO: migration should be handled automatically at server start. + Future migrate() async { + final db = PostgresDb(); + + final res = await db.connection.execute('SELECT semver FROM migration LIMIT 1;'); + final row = res.singleOrNull; + final String semver = row?.toColumnMap()['semver'] ?? '0.0.0'; + final databaseVersion = Version.parse(semver); + + final dir = Directory('database/migration'); + final List entities = await dir.list().toList(); + final migrationMap = entities.map((entity) { + final migrationVersion = entity.uri.pathSegments.last.split('_')[0]; + return MapEntry(Version.parse(migrationVersion.replaceFirst('v', '')), entity); + }).toList(); + migrationMap.sort((a, b) => a.key.compareTo(b.key)); + int migrationStartIndex = 0; + while (migrationStartIndex < migrationMap.length) { + if (databaseVersion.compareTo(migrationMap[migrationStartIndex].key) < 0) { + break; + } + migrationStartIndex++; + } + final migrationRange = migrationMap.sublist(migrationStartIndex, migrationMap.length); + for (var migration in migrationRange) { + _executeSqlFile(migration.value.path); + } + await db.connection.execute("UPDATE migration SET semver = '${migrationRange.last.key.toString()}';"); + } /// Reset all tables Future reset(Request request, User? user) async { @@ -17,7 +45,10 @@ class DatabaseController { Iterable entityControllers = dataTypes.map((t) => ShelfController.getControllerFromDataType(t)); // Remove data await Future.forEach(entityControllers, (e) => e.deleteMany()); - // Create default admin + // No need to restore the database semantic version for migration, + // as it is no entity controller and therefore keeps it default value. + + // Recreate default admin await SecuredUserController().createSingle( User(username: 'admin', createdAt: DateTime.now(), privilege: UserPrivilege.admin, password: 'admin') .toSecuredUser()); @@ -82,14 +113,17 @@ class DatabaseController { Future _restore(String dumpPath) async { final db = PostgresDb(); - { - await db.connection.execute('DROP SCHEMA IF EXISTS public CASCADE;'); - await db.close(); - } + await db.connection.execute('DROP SCHEMA IF EXISTS public CASCADE;'); + await _executeSqlFile(dumpPath); + } + + Future _executeSqlFile(String sqlFilePath) async { + final db = PostgresDb(); + await db.close(); final args = [ '--file', - dumpPath, + sqlFilePath, '--username', db.dbUser, '--host', diff --git a/wrestling_scoreboard_server/lib/controllers/websocket_handler.dart b/wrestling_scoreboard_server/lib/controllers/websocket_handler.dart index eb4d6e8d..1793e3dd 100644 --- a/wrestling_scoreboard_server/lib/controllers/websocket_handler.dart +++ b/wrestling_scoreboard_server/lib/controllers/websocket_handler.dart @@ -14,8 +14,8 @@ import 'package:wrestling_scoreboard_server/controllers/participation_controller import 'package:wrestling_scoreboard_server/controllers/team_controller.dart'; import 'package:wrestling_scoreboard_server/controllers/team_match_bout_controller.dart'; import 'package:wrestling_scoreboard_server/controllers/team_match_controller.dart'; -import 'package:wrestling_scoreboard_server/server.dart'; import 'package:wrestling_scoreboard_server/services/auth.dart'; +import 'package:wrestling_scoreboard_server/services/environment.dart'; import 'entity_controller.dart'; import 'league_team_participation_controller.dart'; @@ -419,5 +419,5 @@ final websocketHandler = webSocketHandler( webSocketPool.remove(webSocket); }); }, - pingInterval: Duration(seconds: int.parse(env['WEB_SOCKET_PING_INTERVAL_SECS'] ?? '30')), + pingInterval: Duration(seconds: env.webSocketPingIntervalSecs ?? 30), ); diff --git a/wrestling_scoreboard_server/lib/middleware/cors.dart b/wrestling_scoreboard_server/lib/middleware/cors.dart index 945fb95a..929f1399 100644 --- a/wrestling_scoreboard_server/lib/middleware/cors.dart +++ b/wrestling_scoreboard_server/lib/middleware/cors.dart @@ -1,8 +1,8 @@ -import 'package:wrestling_scoreboard_server/server.dart'; import 'package:shelf/shelf.dart'; +import 'package:wrestling_scoreboard_server/services/environment.dart'; final corsHeaders = { - 'Access-Control-Allow-Origin': env['CORS_ALLOW_ORIGIN'] ?? '*', + 'Access-Control-Allow-Origin': env.corsAllowOrigin ?? '*', 'Access-Control-Allow-Methods': 'GET, POST, DELETE, OPTIONS, PUT', 'Access-Control-Allow-Headers': '*', }; diff --git a/wrestling_scoreboard_server/lib/server.dart b/wrestling_scoreboard_server/lib/server.dart index 0cc769ca..b133b773 100644 --- a/wrestling_scoreboard_server/lib/server.dart +++ b/wrestling_scoreboard_server/lib/server.dart @@ -4,42 +4,23 @@ import 'dart:io'; -import 'package:dotenv/dotenv.dart' show DotEnv; import 'package:logging/logging.dart'; -import 'package:pubspec_parse/pubspec_parse.dart'; import 'package:shelf/shelf.dart'; import 'package:shelf/shelf_io.dart' as shelf_io; import 'package:shelf_router/shelf_router.dart' as shelf_router; import 'package:shelf_static/shelf_static.dart' as shelf_static; import 'package:wrestling_scoreboard_server/controllers/websocket_handler.dart'; +import 'package:wrestling_scoreboard_server/services/environment.dart'; +import 'package:wrestling_scoreboard_server/services/pubspec.dart'; import 'package:wrestling_scoreboard_server/routes/api_route.dart'; import 'package:wrestling_scoreboard_server/services/postgres_db.dart'; import 'middleware/cors.dart'; -final env = DotEnv(); - -Pubspec? pubspec; - -Future _parsePubspec() async { - if (pubspec == null) { - final file = File('pubspec.yaml'); - if (await file.exists()) { - pubspec = Pubspec.parse(await file.readAsString()); - } else { - throw FileSystemException('No file found', file.absolute.path); - } - } - return pubspec!; -} - Future init() async { - env.load(); // Load dotenv variables - await _parsePubspec(); // Init logger - Logger.root.level = - Level.LEVELS.where((level) => level.name == env['LOG_LEVEL']?.toUpperCase()).singleOrNull ?? Level.INFO; + Logger.root.level = env.logLevel ?? Level.INFO; Logger.root.onRecord.listen( (record) async { String text = '[${record.time}] ${record.level.name}: ${record.message}'; @@ -64,7 +45,7 @@ Future init() async { // If the "PORT" environment variable is set, listen to it. Otherwise, 8080. // https://cloud.google.com/run/docs/reference/container-contract#port - final port = int.parse(env['PORT'] ?? '8080'); + final port = env.port ?? 8080; // Must open the database before initializing any routes. await PostgresDb().open(); @@ -89,7 +70,7 @@ Future init() async { } }) ..mount('/about', (Request request) async { - final pubspec = await _parsePubspec(); + final pubspec = await parsePubspec(); return Response.ok(''' Name: ${pubspec.name} Description: ${pubspec.description} @@ -117,7 +98,7 @@ Future init() async { // See https://pub.dev/documentation/shelf/latest/shelf_io/serve.html final server = await shelf_io.serve( pipeline, - env['HOST'] ?? InternetAddress.anyIPv4, // Allows external connections + env.host ?? InternetAddress.anyIPv4, // Allows external connections port, ); diff --git a/wrestling_scoreboard_server/lib/services/auth.dart b/wrestling_scoreboard_server/lib/services/auth.dart index 05934696..13bb9715 100644 --- a/wrestling_scoreboard_server/lib/services/auth.dart +++ b/wrestling_scoreboard_server/lib/services/auth.dart @@ -2,7 +2,8 @@ import 'package:dart_jsonwebtoken/dart_jsonwebtoken.dart'; import 'package:shelf/shelf.dart'; import 'package:wrestling_scoreboard_common/common.dart'; import 'package:wrestling_scoreboard_server/controllers/user_controller.dart'; -import 'package:wrestling_scoreboard_server/server.dart'; + +import 'environment.dart'; extension AuthRequest on Request { Future restricted({ @@ -37,14 +38,12 @@ extension AuthRequest on Request { } } -final jwtSecret = env['JWT_SECRET']; - extension UserFromAuthService on BearerAuthService { Future getUser() async { - if (jwtSecret == null) { + if (env.jwtSecret == null) { throw Exception('JWT_SECRET not specified!'); } - final jwt = JWT.verify(token, SecretKey(jwtSecret!)); + final jwt = JWT.verify(token, SecretKey(env.jwtSecret!)); final user = await SecuredUserController().getSingleByUsername(jwt.payload['username'] as String); return user?.toUser(); } diff --git a/wrestling_scoreboard_server/lib/services/environment.dart b/wrestling_scoreboard_server/lib/services/environment.dart new file mode 100644 index 00000000..157741cd --- /dev/null +++ b/wrestling_scoreboard_server/lib/services/environment.dart @@ -0,0 +1,38 @@ +import 'package:dotenv/dotenv.dart' show DotEnv; +import 'package:logging/logging.dart'; + +final env = Environment(); + +class Environment { + late final Level? logLevel; + late final String? host; + late final int? port; + late final String? jwtSecret; + late final int? jwtExpiresInDays; + late final String? jwtIssuer; + late final String? databaseHost; + late final int? databasePort; + late final String? databaseUser; + late final String? databasePassword; + late final String? databaseName; + late final String? corsAllowOrigin; + late final int? webSocketPingIntervalSecs; + + Environment() { + final dotEnv = DotEnv(); + dotEnv.load(); // Load dotenv variables + logLevel = Level.LEVELS.where((level) => level.name == dotEnv['LOG_LEVEL']?.toUpperCase()).singleOrNull; + host = dotEnv['HOST']; + port = int.tryParse(dotEnv['PORT'] ?? ''); + jwtSecret = dotEnv['JWT_SECRET']; + jwtExpiresInDays = int.tryParse(dotEnv['JWT_EXPIRES_IN_DAYS'] ?? ''); + jwtIssuer = dotEnv['JWT_ISSUER']; + databaseHost = dotEnv['DATABASE_HOST']; + databasePort = int.tryParse(dotEnv['DATABASE_PORT'] ?? ''); + databaseUser = dotEnv['DATABASE_USER']; + databasePassword = dotEnv['DATABASE_PASSWORD']; + databaseName = dotEnv['DATABASE_NAME']; + corsAllowOrigin = dotEnv['CORS_ALLOW_ORIGIN']; + webSocketPingIntervalSecs = int.tryParse(dotEnv['WEB_SOCKET_PING_INTERVAL_SECS'] ?? ''); + } +} diff --git a/wrestling_scoreboard_server/lib/services/postgres_db.dart b/wrestling_scoreboard_server/lib/services/postgres_db.dart index e42ceca0..22e098cc 100644 --- a/wrestling_scoreboard_server/lib/services/postgres_db.dart +++ b/wrestling_scoreboard_server/lib/services/postgres_db.dart @@ -1,14 +1,15 @@ import 'package:postgres/postgres.dart' as psql; -import 'package:wrestling_scoreboard_server/server.dart'; + +import 'environment.dart'; const _isReleaseMode = bool.fromEnvironment("dart.vm.product"); class PostgresDb { - final String postgresHost = env['DATABASE_HOST'] ?? 'localhost'; - final int postgresPort = int.parse(env['DATABASE_PORT'] ?? '5432'); - final String dbUser = env['DATABASE_USER'] ?? 'postgres'; - final String dbPW = env['DATABASE_PASSWORD'] ?? ''; - final String postgresDatabaseName = env['DATABASE_NAME'] ?? 'wrestling_scoreboard'; + final String postgresHost = env.databaseHost ?? 'localhost'; + final int postgresPort = env.databasePort ?? 5432; + final String dbUser = env.databaseUser ?? 'postgres'; + final String dbPW = env.databasePassword ?? ''; + final String postgresDatabaseName = env.databaseName ?? 'wrestling_scoreboard'; static final PostgresDb _singleton = PostgresDb._internal(); late psql.Connection connection; diff --git a/wrestling_scoreboard_server/lib/services/pubspec.dart b/wrestling_scoreboard_server/lib/services/pubspec.dart new file mode 100644 index 00000000..c7053379 --- /dev/null +++ b/wrestling_scoreboard_server/lib/services/pubspec.dart @@ -0,0 +1,17 @@ +import 'dart:io'; + +import 'package:pubspec_parse/pubspec_parse.dart'; + +Pubspec? pubspec; + +Future parsePubspec() async { + if (pubspec == null) { + final file = File('pubspec.yaml'); + if (await file.exists()) { + pubspec = Pubspec.parse(await file.readAsString()); + } else { + throw FileSystemException('No file found', file.absolute.path); + } + } + return pubspec!; +} diff --git a/wrestling_scoreboard_server/pubspec.lock b/wrestling_scoreboard_server/pubspec.lock index 43115579..6dd1378d 100644 --- a/wrestling_scoreboard_server/pubspec.lock +++ b/wrestling_scoreboard_server/pubspec.lock @@ -351,7 +351,7 @@ packages: source: hosted version: "3.2.1" pub_semver: - dependency: transitive + dependency: "direct main" description: name: pub_semver sha256: "40d3ab1bbd474c4c2328c91e3a7df8c6dd629b79ece4c4bd04bee496a224fb0c" diff --git a/wrestling_scoreboard_server/pubspec.yaml b/wrestling_scoreboard_server/pubspec.yaml index f7cce030..e6e2ef59 100644 --- a/wrestling_scoreboard_server/pubspec.yaml +++ b/wrestling_scoreboard_server/pubspec.yaml @@ -21,6 +21,7 @@ dependencies: collection: ^1.19.0 logging: ^1.2.0 dart_jsonwebtoken: ^2.14.0 + pub_semver: ^2.1.4 dev_dependencies: lints: ^3.0.0