Improve UI tests by running each in isolation (separate Tobira proces…

…s & DB) with fixed offline data (#1093)

This should improve our UI tests by isolating them, making them more
robust and easier to write. Each test (well, worker) has its own Tobira
process and DB. The DB test data is now fixed and statically specified
and all referenced static files are served locally, contained inside the
repository. That way we are not dependent on online resources or the
ever-changing DB dumps. Tests can now assume this fixed dataset (only a
few tests take advantage of that yet).

See commits for more details on how this is achieved. This PR should
likely be reviewed commit by commit anyway.

All tests that previously worked, work with this as well. I also
re-enabled a large part of the `realm.spec.ts` test, which I think runs
reliably now. This still leaves some tests disabled/skipped that we
should re-enable as well. And there are of course tons of new tests one
could add. But this PR was took long enough already and presents an
improvement in itself IMO. So the other tests can be added in follow up
PRs.

This PR also adds the `"@typescript-eslint/no-floating-promises":
"warn",` lint, which helps a lot with writing these tests.

.gitattributes

@@ -0,0 +1,2 @@
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -152,6 +152,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        lfs: true
     - uses: actions/download-artifact@v4
       with:
         name: test-deployment-files
@@ -213,22 +215,18 @@ jobs:
       run: |
         docker-compose -f docker-compose.yml up -d \
           tobira-login-handler \
-          tobira-meilisearch
-
-    - name: Rebuild search index
-      run: ./tobira search-index update --config util/dev-config/config.toml
-
+          tobira-meilisearch \
+          tobira-ui-test-files
+    - name: Link Tobira binary to location expected by Playwright tests
+      run: |
+        mkdir -p backend/target/debug/
+        ln -s "$(pwd)/tobira" backend/target/debug/tobira
     - name: Install Playwright
       working-directory: frontend
       run: npm i @playwright/test
     - name: Install Playwright browsers
       working-directory: frontend
       run: npx playwright install --with-deps
-    - name: Start Tobira for playwright tests
-      uses: JarvusInnovations/background-action@v1
-      with:
-        run: ./tobira serve --config util/dev-config/config.toml &
-        wait-on: http://localhost:3080
     - name: Run playwright tests
       working-directory: frontend
       run: npx playwright test

backend/src/db/cmd.rs

@@ -252,6 +252,20 @@ async fn clear(db: &mut Db, config: &Config, yes: bool) -> Result<()> {
     tx.commit().await.context("failed to commit clear transaction")?;
     info!("Dropped everything inside schema '{schema}' of the database");
 
+    // Try to disconnect all active connections. This is useful as those might
+    // hold cached statements that are now invalid.
+    let db_name = &config.db.database;
+    let sql = format!("
+        select pg_terminate_backend(pg_stat_activity.pid)
+        from pg_stat_activity
+        where pg_stat_activity.datname = '{db_name}'
+            and pid <> pg_backend_pid();"
+    );
+    match db.execute(&sql, &[]).await {
+        Ok(_) => info!("Disconnected all existing connection to this database"),
+        Err(e) => warn!("Could not disconnect all active connections: {e}"),
+    }
+
 
     // ### Step 4: Also clear the search index ###
     let meili = config.meili.connect().await?;

docs/docs/dev/create-release.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 7
+sidebar_position: 8
 ---
 
 # Create a release

docs/docs/dev/tests.md

@@ -0,0 +1,38 @@
+---
+sidebar_position: 7
+---
+
+# Tests
+
+Tobira comes with various different automated tests to prevent bugs or unintended changes.
+These are all automatically run by our GitHub-based continious integration (CI) for every pull request.
+But it is also useful to run the tests locally, for example to add new tests.
+
+
+## Backend & DB tests
+
+These are tests in `backend/`, written with the built-in Rust test framework, i.e. `#[test]`.
+You can find various ones throughout the backend codebase.
+
+There are some very simple tests that test a small piece of code and are completely isolated.
+But there are also "database tests" defined in the backend that make sure the DB with our migrations behaves as expected.
+These are defined in `db::tests`.
+Each DB test sets up an isolated new database and performs all tests inside so that tests running at the same time do not influence each other.
+
+You can run all of these tests via **`cargo test`** in the `backend/` folder.
+The DB tests require the development PostgreSQL database to be running, so make sure you ran `./x.sh containers start`.
+
+
+## Playwright UI tests
+
+These tests are "end to end" tests as they test the whole Tobira application in the same way a user would.
+We use [Playwright](https://playwright.dev/) to define those tests.
+They live in `frontend/tests`.
+
+In order for these tests to run without interfering with one another, each test is isolated, using its own Tobira binary and database.
+A small set of fixed test data can be inserted for the test and a number of small static files (videos, images) are available via a development container.
+
+Therefore, you also have to run `./x.sh containers start` before running these tests.
+Further, you have to run `npm ci` in `frontend/` and build the Tobira binary, both done by `./x.sh start`.
+Finally, in some situations, git might not have downloaded the static files properly, which you can fix by manually running `git lfs checkout` or `git lfs fetch`.
+With all that done, you can run the tests via `npx playwright test` inside `frontend/`.

frontend/.eslintrc.js

@@ -67,6 +67,30 @@ module.exports = {
                 project: false,
             },
         },
+
+        {
+            files: ["./tests/**/*"],
+            rules: {
+                // Playwright uses fixtures where only destructuring in the arg
+                // object already has an effect. We just ignore fixtures that
+                // can be used that way.
+                "@typescript-eslint/no-unused-vars": ["warn", {
+                    args: "all",
+                    varsIgnorePattern: "^_",
+                    argsIgnorePattern: "^_|standardData|activeSearchIndex",
+                    caughtErrors: "all",
+                    caughtErrorsIgnorePattern: "^_",
+                    ignoreRestSiblings: true,
+
+                }],
+
+                "no-empty-pattern": "off",
+
+                // Playwright tests use tons of async and it's easy to forget
+                // writing `await`, which leads to confusing test behavior.
+                "@typescript-eslint/no-floating-promises": "error",
+            },
+        },
     ],
 
     ignorePatterns: [

frontend/package.json

@@ -80,8 +80,10 @@
     "eslint-plugin-react": "^7.34.0",
     "eslint-plugin-react-hooks": "^4.6.0",
     "fork-ts-checker-webpack-plugin": "^6.5.3",
+    "postgres": "^3.4.3",
     "schema-dts": "^1.1.2",
     "ts-node": "^10.9.2",
-    "typescript": "~5.1.6"
+    "typescript": "~5.1.6",
+    "wait-port": "^1.1.0"
   }
 }

frontend/playwright.config.ts

@@ -4,12 +4,10 @@ import { defineConfig, devices } from "@playwright/test";
 export default defineConfig({
     testDir: "./tests",
     workers: process.env.CI ? 1 : undefined,
-    retries: 1,
+    retries: 0, // TODO
     reporter: "html",
-    expect: { timeout: 20 * 1000 },
 
     use: {
-        baseURL: "http://localhost:3080",
         headless: true,
         locale: "en",
         trace: "retain-on-failure",
@@ -32,10 +30,4 @@ export default defineConfig({
             use: { ...devices["Desktop Safari"] },
         },
     ],
-
-    webServer: {
-        command: "cargo run --manifest-path=../backend/Cargo.toml -- serve",
-        url: "http://localhost:3080",
-        reuseExistingServer: true,
-    },
 });

frontend/src/i18n/locales/de.yaml

@@ -502,7 +502,7 @@ manage:
         den Seiteneinstellungen ändern.
 
       title:
-        content: Hier können Sie Ihren Titel einfügen.
+        content: Titel
 
       text:
         content: Hier können Sie Ihren Text einfügen. Sie können auch Markdown verwenden.

frontend/src/i18n/locales/en.yaml

@@ -478,7 +478,7 @@ manage:
         the page settings.
 
       title:
-        content: You can put your title here.
+        content: Title
 
       text:
         content: You can add your text content here. You can also use Markdown.