chore(e2e): Harden e2e tests, add cleanup routine after all-tests run (…

…#1314)

## Description

This hopefully fixes an intermittent issue in e2e tests where leftovers
from previously running tests sometimes were not cleaned up properly,
and interfered with subsequent tests.

* Now using UpdatedAfter instead of ChangedAfter in default filter to
avoid clashing with other dialogs
* Using a new default synthetic person/organization
* Added missing cleanups for some tests
* Added a sentinel mechanism that detects if some tests aren't cleaning
up after themselves. The remaining dialogs will be attempted removed,
but the run will be failed as a whole
* Updated README
* Cleanups

## Verification

- [x] **Your** code builds clean without any errors or warnings
- [x] Manual testing done (required)
- [x] Relevant automated test added

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **Documentation**
- Enhanced clarity in the testing framework documentation, emphasizing
self-contained test files and cleanup requirements.
  
- **New Features**
	- Introduced a `sentinelValue` for tracking unpurged dialogs.
- Added a new test suite to check for unpurged dialogs and ensure proper
cleanup.

- **Improvements**
	- Updated dialog search filters to focus on updated timestamps.
	- Streamlined test output by removing unnecessary logging.
	- Ensured cleanup operations after successful dialog creation tests.

- **Bug Fixes**
- Corrected validation checks in dialog detail tests to ensure proper
authorization and API response handling.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

tests/k6/README.md

@@ -37,10 +37,12 @@ See `suites/all-single-pass.js` for a basic example, running all tests with defa
 Tests reside within `tests/serviceowner` and `tests/enduser` depending on the type of test.
 
 Each file can contain several tests related to a particular functionality in the APIs. The test should export one default function containing the tests. 
-> Make sure you do not introduce any global state; only create variables within the function scope. 
+> Make sure you do not introduce any global state; only create variables within the function scope.  All test files should be self-contained and handle both being run alone or with others in any sequential order, ie not rely on state created by tests in other files.
 
 There are several utility functions provided in order to create tests, that can be imported from `testimports.js` residing the `common` directory.
 
+Test files **must** clean up any state they create (see `purgeSO` below), to ensure that the test environment is in a consistent state. A check is implemented in `tests/all-tests.js` to ensure that all dialogs created during the gets purged. Failing to clean up will fail the test run. 
+
 ### Making requests
 
 There are functions to performing a bearer token-authorized request to the service owner and end user endpoints for the selected enviroment and API-version. The `path` parameter are appended to the base URL, can contain query parameters and should not contain a leading slash. `data` can be any javascript object, which will be serialized to JSON.
@@ -90,7 +92,6 @@ export default function () {
 
 ### Notes
 - The request scripts uses the token generator from [Altinn Test Tools](https://github.com/Altinn/AltinnTestTools). The tokens produced contain all scopes required for all endpoints of Dialogporten, and is generated once per run, then re-used and refreshed as needed.
-- 
 
 ## TODO
-* Add support for getting real Maskinporten tokens, see https://github.com/mulesoft-labs/js-client-oauth2
+* Add support for getting real Maskinporten tokens, see https://github.com/mulesoft-labs/js-client-oauth2

tests/k6/common/config.js

@@ -19,8 +19,8 @@ export const baseUrls = {
     }    
 };
 
-export const defaultEndUserOrgNo = "212475912";
-export const defaultEndUserSsn = "14886498226"; // has "DAGL" for 212475912
+export const defaultEndUserOrgNo = "310923044"; // ÆRLIG UROKKELIG TIGER AS
+export const defaultEndUserSsn = "08844397713"; // UROMANTISK LITTERATUR, has "DAGL" for 310923044
 export const defaultServiceOwnerOrgNo = "991825827";
 
 if (__ENV.IS_DOCKER && __ENV.API_ENVIRONMENT == "localdev") {
@@ -41,3 +41,5 @@ if (!baseUrls[__ENV.API_VERSION]["serviceowner"][__ENV.API_ENVIRONMENT]) {
 
 export const baseUrlEndUser = baseUrls[__ENV.API_VERSION]["enduser"][__ENV.API_ENVIRONMENT];
 export const baseUrlServiceOwner = baseUrls[__ENV.API_VERSION]["serviceowner"][__ENV.API_ENVIRONMENT];
+
+export const sentinelValue = "dialogporten-e2e-sentinel";

tests/k6/common/dialog.js

@@ -1,4 +1,5 @@
 import { customConsole as console } from './console.js';
+import { sentinelValue } from './config.js';
 
 export function setTitle(dialog, title, language = "nb") {
     setContent(dialog, "Title", title, language);
@@ -40,6 +41,9 @@ export function setSearchTags(dialog, searchTags) {
         tags.push({ "value": t });
     })
 
+    // Always set the sentinel string that we use to check for leftover dialogs after the run
+    tags.push({ "value": sentinelValue });
+
     dialog.searchTags = tags;
 }
 

tests/k6/common/sentinel.js

@@ -0,0 +1,30 @@
+import { expect, expectStatusFor, describe, getSO, purgeSO } from './testimports.js'
+import { sentinelValue } from './config.js';
+
+export default function () {
+
+    let dialogId = null;
+    const tokenOptions = {
+        scopes: "digdir:dialogporten.serviceprovider digdir:dialogporten.serviceprovider.search digdir:dialogporten.serviceprovider.admin digdir:dialogporten.correspondence"
+    }
+    describe('Post run: checking for unpurged dialogs', () => {
+        let r = getSO('dialogs/?Search=' + sentinelValue, null, tokenOptions);
+        expectStatusFor(r).to.equal(200);
+        expect(r, 'response').to.have.validJsonBody();
+        var response = r.json();
+        if (response.items && response.items.length > 0) {
+            console.error("Found " + response.items.length + " unpurged dialogs, make sure that all tests clean up after themselves. Purging ...");
+            response.items.forEach((item) => {
+                console.warn("Sentinel purging dialog with id: " + item.id)
+                let r = purgeSO('dialogs/' + item.id, null, tokenOptions);
+                if (r.status != 204) {
+                    console.error("Failed to purge dialog with id: " + item.id);
+                    console.log(r);
+                }
+            });
+
+            // Fail the test after purging for visibility
+            expect(response.items.length, 'unpurged dialogs').to.equal(0);
+        }
+    });
+}

tests/k6/tests/all-tests.js

@@ -1,7 +1,11 @@
 import { default as serviceOwnerTests } from './serviceowner/all-tests.js';
 import { default as enduserTests } from './enduser/all-tests.js';
+import { default as sentinelCheck } from '../common/sentinel.js';
 
 export function runAllTests() {
     serviceOwnerTests();
     enduserTests();
+
+    // Run sentinel check last, which will warn about and purge any leftover dialogs
+    sentinelCheck();
 };

tests/k6/tests/enduser/dialogDetails.js

@@ -59,7 +59,6 @@ export default function () {
         expect(dialog.apiActions[0], 'api action').to.have.property("isAuthorized").to.equal(false);
         expect(dialog.apiActions[0], 'url').to.have.property("endpoints");
         for (let i=0; i<dialog.apiActions[0].length; i++) {
-            console.log(dialog.apiActions[i]);
             expect(dialog.apiActions[i], 'endpoint').to.have.property("url").to.equal("urn:dialogporten:unauthorized");
         }
     });

tests/k6/tests/enduser/dialogSearch.js

@@ -45,8 +45,8 @@ export default function () {
     let titleForUpdatedItem = "updated_" + uuidv4();
     let titleForLastItem = "last_" + uuidv4();
     let idForCustomOrg = uuidv7();
-    let createdAfter = (new Date()).toISOString(); // We use this on all tests to hopefully avoid clashing with unrelated dialogs
-    let defaultFilter = "?CreatedAfter=" + createdAfter + "&Party=" + defaultParty;
+    let updatedAfter = (new Date()).toISOString(); // We use this on all tests to avoid clashing with unrelated dialogs
+    let defaultFilter = "?UpdatedAfter=" + updatedAfter + "&Party=" + defaultParty;
     let auxOrg = "ttd";
 
     describe('Arrange: Create some dialogs to test against', () => {

tests/k6/tests/enduser/dialogSystemLabelLog.js

@@ -24,7 +24,6 @@ export default function () {
         response = getEU('dialogs/' + dialogId + '/labellog');
         expectStatusFor(response).to.equal(200);
         expect(response, 'response').to.have.validJsonBody();
-        console.log(response.json());
         expect(response.json(), 'response body').to.have.lengthOf(1);
     });
 
@@ -38,7 +37,6 @@ export default function () {
         response = getEU('dialogs/' + dialogId + '/labellog');
         expectStatusFor(response).to.equal(200);
         expect(response, 'response').to.have.validJsonBody();
-        console.log(response.json());
         expect(response.json(), 'response body').to.have.lengthOf(3);
     })
 
@@ -49,7 +47,6 @@ export default function () {
         response = getEU('dialogs/' + dialogId + '/labellog');
         expectStatusFor(response).to.equal(200);
         expect(response, 'response').to.have.validJsonBody();
-        console.log(response.json());
         expect(response.json(), 'response body').to.have.lengthOf(4);
     })
 

tests/k6/tests/serviceowner/dialogCreatePatchDelete.js

@@ -106,6 +106,10 @@ export default function () {
         expectStatusFor(r).to.equal(201);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.match(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/)
+
+        // Cleanup
+        r = purgeSO('dialogs/' + r.json());
+        expectStatusFor(r).to.equal(204);
     });
 
     describe('Perform dialog create with invalid activity type (transmissionOpened)', () => {
@@ -154,8 +158,11 @@ export default function () {
         // Assert
         expectStatusFor(r).to.equal(201);
         expect(r, 'response').to.have.validJsonBody();
-
         expect(r.json(), 'response json').to.match(/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/)
+
+        // Cleanup
+        r = purgeSO('dialogs/' + r.json());
+        expectStatusFor(r).to.equal(204);
     });
     describe('Perform dialog create with invalid activity type (dialogOpened)', () => {
         // Setup
@@ -182,4 +189,5 @@ export default function () {
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property('errors');
     });
+
 }

tests/k6/tests/serviceowner/dialogDetails.js

@@ -3,7 +3,8 @@ import {
     expect,
     expectStatusFor,
     getSO,
-    postSO
+    postSO,
+    purgeSO
 } from '../../common/testimports.js'
 import {default as dialogToInsert} from './testdata/01-create-dialog.js';
 import { getDefaultEnduserSsn } from "../../common/token.js";
@@ -33,4 +34,9 @@ export default function () {
         let r = getSO('dialogs/' + dialogId + '?endUserId=' + invalidEndUserId);
         expectStatusFor(r).to.equal(404);
     });
+
+    describe('Cleanup', () => {
+        let r = purgeSO('dialogs/' + dialogId);
+        expectStatusFor(r).to.equal(204);
+    });
 }

tests/k6/tests/serviceowner/dialogSearch.js

@@ -43,8 +43,9 @@ export default function () {
     let titleForVisibleFromItem = uuidv4();
     let titleForUpdatedItem = uuidv4();
     let titleForLastItem = uuidv4();
-    let createdAfter = (new Date()).toISOString(); // We use this on all tests to hopefully avoid clashing with unrelated dialogs
-
+    let updatedAfter = (new Date()).toISOString(); // We use this on all tests to avoid clashing with unrelated dialogs
+    let defaultFilter = "?UpdatedAfter=" + updatedAfter;
+
     describe('Arrange: Create some dialogs to test against', () => {
 
         for (let i = 0; i < 20; i++) {
@@ -100,42 +101,42 @@ export default function () {
     });
 
     describe('Search for title', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Search=' + titleToSearchFor);
+        let r = getSO('dialogs/' + defaultFilter + '&Search=' + titleToSearchFor);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(1);
     });
 
     describe('Search for body', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Search=' + additionalInfoToSearchFor);
+        let r = getSO('dialogs/' + defaultFilter + '&Search=' + additionalInfoToSearchFor);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(1);
     });
 
     describe('Search for sender name ', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Search=' + senderNameToSearchFor);
+        let r = getSO('dialogs/' + defaultFilter + '&Search=' + senderNameToSearchFor);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(1);
     });
 
     describe('Filter by extended status', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&ExtendedStatus=' + extendedStatusToSearchFor + "&ExtendedStatus=" + secondExtendedStatusToSearchFor);
+        let r = getSO('dialogs/' + defaultFilter + '&ExtendedStatus=' + extendedStatusToSearchFor + "&ExtendedStatus=" + secondExtendedStatusToSearchFor);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(2);
     });
 
     describe('List with limit', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Limit=3');
+        let r = getSO('dialogs/' + defaultFilter + '&Limit=3');
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(3);
         expect(r.json(), 'response json').to.have.property("hasNextPage").to.be.true;
         expect(r.json(), 'response json').to.have.property("continuationToken");
 
-        let r2 = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Limit=3&ContinuationToken=' + r.json().continuationToken);
+        let r2 = getSO('dialogs/' + defaultFilter + '&Limit=3&ContinuationToken=' + r.json().continuationToken);
         expectStatusFor(r2).to.equal(200);
         expect(r2, 'response').to.have.validJsonBody();
         expect(r2.json(), 'response json').to.have.property("items").with.lengthOf(3);
@@ -146,15 +147,15 @@ export default function () {
     });
 
     describe('List with custom orderBy', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Limit=3&OrderBy=dueAt_desc,updatedAt_desc');
+        let r = getSO('dialogs/' + defaultFilter + '&Limit=3&OrderBy=dueAt_desc,updatedAt_desc');
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(3);
         expect(r.json().items[0], 'first dialog title').to.haveContentOfType("title").that.hasLocalizedText(titleForDueAtItem);
         expect(r.json().items[1], 'second dialog title').to.haveContentOfType("title").that.hasLocalizedText(titleForUpdatedItem);
         expect(r.json().items[2], 'third dialog title').to.haveContentOfType("title").that.hasLocalizedText(titleForLastItem);
 
-        r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Limit=3&OrderBy=dueAt_asc,updatedAt_desc');
+        r = getSO('dialogs/' + defaultFilter + '&Limit=3&OrderBy=dueAt_asc,updatedAt_desc');
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(3);
@@ -163,38 +164,38 @@ export default function () {
     });
 
     describe('List with party filter', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&Party=' + auxParty);
+        let r = getSO('dialogs/' + defaultFilter + '&Party=' + auxParty);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(1);
         expect(r.json().items[0], 'party').to.have.property("party").that.equals(auxParty);
     });
 
     describe('List with resource filter', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&ServiceResource=' + auxResource);
+        let r = getSO('dialogs/' + defaultFilter + '&ServiceResource=' + auxResource);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(1);
         expect(r.json().items[0], 'party').to.have.property("serviceResource").that.equals(auxResource);
     });
 
     describe('List with invalid process', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&process=inval|d');
+        let r = getSO('dialogs/' + defaultFilter + '&process=inval|d');
         expectStatusFor(r).to.equal(400);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("errors");
     })
 
     describe('List with process', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&process=' + processToSeachFor);
+        let r = getSO('dialogs/' + defaultFilter + '&process=' + processToSeachFor);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(1);
         expect(r.json().items[0], 'process').to.have.property("process").that.equals(processToSeachFor);
     })
 
     describe('List with enduserid', () => {
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&EndUserId=' + endUserId + '&ServiceResource=' + auxResource);    
+        let r = getSO('dialogs/' + defaultFilter + '&EndUserId=' + endUserId + '&ServiceResource=' + auxResource);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').to.have.property("items").with.lengthOf(1);
@@ -203,16 +204,17 @@ export default function () {
 
     describe('List with invalid enduserid', () => {
         let invalidEndUserId = "urn:altinn:person:identifier-no:08895699684";
-        let r = getSO('dialogs/?CreatedAfter=' + createdAfter + '&EndUserId=' + invalidEndUserId + '&ServiceResource=' + auxResource);    
+        let r = getSO('dialogs/' + defaultFilter + '&EndUserId=' + invalidEndUserId + '&ServiceResource=' + auxResource);
         expectStatusFor(r).to.equal(200);
         expect(r, 'response').to.have.validJsonBody();
         expect(r.json(), 'response json').not.to.have.property("items");
     })
-    
+
     describe("Cleanup", () => {
         dialogIds.forEach((d) => {
             let r = purgeSO("dialogs/" + d);
             expect(r.status, 'response status').to.equal(204);
         });
+
     });
 }

tests/k6/tests/serviceowner/testdata/01-create-dialog.js

@@ -1,5 +1,6 @@
 import { uuidv4 } from '../../../common/testimports.js'
 import { getDefaultEnduserSsn } from "../../../common/token.js";
+import { sentinelValue } from "../../../common/config.js";
 
 export default function () {
     return {
@@ -13,7 +14,8 @@ export default function () {
         "process": "urn:test:process:1",
         "searchTags": [
             { "value": "something searchable" },
-            { "value": "something else searchable" }
+            { "value": "something else searchable" },
+            { "value": sentinelValue } // Do not remove this, this is used to look for unpurged dialogs after a run
         ],
         "content": {
             "Title": {