feat: build package logging and checks (#199)

* feat: build package logging and checks

* fix: remove commented code

* feat: add installation history to builds

* feat: display basic build history inspector

* feat: add setting for accounts to bypass semver check

package.json

@@ -23,14 +23,14 @@
     "typeorm": "typeorm-ts-node-commonjs"
   },
   "dependencies": {
-    "@aws-sdk/client-kinesis": "^3.554.0",
+    "@aws-sdk/client-kinesis": "^3.556.0",
     "@nestjs/axios": "^3.0.2",
-    "@nestjs/common": "^10.3.7",
+    "@nestjs/common": "^10.3.8",
     "@nestjs/config": "^3.2.2",
-    "@nestjs/core": "^10.3.7",
+    "@nestjs/core": "^10.3.8",
     "@nestjs/jwt": "^10.2.0",
     "@nestjs/passport": "^10.0.3",
-    "@nestjs/platform-express": "^10.3.7",
+    "@nestjs/platform-express": "^10.3.8",
     "@nestjs/schedule": "^4.0.2",
     "@nestjs/serve-static": "^4.0.2",
     "@nestjs/swagger": "^7.3.1",
@@ -64,7 +64,7 @@
     "@golevelup/ts-jest": "^0.5.0",
     "@nestjs/cli": "^10.3.2",
     "@nestjs/schematics": "^10.1.1",
-    "@nestjs/testing": "^10.3.7",
+    "@nestjs/testing": "^10.3.8",
     "@types/cron": "^2.4.0",
     "@types/deep-equal": "^1.0.4",
     "@types/ejs": "^3.1.5",
@@ -78,8 +78,8 @@
     "@types/passport-jwt": "^4.0.1",
     "@types/supertest": "^6.0.2",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/eslint-plugin": "^7.7.0",
-    "@typescript-eslint/parser": "^7.7.0",
+    "@typescript-eslint/eslint-plugin": "^7.7.1",
+    "@typescript-eslint/parser": "^7.7.1",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.1.3",

scripts/db/mongo-setup.js

@@ -456,6 +456,13 @@ result = db.collectionConfig.insertOne({
       hint: 'Defaults unknown users to developer access',
       value: false,
     },
+    maskSemverFailures: {
+      name: 'Mask Semver Failures',
+      required: true,
+      type: 'boolean',
+      hint: 'Replace invalid version with 0.0.0',
+      value: false,
+    },
   },
   browseFields: ['name', 'email', 'clientId', 'website', 'requireRoleId'],
   name: 'Broker Account',

scripts/provision-app-quick-build.json

@@ -15,7 +15,8 @@
         "environment": "tools"
       },
       "package": {
-        "version": "2.73.2-snapshot"
+        "version": "2.73.2-snapshot",
+        "name": "superapp-backend"
       }
     }
   ],

scripts/provision-app-quick-build.sh

@@ -6,8 +6,6 @@ cd "$this_dir"
 PACKAGE_BUILD_VERSION=$(git rev-parse --verify HEAD)
 PACKAGE_VERSION="12.0.3"
 sha256=($(echo $RANDOM $RANDOM $RANDOM | shasum -a 256))
-echo -n $sha256 > provision-app-quick-build.artifact.sha256
-echo "sha256: $sha256"
 
 echo "===> Intention open"
 # Open intention
@@ -36,17 +34,27 @@ echo -n $INTENTION_ID > provision-app-quick-build.intention.id
 
 echo "===> Build"
 
-# Not shown: Build superapp and create artifact (build.zip)
+# Not shown: Build superapp and create artifact
 echo "===> ..."
 echo "===> Build - Success!"
-# Add artifact to action
+# Patch action with build info
 ACTIONS_BUILD_TOKEN=$(echo $RESPONSE | jq -r '.actions.build.token')
-curl -s -X POST $BROKER_URL/v1/intention/action/artifact \
+RESPONSE=$(curl -s -X POST $BROKER_URL/v1/intention/action/patch \
         -H 'Content-Type: application/json' \
         -H 'X-Broker-Token: '"$ACTIONS_BUILD_TOKEN"'' \
-        -d '{"checksum": "sha256:'$sha256'", "name": "build.zip", "size": '$RANDOM', "type": "zip" }'
+        -d '{"package":{"checksum": "sha256:'$sha256'", "size": '$RANDOM', "type": "zip" }}' \
+    )
+echo $RESPONSE | jq '.'
+if [ "$(echo $RESPONSE | jq '.error')" != "null" ]; then
+    # Use saved intention token to close intention
+    echo "Exit: Error detected"
+    exit 0
+fi
 
 echo "===> Intention close"
 
 # Use saved intention token to close intention
 curl -s -X POST $BROKER_URL/v1/intention/close -H 'X-Broker-Token: '"$INTENTION_TOKEN"''
+
+echo -n $sha256 > provision-app-quick-build.artifact.sha256
+echo "sha256: $sha256"

scripts/provision-app-quick-install.json

@@ -9,6 +9,13 @@
       "action": "package-installation",
       "id": "install",
       "provision": [],
+      "cloud": {
+        "target": {
+          "instance": {
+            "name": "lollipop"
+          }
+        }
+      },
       "service": {
         "project": "superapp",
         "name": "superapp-backend",
@@ -17,9 +24,6 @@
       "source": {
         "action": "package-build#build",
         "intention": "feef-ffwe-343432-42fefds-ffew"
-      },
-      "package": {
-        "type": "zip"
       }
     }
   ],

scripts/provision-app-quick-install.sh

@@ -36,10 +36,10 @@ echo "===> Install - Success!"
 
 # Add install to action
 ACTIONS_INSTALL_TOKEN=$(echo $RESPONSE | jq -r '.actions.install.token')
-curl -s -X POST $BROKER_URL/v1/intention/action/install \
+curl -s -X POST $BROKER_URL/v1/intention/action/patch \
         -H 'Content-Type: application/json' \
         -H 'X-Broker-Token: '"$ACTIONS_INSTALL_TOKEN"'' \
-        -d '{"cloudTarget":{"instance": {"name": "lollipop"}},"prop": {"port": "3243"}}'
+        -d '{"cloud":{"target":{"propStrategy":"replace","prop":{"port": "5000", "something": "else"}}}}'
 
 echo "===> Intention close"
 

src/collection/collection.service.ts

@@ -14,10 +14,12 @@ import { IntentionService } from '../intention/intention.service';
 import { PERSISTENCE_TYPEAHEAD_SUBQUERY_LIMIT } from '../persistence/persistence.constants';
 import { RedisService } from '../redis/redis.service';
 import { IntentionActionPointerDto } from '../persistence/dto/intention-action-pointer.dto';
+import { BuildRepository } from '../persistence/interfaces/build.repository';
 
 @Injectable()
 export class CollectionService {
   constructor(
+    private readonly buildRepository: BuildRepository,
     private readonly collectionRepository: CollectionRepository,
     private readonly graphRepository: GraphRepository,
     private readonly intentionService: IntentionService,
@@ -244,6 +246,7 @@ export class CollectionService {
 
   async getServiceDetails(serviceId: string) {
     const service = await this.graphRepository.getServiceDetails(serviceId);
+    const builds = await this.buildRepository.searchBuild(serviceId, 0, 5);
     if (!service) {
       throw new NotFoundException({
         statusCode: 404,
@@ -259,7 +262,10 @@ export class CollectionService {
           );
       }
     }
-    return service;
+    return {
+      ...service,
+      builds,
+    };
   }
 
   async getServiceSecureInfo(serviceId: string) {

src/constants.ts

@@ -18,9 +18,15 @@ export const INTENTION_MIN_TTL_SECONDS = 30;
 export const INTENTION_MAX_TTL_SECONDS = 1800;
 export const INTENTION_TRANSIENT_TTL_MS = 7 * 24 * 60 * 60 * 1000;
 
-export const INTENTION_SERVICE_INSTANCE_SEARCH_PATHS = [
+// Search paths use last existing path as the value
+export const INTENTION_SERVICE_ENVIRONMENT_SEARCH_PATHS = [
   'action.service.environment',
+  'action.service.target.environment',
+] as const;
+export const INTENTION_SERVICE_INSTANCE_SEARCH_PATHS = [
+  ...INTENTION_SERVICE_ENVIRONMENT_SEARCH_PATHS,
   'action.service.instanceName',
+  'action.service.target.instanceName',
 ] as const;
 
 export const SHORT_ENV_CONVERSION = {

src/graph/intention-sync.service.ts

@@ -1,6 +1,9 @@
 import { Injectable } from '@nestjs/common';
 import { get, set } from 'radash';
 import deepEqual from 'deep-equal';
+import { plainToClass } from 'class-transformer';
+
+import { INTENTION_SERVICE_INSTANCE_SEARCH_PATHS } from '../constants';
 import { IntentionDto } from '../intention/dto/intention.dto';
 import { ActionDto } from '../intention/dto/action.dto';
 import { CollectionNames } from '../persistence/dto/collection-dto-union.type';
@@ -9,8 +12,10 @@ import { PersistenceUtilService } from '../persistence/persistence-util.service'
 import { GraphService } from './graph.service';
 import { GraphRepository } from '../persistence/interfaces/graph.repository';
 import { EdgePropDto } from '../persistence/dto/edge-prop.dto';
-import { INTENTION_SERVICE_INSTANCE_SEARCH_PATHS } from '../constants';
-import { InstallDto } from 'src/intention/dto/install.dto';
+import { ActionUtil } from '../util/action.util';
+import { CollectionRepository } from '../persistence/interfaces/collection.repository';
+import { IntentionActionPointerDto } from '../persistence/dto/intention-action-pointer.dto';
+import { BuildRepository } from '../persistence/interfaces/build.repository';
 
 interface OverlayMapBase {
   key: string;
@@ -33,9 +38,12 @@ type OverlayMap = OverlayMapWithPath | OverlayMapWithValue;
 @Injectable()
 export class IntentionSyncService {
   constructor(
+    private readonly buildRepository: BuildRepository,
+    private readonly collectionRepository: CollectionRepository,
     private readonly graphService: GraphService,
     private readonly graphRepository: GraphRepository,
     private readonly persistenceUtilService: PersistenceUtilService,
+    private readonly actionUtil: ActionUtil,
   ) {}
 
   public async sync(intention: IntentionDto) {
@@ -63,10 +71,69 @@ export class IntentionSyncService {
         action.action === 'package-installation'
       ) {
         await this.syncPackageInstall(intention, action, serviceVertex);
+        await this.syncPackageBuild(intention, action, serviceVertex);
+      }
+
+      if (action.action === 'package-build') {
+        await this.syncPackageBuild(intention, action, serviceVertex);
       }
     }
   }
 
+  private async syncPackageBuild(
+    intention: IntentionDto,
+    action: ActionDto,
+    serviceVertex: VertexDto,
+  ) {
+    if (!action.package || !action.package.name || !action.package.version) {
+      // Not enough package information to save
+      return;
+    }
+
+    const parsedVersion = this.actionUtil.parseVersion(action.package.version);
+    if (!parsedVersion || parsedVersion.prerelease) {
+      // Not a valid version. Should not occur.
+      return;
+    }
+    if (parsedVersion.prerelease) {
+      // Pre-release versions are not release candidates -- not recorded
+      return;
+    }
+
+    const service = await this.collectionRepository.getCollectionByVertexId(
+      'service',
+      serviceVertex.id.toString(),
+    );
+    if (!service) {
+      // Awkward. There should be a service here...
+      return;
+    }
+
+    let packageBuild = await this.buildRepository.getBuildByPackageDetail(
+      service.id.toString(),
+      action.package.name,
+      parsedVersion,
+    );
+    if (!packageBuild) {
+      packageBuild = await this.buildRepository.addBuild(
+        service.id.toString(),
+        action.package.name,
+        parsedVersion,
+        action.package,
+      );
+    }
+
+    if (action.action === 'package-installation') {
+      await this.buildRepository.addInstallActionToBuild(
+        packageBuild.id.toString(),
+        plainToClass(IntentionActionPointerDto, {
+          action: this.actionUtil.actionToIdString(action),
+          intention: intention.id,
+        }),
+      );
+    }
+  }
+
   public async syncPackageInstall(
     intention: IntentionDto,
     action: ActionDto,
@@ -91,15 +158,15 @@ export class IntentionSyncService {
         },
         {
           key: 'action.action',
-          value: `${action.action}#${action.id}`,
+          value: this.actionUtil.actionToIdString(action),
         },
         {
           key: 'actionHistory[0].intention',
           value: intention.id,
         },
         {
           key: 'actionHistory[0].action',
-          value: `${action.action}#${action.id}`,
+          value: this.actionUtil.actionToIdString(action),
         },
       ],
       'parentId',
@@ -113,12 +180,22 @@ export class IntentionSyncService {
         envMap[action.service.environment].vertex.toString(),
       );
     }
+
+    const serverVertex = await this.syncServer(action);
+    if (serverVertex) {
+      const instanceName = this.actionUtil.instanceName(action);
+      this.syncInstallationProperties(
+        serviceVertex,
+        instanceName,
+        serverVertex,
+        action.cloud.target.propStrategy,
+        action.cloud.target.prop,
+      );
+    }
   }
 
-  public async syncServer(action: ActionDto, install: InstallDto) {
-    const serverName =
-      install.cloudTarget?.instance?.name ??
-      action.cloud?.target?.instance?.name;
+  public async syncServer(action: ActionDto) {
+    const serverName = action.cloud?.target?.instance?.name;
     if (!serverName) {
       return null;
     }