feat: slice graph data by visible to reduce load times (#275)

* feat: slice graph data by visible to reduce load times
bcgov-nr · Oct 1, 2024 · a722536 · a722536
1 parent 266d470
commit a722536
Show file tree

Hide file tree

Showing 10 changed files with 124 additions and 4 deletions.
diff --git a/src/graph/graph.controller.ts b/src/graph/graph.controller.ts
@@ -36,6 +36,7 @@ import { AllowBodyValue } from '../allow-body-value.decorator';
 import { AllowEmptyEdges } from '../allow-empty-edges.decorator';
 import { PersistenceCacheInterceptor } from '../persistence/persistence-cache.interceptor';
 import { PersistenceCacheKey } from '../persistence/persistence-cache-key.decorator';
+import { PersistenceCacheSuffix } from '../persistence/persistence-cache-suffix.decorator';
 import { GraphTypeaheadQuery } from './dto/graph-typeahead-query.dto';
 import { PERSISTENCE_CACHE_KEY_GRAPH } from '../persistence/persistence.constants';
 import { REDIS_PUBSUB } from '../constants';
@@ -51,6 +52,16 @@ export class GraphController {
     private readonly redis: RedisService,
   ) {}
 
+  @Get('data-slice/:collections')
+  @UseGuards(BrokerCombinedAuthGuard)
+  @ApiBearerAuth()
+  @PersistenceCacheKey(PERSISTENCE_CACHE_KEY_GRAPH)
+  @PersistenceCacheSuffix('collections')
+  @UseInterceptors(PersistenceCacheInterceptor)
+  getDataSlice(@Param('collections') collections: string) {
+    return this.graph.getDataSlice(collections.split(','));
+  }
+
   @Get('data')
   @UseGuards(BrokerCombinedAuthGuard)
   @ApiBearerAuth()

diff --git a/src/graph/graph.service.ts b/src/graph/graph.service.ts
@@ -50,6 +50,12 @@ export class GraphService {
     return this.graphRepository.getData(includeCollection);
   }
 
+  public async getDataSlice(
+    collections: string[],
+  ): Promise<GraphDataResponseDto> {
+    return this.graphRepository.getDataSlice(collections);
+  }
+
   public async getProjectServices() {
     return this.graphRepository.getProjectServices();
   }

diff --git a/src/persistence/interfaces/graph.repository.ts b/src/persistence/interfaces/graph.repository.ts
@@ -25,6 +25,9 @@ export abstract class GraphRepository {
   public abstract getData(
     includeCollection: boolean,
   ): Promise<GraphDataResponseDto>;
+  public abstract getDataSlice(
+    collections: string[],
+  ): Promise<GraphDataResponseDto>;
   public abstract getProjectServices(): Promise<
     GraphProjectServicesResponseDto[]
   >;

diff --git a/src/persistence/mongo/graph-mongo.repository.ts b/src/persistence/mongo/graph-mongo.repository.ts
@@ -49,6 +49,42 @@ export class GraphMongoRepository implements GraphRepository {
     private readonly actionUtil: ActionUtil,
   ) {}
 
+  public async getDataSlice(
+    collections: string[],
+  ): Promise<GraphDataResponseDto> {
+    const configs = await this.collectionConfigRepository.find();
+    const filteredConfigs = configs.filter(
+      (config) => collections.indexOf(config.collection) !== -1,
+    );
+    const verticeArrs = await Promise.all(
+      configs.map((config, category: number) => {
+        if (collections.indexOf(config.collection) === -1) {
+          return Promise.resolve([]);
+        }
+        return this.aggregateVertex(
+          config.collection,
+          category,
+          config.index,
+          false,
+        );
+      }),
+    );
+
+    const edges = await this.edgeRepository.find({
+      is: { $in: filteredConfigs.map((config) => config.index) },
+      it: { $in: filteredConfigs.map((config) => config.index) },
+    });
+    return {
+      edges: edges.map((edge) => edge.toEdgeResponse(false)),
+      vertices: [].concat(...verticeArrs),
+      categories: configs.map((config) => {
+        return {
+          name: config.name,
+        };
+      }),
+    };
+  }
+
   public async getData(
     includeCollection: boolean,
   ): Promise<GraphDataResponseDto> {
@@ -65,7 +101,6 @@ export class GraphMongoRepository implements GraphRepository {
     );
 
     const edges = await this.edgeRepository.find();
-    // console.log(edges);
     return {
       edges: edges.map((edge) => edge.toEdgeResponse(false)),
       vertices: [].concat(...verticeArrs),

diff --git a/src/persistence/persistence-cache-suffix.decorator.ts b/src/persistence/persistence-cache-suffix.decorator.ts
@@ -0,0 +1,5 @@
+import { SetMetadata } from '@nestjs/common';
+import { PERSISTENCE_CACHE_METADATA_SUFFIX } from './persistence.constants';
+
+export const PersistenceCacheSuffix = (suffix: any) =>
+  SetMetadata(PERSISTENCE_CACHE_METADATA_SUFFIX, suffix);
diff --git a/src/persistence/persistence-cache.interceptor.ts b/src/persistence/persistence-cache.interceptor.ts
@@ -9,9 +9,11 @@ import { Reflector } from '@nestjs/core';
 import { Response as ExpressResponse } from 'express';
 import { Observable, from, of } from 'rxjs';
 import { switchMap, tap } from 'rxjs/operators';
+import * as crypto from 'crypto';
 import { RedisClientType } from 'redis';
 import {
   PERSISTENCE_CACHE_METADATA_KEY,
+  PERSISTENCE_CACHE_METADATA_SUFFIX,
   PERSISTENCE_CACHE_METADATA_TTL,
 } from './persistence.constants';
 
@@ -39,18 +41,45 @@ export class PersistenceCacheInterceptor<T>
       ) ??
       this.reflector.get(PERSISTENCE_CACHE_METADATA_TTL, context.getClass()) ??
       null;
-    const keyValue =
+    let keyValue =
       this.reflector.get(
         PERSISTENCE_CACHE_METADATA_KEY,
         context.getHandler(),
       ) ??
       this.reflector.get(PERSISTENCE_CACHE_METADATA_KEY, context.getClass()) ??
       null;
+    const suffixValue =
+      this.reflector.get(
+        PERSISTENCE_CACHE_METADATA_SUFFIX,
+        context.getHandler(),
+      ) ??
+      this.reflector.get(
+        PERSISTENCE_CACHE_METADATA_SUFFIX,
+        context.getClass(),
+      ) ??
+      null;
 
     if (!keyValue) {
       return next.handle();
     }
 
+    const request = context.switchToHttp().getRequest();
+    console.log(request.params);
+    console.log(suffixValue);
+    if (
+      suffixValue &&
+      request.params[suffixValue] &&
+      typeof request.params[suffixValue] === 'string'
+    ) {
+      const suffixArr = request.params[suffixValue].split(',').sort();
+      const hash = crypto
+        .createHash('sha256')
+        .update(suffixArr.join('|'))
+        .digest('hex');
+      keyValue = `${keyValue}-${hash}`;
+      console.log(keyValue);
+    }
+
     const response = context.switchToHttp().getResponse<ExpressResponse>();
     response.header('Cache-Control', 'no-cache');
 

diff --git a/src/persistence/persistence.constants.ts b/src/persistence/persistence.constants.ts
@@ -1,5 +1,6 @@
 export const PERSISTENCE_CACHE_METADATA_TTL = 'PERSISTENCE_CACHE_TTL';
 export const PERSISTENCE_CACHE_METADATA_KEY = 'PERSISTENCE_CACHE_KEY';
+export const PERSISTENCE_CACHE_METADATA_SUFFIX = 'PERSISTENCE_CACHE_SUFFIX';
 
 export const PERSISTENCE_CACHE_KEY_GRAPH = 'graph-data';
 export const PERSISTENCE_CACHE_KEY_CONFIG = 'collection-config';

diff --git a/src/persistence/redis-composition/graph-redis.repository.ts b/src/persistence/redis-composition/graph-redis.repository.ts
@@ -60,6 +60,12 @@ export class GraphRedisRepository implements GraphRepository {
     return this.repo.getData(includeCollection);
   }
 
+  public async getDataSlice(
+    collections: string[],
+  ): Promise<GraphDataResponseDto> {
+    return this.repo.getDataSlice(collections);
+  }
+
   public async getProjectServices(): Promise<
     GraphProjectServicesResponseDto[]
   > {
@@ -419,9 +425,13 @@ export class GraphRedisRepository implements GraphRepository {
     return `collection_json:${vertex.collection}:${vertex.id.toString()}`;
   }
 
-  private invalidateCache() {
+  private async invalidateCache() {
     // console.log('invalidate: cache');
+    const keys = await this.client.keys(`${PERSISTENCE_CACHE_KEY_GRAPH}-*`);
+    const suffixDelArr = keys.map((key) => this.client.del(key));
+
     return Promise.all([
+      ...suffixDelArr,
       this.client.del(PERSISTENCE_CACHE_KEY_GRAPH),
       this.client.del(PERSISTENCE_CACHE_KEY_CONFIG),
     ]);

diff --git a/ui/src/app/graph/graph.component.ts b/ui/src/app/graph/graph.component.ts
@@ -91,12 +91,23 @@ export class GraphComponent implements OnInit, OnDestroy {
   ) {}
 
   ngOnInit(): void {
+    this.preferences.onSet
+      .pipe(takeUntil(this.ngUnsubscribe))
+      .subscribe((pref) => {
+        if (pref.key === 'graphVertexVisibility') {
+          this.refreshData();
+        }
+      });
     this.data = this.triggerRefresh.pipe(
       takeUntil(this.ngUnsubscribe),
       switchMap(() =>
         combineLatest([
           combineLatest([
-            this.graphApi.getData(),
+            this.graphApi.getDataSlice(
+              this.configArr
+                .filter((config) => this.isCollectionVisible(config.collection))
+                .map((config) => config.collection),
+            ),
             this.graphApi.getVertexConnected(),
             this.graphApi
               .createEventSource()

diff --git a/ui/src/app/service/graph-api.service.ts b/ui/src/app/service/graph-api.service.ts
@@ -66,6 +66,15 @@ export class GraphApiService {
     );
   }
 
+  getDataSlice(collections: string[]) {
+    return this.http.get<GraphDataResponseDto>(
+      `${environment.apiUrl}/v1/graph/data-slice/${collections.sort().join(',')}`,
+      {
+        responseType: 'json',
+      },
+    );
+  }
+
   getVertexConnected() {
     return this.http.post<string[]>(
       `${environment.apiUrl}/v1/graph/vertex/connected`,