From 3547ecdf1476167ec8c1cc19a66f0c649bff05e0 Mon Sep 17 00:00:00 2001 From: Gayathri Date: Mon, 25 Sep 2023 18:35:31 +0530 Subject: [PATCH] minor changes to the query load process --- .changeset/clean-toes-learn.md | 1 + .../QueryEditorComponentTestUtils.tsx | 7 +- .../stores/LegendQueryApplicationPlugin.ts | 13 +- .../src/stores/QueryEditorStore.ts | 67 +++--- .../editor/editor-group/FunctionEditor.tsx | 3 +- .../DEPRECATED__MappingTestEditor.tsx | 2 +- .../MappingExecutionBuilder.tsx | 2 +- .../mapping-editor/MappingTestableEditor.tsx | 2 +- .../ServiceExecutionQueryEditor.tsx | 4 +- .../uml-editor/ClassQueryBuilder.tsx | 2 +- .../editor/EmbeddedQueryBuilderState.ts | 5 +- ...DataSpace_LegendQueryApplicationPlugin.tsx | 99 +++++---- .../query/DataSpaceQueryBuilder.tsx | 20 +- ...DSL_DataSpace_PureGraphManagerExtension.ts | 15 +- ...DSL_DataSpace_PureGraphManagerExtension.ts | 53 +++-- .../query/DataSpaceQueryBuilderState.ts | 199 ++++++++++-------- .../query/DataSpaceQueryCreatorStore.ts | 65 +++--- .../graph-manager/AbstractPureGraphManager.ts | 12 +- .../analytics/MappingModelCoverageAnalysis.ts | 24 ++- .../src/graph-manager/action/query/Query.ts | 13 +- .../protocol/pure/v1/V1_PureGraphManager.ts | 141 +++++++++---- .../pure/v1/engine/V1_EngineHelper.ts | 42 +--- .../V1_MappingModelCoverageAnalysis.ts | 38 +++- 23 files changed, 451 insertions(+), 378 deletions(-) diff --git a/.changeset/clean-toes-learn.md b/.changeset/clean-toes-learn.md index d78bbd6f642..7b87f06420a 100644 --- a/.changeset/clean-toes-learn.md +++ b/.changeset/clean-toes-learn.md @@ -1,3 +1,4 @@ --- '@finos/legend-graph': patch +'@finos/legend-application-studio': patch --- diff --git a/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx b/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx index e8f1e769e50..e7a2f9024ca 100644 --- a/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx +++ b/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx @@ -22,7 +22,6 @@ import { Query, LightQuery, RawLambda, - PackageableElementExplicitReference, type RawMappingModelCoverageAnalysisResult, } from '@finos/legend-graph'; import { DepotServerClient } from '@finos/legend-server-depot'; @@ -128,10 +127,8 @@ export const TEST__setUpQueryEditor = async ( query.owner = lightQuery.owner; query.isCurrentUserQuery = lightQuery.isCurrentUserQuery; const _mapping = graphManagerState.graph.getMapping(mappingPath); - query.mapping = PackageableElementExplicitReference.create(_mapping); - query.runtime = PackageableElementExplicitReference.create( - graphManagerState.graph.getRuntime(runtimePath), - ); + query.mapping = mappingPath; + query.runtime = runtimePath; query.content = 'some content'; createSpy( diff --git a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts index c54479278cd..f8d127a771b 100644 --- a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts +++ b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts @@ -16,6 +16,7 @@ import { LegendApplicationPlugin } from '@finos/legend-application'; import type { QueryBuilderState } from '@finos/legend-query-builder'; +import type { GeneratorFn } from '@finos/legend-shared'; import type React from 'react'; import type { LegendQueryPluginManager } from '../application/LegendQueryPluginManager.js'; import type { @@ -23,6 +24,7 @@ import type { QueryEditorStore, } from './QueryEditorStore.js'; import type { QuerySetupLandingPageStore } from './QuerySetupStore.js'; +import type { Query } from '@finos/legend-graph'; export enum QuerySetupActionTag { PRODUCTIONIZATION = 'Productionization', @@ -51,12 +53,13 @@ export type QuerySetupActionConfiguration = { }; export type ExistingQueryEditorStateBuilder = ( + query: Query, editorStore: ExistingQueryEditorStore, ) => Promise; -export type ExistingQueryGraphBuilderChecker = ( - editorStore: ExistingQueryEditorStore, -) => boolean | undefined; +export type QueryGraphBuilderGetter = ( + editorStore: QueryEditorStore, +) => ((editorStore: QueryEditorStore) => GeneratorFn) | undefined; export type QueryEditorActionConfiguration = { key: string; @@ -101,9 +104,9 @@ export abstract class LegendQueryApplicationPlugin extends LegendApplicationPlug getExtraExistingQueryEditorStateBuilders?(): ExistingQueryEditorStateBuilder[]; /** - * Get the list of existing query graph builder checkers + * Get the list of query graph builders */ - getExtraExistingQueryGraphBuilderCheckers?(): ExistingQueryGraphBuilderChecker[]; + getExtraQueryGraphBuilderGetters?(): QueryGraphBuilderGetter[]; /** * Get the list of query editor action renderer configurations. diff --git a/packages/legend-application-query/src/stores/QueryEditorStore.ts b/packages/legend-application-query/src/stores/QueryEditorStore.ts index c055e4bc70e..27edf88c243 100644 --- a/packages/legend-application-query/src/stores/QueryEditorStore.ts +++ b/packages/legend-application-query/src/stores/QueryEditorStore.ts @@ -293,6 +293,7 @@ export abstract class QueryEditorStore { setExistingQueryName: action, initialize: flow, buildGraph: flow, + buildFullGraph: flow, searchExistingQueryName: flow, }); @@ -405,10 +406,9 @@ export abstract class QueryEditorStore { RuntimePointer, 'Query runtime must be of type runtime pointer', ); - query.mapping = PackageableElementExplicitReference.create( - this.queryBuilderState.executionContextState.mapping, - ); - query.runtime = runtimeValue.packageableRuntime; + (query.mapping = + this.queryBuilderState.executionContextState.mapping.path), + (query.runtime = runtimeValue.packageableRuntime.value.path); query.content = await this.graphManagerState.graphManager.lambdaToPureCode(rawLambda); config?.decorator?.(query); @@ -489,9 +489,8 @@ export abstract class QueryEditorStore { ); yield this.setUpEditorState(); - if (this.requiresGraphBuilding()) { - yield flowResult(this.buildGraph()); - } + yield flowResult(this.buildGraph()); + this.queryBuilderState = (yield this.initializeQueryBuilderState( stopWatch, )) as QueryBuilderState; @@ -543,7 +542,7 @@ export abstract class QueryEditorStore { ); } - *buildGraph(): GeneratorFn { + *buildFullGraph(): GeneratorFn { const stopWatch = new StopWatch(); const { groupId, artifactId, versionId } = this.getProjectInfo(); @@ -632,6 +631,20 @@ export abstract class QueryEditorStore { graphBuilderReportData, ); } + + *buildGraph(): GeneratorFn { + const queryGraphBuilderGetters = this.applicationStore.pluginManager + .getApplicationPlugins() + .flatMap((plugin) => plugin.getExtraQueryGraphBuilderGetters?.() ?? []); + for (const getter of queryGraphBuilderGetters) { + const builderFunction = getter(this); + if (builderFunction) { + yield flowResult(builderFunction(this)); + return; + } + } + yield flowResult(this.buildFullGraph()); + } } export class MappingQueryCreatorStore extends QueryEditorStore { @@ -992,34 +1005,12 @@ export class ExistingQueryEditorStore extends QueryEditorStore { } override async setUpEditorState(): Promise { - this.setLightQuery( - await this.graphManagerState.graphManager.getLightQuery(this.queryId), - ); - } - - override requiresGraphBuilding(): boolean { - const existingQueryGraphBuilderCheckers = - this.applicationStore.pluginManager - .getApplicationPlugins() - .flatMap( - (plugin) => - plugin.getExtraExistingQueryGraphBuilderCheckers?.() ?? [], - ); - for (const checker of existingQueryGraphBuilderCheckers) { - const isGraphBuildingRequired = checker(this); - if (isGraphBuildingRequired !== undefined) { - return isGraphBuildingRequired; - } - } - return true; - } - - async setupQuery(): Promise { const query = await this.graphManagerState.graphManager.getQuery( this.queryId, this.graphManagerState.graph, ); this.setQuery(query); + this.setLightQuery(toLightQuery(query)); LegendQueryUserDataHelper.addRecentlyViewedQuery( this.applicationStore.userDataService, query.id, @@ -1029,6 +1020,7 @@ export class ExistingQueryEditorStore extends QueryEditorStore { async initializeQueryBuilderState( stopWatch: StopWatch, ): Promise { + const query = guaranteeNonNullable(this.query); let queryBuilderState: QueryBuilderState | undefined; const existingQueryEditorStateBuilders = this.applicationStore.pluginManager .getApplicationPlugins() @@ -1036,15 +1028,12 @@ export class ExistingQueryEditorStore extends QueryEditorStore { (plugin) => plugin.getExtraExistingQueryEditorStateBuilders?.() ?? [], ); for (const builder of existingQueryEditorStateBuilders) { - queryBuilderState = await builder(this); + queryBuilderState = await builder(query, this); if (queryBuilderState) { break; } } - await this.setupQuery(); - const query = guaranteeNonNullable(this.query); - // if no extension found, fall back to basic `class -> mapping -> runtime` mode queryBuilderState = queryBuilderState ?? @@ -1054,11 +1043,11 @@ export class ExistingQueryEditorStore extends QueryEditorStore { this.applicationStore.config.options.queryBuilderConfig, ); - queryBuilderState.executionContextState.setMapping(query.mapping.value); + const mapping = this.graphManagerState.graph.getMapping(query.mapping); + const runtime = this.graphManagerState.graph.getRuntime(query.runtime); + queryBuilderState.executionContextState.setMapping(mapping); queryBuilderState.executionContextState.setRuntimeValue( - new RuntimePointer( - PackageableElementExplicitReference.create(query.runtime.value), - ), + new RuntimePointer(PackageableElementExplicitReference.create(runtime)), ); // leverage initialization of query builder state to ensure we handle unsupported queries diff --git a/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx index e5e7e47587e..e107d39dbe5 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx @@ -1105,8 +1105,7 @@ export const FunctionEditor = observer(() => { ); await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => - functionQueryBuilderState, + setupQueryBuilderState: async () => functionQueryBuilderState, actionConfigs: [ { key: 'save-query-btn', diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx index f56f46469f8..f4f9328fb62 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx @@ -112,7 +112,7 @@ const MappingTestQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx index 092f2fc3454..6355d15cff7 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx @@ -205,7 +205,7 @@ const MappingExecutionQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx index bea58ebc0c0..b369c793731 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx @@ -589,7 +589,7 @@ const MappingTestSuiteQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx index 60aeb287588..2cc4ccb7405 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx @@ -125,7 +125,7 @@ export const ServiceExecutionQueryEditor = observer( executionState.selectedExecutionContextState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new ServiceQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, @@ -459,7 +459,7 @@ export const queryService = async ( : undefined; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new ServiceQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx b/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx index ccd8c833260..6bf11b1058a 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx @@ -375,7 +375,7 @@ export const queryClass = async ( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: () => { + setupQueryBuilderState: async () => { const queryBuilderState = new ClassQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts b/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts index 03581d7bd73..7b5f5ee2eb5 100644 --- a/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts +++ b/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts @@ -30,7 +30,7 @@ type EmbeddedQueryBuilderActionConfiguration = { }; type EmbeddedQueryBuilderConfiguration = { - setupQueryBuilderState: () => QueryBuilderState; + setupQueryBuilderState: () => Promise; disableCompile?: boolean | undefined; actionConfigs: EmbeddedQueryBuilderActionConfiguration[]; }; @@ -100,7 +100,8 @@ export class EmbeddedQueryBuilderState { } } if (!this.editorStore.graphState.error) { - this.queryBuilderState = config.setupQueryBuilderState(); + this.queryBuilderState = + (yield config.setupQueryBuilderState()) as QueryBuilderState; this.actionConfigs = config.actionConfigs; this.editorStore.applicationStore.layoutService.setBackdropContainerElementID( QUERY_BUILDER_COMPONENT_ELEMENT_ID.BACKDROP_CONTAINER, diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx index 6e142a080fa..677b749ae18 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx @@ -17,7 +17,7 @@ import packageJson from '../../../package.json' assert { type: 'json' }; import { type QuerySetupActionConfiguration, - type ExistingQueryEditorStore, + ExistingQueryEditorStore, LegendQueryApplicationPlugin, generateExistingQueryEditorRoute, LEGEND_QUERY_APP_EVENT, @@ -25,7 +25,8 @@ import { createViewProjectHandler, createViewSDLCProjectHandler, type ExistingQueryEditorStateBuilder, - type ExistingQueryGraphBuilderChecker, + type QueryGraphBuilderGetter, + type QueryEditorStore, } from '@finos/legend-application-query'; import { SquareIcon } from '@finos/legend-art'; import { @@ -43,7 +44,6 @@ import { Query, isValidFullPath, GRAPH_MANAGER_EVENT, - type V1_PureModelContextData, createGraphBuilderReport, } from '@finos/legend-graph'; import { @@ -58,11 +58,11 @@ import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { getOwnDataSpace } from '../../graph-manager/DSL_DataSpace_GraphManagerHelper.js'; import { assertErrorThrown, - guaranteeNonNullable, isString, LogEvent, StopWatch, uuid, + type GeneratorFn, } from '@finos/legend-shared'; import type { QueryBuilderState } from '@finos/legend-query-builder'; import { DataSpaceQuerySetup } from './DataSpaceQuerySetup.js'; @@ -72,6 +72,11 @@ import { StoreProjectData, } from '@finos/legend-server-depot'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; +import { flowResult } from 'mobx'; + +function* buildGraph(): GeneratorFn { + // do nothing +} export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin { constructor() { @@ -118,9 +123,10 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli override getExtraExistingQueryEditorStateBuilders(): ExistingQueryEditorStateBuilder[] { return [ async ( + query: Query, editorStore: ExistingQueryEditorStore, ): Promise => { - const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( + const dataSpaceTaggedValue = query.taggedValues?.find( (taggedValue) => taggedValue.profile === QUERY_PROFILE_PATH && taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && @@ -129,11 +135,10 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli let isLightGraphEnabled = true; if (dataSpaceTaggedValue) { const dataSpacePath = dataSpaceTaggedValue.value; - const mappingPath = editorStore.lightQuery?.mapping; - let dataSpaceAnalysisResult; - let dataSpaceAnalysisResultMetaModel; + const mappingPath = query.mapping; const { groupId, artifactId, versionId } = editorStore.getProjectInfo(); + let dataSpaceAnalysisResult; if (dataSpacePath && isString(mappingPath)) { try { editorStore.initState.setMessage( @@ -145,6 +150,15 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli artifactId, ), ); + const graph_buildReport = createGraphBuilderReport(); + const stopWatch = new StopWatch(); + // initialize system + stopWatch.record(); + await editorStore.graphManagerState.initializeSystem(); + stopWatch.record( + GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS, + ); + const dependency_buildReport = createGraphBuilderReport(); dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( editorStore.graphManagerState.graphManager, @@ -163,37 +177,15 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli dataSpacePath, editorStore.depotServerClient, ), - ); - const stopWatch = new StopWatch(); - // initialize system - stopWatch.record(); - await editorStore.graphManagerState.initializeSystem(); - stopWatch.record( - GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS, - ); - - // build graph - let pmcd: V1_PureModelContextData | undefined = undefined; - if (dataSpaceAnalysisResult) { - const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResult?.executionContexts.find( - (value) => value.mapping === mappingPath, - )?.mappingModelCoverageAnalysisResult; - pmcd = mappingModelCoverageAnalysisResult?.model; - } - const graph_buildReport = createGraphBuilderReport(); - dataSpaceAnalysisResultMetaModel = - await DSL_DataSpace_getGraphManagerExtension( - editorStore.graphManagerState.graphManager, - ).buildDataSpaceAnalytics( - dataSpaceAnalysisResult, - editorStore.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + undefined, graph_buildReport, editorStore.graphManagerState.graph, - pmcd, + undefined, + mappingPath, editorStore.getProjectInfo(), + editorStore.applicationStore.notificationService, ); - const dependency_buildReport = createGraphBuilderReport(); + // report stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); const graphBuilderReportData = { @@ -221,7 +213,7 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli isLightGraphEnabled = false; editorStore.graphManagerState.graph = editorStore.graphManagerState.createNewGraph(); - await editorStore.buildGraph(); + await flowResult(editorStore.buildFullGraph()); } const dataSpace = getOwnDataSpace( @@ -238,8 +230,6 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli // properly created from a data space, therefore, we cannot support this case return undefined; } - await editorStore.setupQuery(); - const query = guaranteeNonNullable(editorStore.query); const projectInfo = new DataSpaceProjectInfo( query.groupId, query.artifactId, @@ -257,7 +247,7 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli dataSpace, matchingExecutionContext, isLightGraphEnabled, - (dataSpaceInfo: DataSpaceInfo) => { + async (dataSpaceInfo: DataSpaceInfo) => { if (dataSpaceInfo.defaultExecutionContext) { const createQuery = async (): Promise => { // prepare the new query to save @@ -354,14 +344,14 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli } }, true, - dataSpaceAnalysisResultMetaModel, + dataSpaceAnalysisResult, undefined, undefined, undefined, projectInfo, ); const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResultMetaModel?.executionContextsIndex.get( + dataSpaceAnalysisResult?.executionContextsIndex.get( matchingExecutionContext.name, )?.mappingModelCoverageAnalysisResult; if (mappingModelCoverageAnalysisResult) { @@ -376,19 +366,24 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli ]; } - override getExtraExistingQueryGraphBuilderCheckers(): ExistingQueryGraphBuilderChecker[] { + override getExtraQueryGraphBuilderGetters(): QueryGraphBuilderGetter[] { return [ - (editorStore: ExistingQueryEditorStore): boolean | undefined => { - const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( - (taggedValue) => - taggedValue.profile === QUERY_PROFILE_PATH && - taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && - isValidFullPath(taggedValue.value), - ); - if (dataSpaceTaggedValue) { - return false; + ( + editorStore: QueryEditorStore, + ): ((editorStore: QueryEditorStore) => GeneratorFn) | undefined => { + if (editorStore instanceof ExistingQueryEditorStore) { + const query = editorStore.query; + const dataSpaceTaggedValue = query?.taggedValues?.find( + (taggedValue) => + taggedValue.profile === QUERY_PROFILE_PATH && + taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && + isValidFullPath(taggedValue.value), + ); + if (dataSpaceTaggedValue) { + return buildGraph; + } } - return true; + return undefined; }, ]; } diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx index 04713bf8315..dd5ac9b5a4e 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx @@ -174,8 +174,10 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( queryBuilderState.dataSpace.defaultExecutionContext.name, }, }; - const onDataSpaceOptionChange = (option: DataSpaceOption): void => { - queryBuilderState.onDataSpaceChange(option.value); + const onDataSpaceOptionChange = async ( + option: DataSpaceOption, + ): Promise => { + await queryBuilderState.onDataSpaceChange(option.value); }; // data space search text @@ -206,14 +208,14 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( const selectedExecutionContextOption = buildExecutionContextOption( queryBuilderState.executionContext, ); - const onExecutionContextOptionChange = ( + const onExecutionContextOptionChange = async ( option: ExecutionContextOption, - ): void => { + ): Promise => { if (option.value === queryBuilderState.executionContext) { return; } queryBuilderState.setExecutionContext(option.value); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( option.value, editorStore, ); @@ -422,7 +424,7 @@ export const queryDataSpace = async ( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: () => { + setupQueryBuilderState: async () => { const queryBuilderState = new DataSpaceQueryBuilderState( editorStore.applicationStore, editorStore.graphManagerState, @@ -430,7 +432,7 @@ export const queryDataSpace = async ( dataSpace, dataSpace.defaultExecutionContext, false, - (dataSpaceInfo: DataSpaceInfo) => { + async (dataSpaceInfo: DataSpaceInfo) => { queryBuilderState.dataSpace = guaranteeType( queryBuilderState.graphManagerState.graph.getElement( dataSpaceInfo.path, @@ -440,7 +442,7 @@ export const queryDataSpace = async ( queryBuilderState.setExecutionContext( queryBuilderState.dataSpace.defaultExecutionContext, ); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( queryBuilderState.dataSpace.defaultExecutionContext, ); }, @@ -455,7 +457,7 @@ export const queryDataSpace = async ( queryBuilderState.setExecutionContext( dataSpace.defaultExecutionContext, ); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( dataSpace.defaultExecutionContext, ); return queryBuilderState; diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts index a8333234336..bdb0538dd74 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts @@ -17,8 +17,6 @@ import { type AbstractPureGraphManager, AbstractPureGraphManagerExtension, - type PureProtocolProcessorPlugin, - type V1_PureModelContextData, type PureModel, type GraphManagerOperationReport, } from '@finos/legend-graph'; @@ -29,7 +27,7 @@ import { type PlainObject, } from '@finos/legend-shared'; import type { DataSpaceAnalysisResult } from '../../action/analytics/DataSpaceAnalysis.js'; -import type { V1_DataSpaceAnalysisResult } from './v1/engine/analytics/V1_DataSpaceAnalysis.js'; +import type { NotificationService } from '@finos/legend-application'; export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPureGraphManagerExtension { abstract analyzeDataSpace( @@ -44,17 +42,12 @@ export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPu entitiesRetriever: () => Promise, cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise; - - abstract buildDataSpaceAnalytics( - analytics: - | PlainObject - | V1_DataSpaceAnalysisResult, - plugins: PureProtocolProcessorPlugin[], graphReport?: GraphManagerOperationReport | undefined, pureGraph?: PureModel | undefined, - pmcd?: V1_PureModelContextData | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, projectInfo?: ProjectGAVCoordinates, + notificationService?: NotificationService | undefined, ): Promise; } diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts index 8f67084c1a4..f6821942ef8 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts @@ -31,7 +31,6 @@ import { V1_buildDatasetSpecification, type PureProtocolProcessorPlugin, V1_buildModelCoverageAnalysisResult, - type V1_PureModelContextData, type GraphManagerOperationReport, LegendSDLC, } from '@finos/legend-graph'; @@ -76,7 +75,7 @@ import { } from '../../../action/analytics/DataSpaceAnalysis.js'; import { DSL_DataSpace_PureGraphManagerExtension } from '../DSL_DataSpace_PureGraphManagerExtension.js'; import { - V1_DataSpaceAnalysisResult, + type V1_DataSpaceAnalysisResult, V1_DataSpaceAssociationDocumentationEntry, V1_DataSpaceClassDocumentationEntry, V1_DataSpaceEnumerationDocumentationEntry, @@ -87,6 +86,7 @@ import { } from './engine/analytics/V1_DataSpaceAnalysis.js'; import { getDiagram } from '@finos/legend-extension-dsl-diagram/graph'; import { resolveVersion } from '@finos/legend-server-depot'; +import type { NotificationService } from '@finos/legend-application'; const ANALYZE_DATA_SPACE_TRACE = 'analyze data space'; const TEMPORARY__TDS_SAMPLE_VALUES__DELIMETER = '-- e.g.'; @@ -150,11 +150,20 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu entitiesRetriever: () => Promise, cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise { + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, + projectInfo?: ProjectGAVCoordinates, + notificationService?: NotificationService | undefined, + ): Promise { const cacheResult = cacheRetriever ? await this.fetchDataSpaceAnalysisFromCache(cacheRetriever, actionState) : undefined; const engineClient = this.graphManager.engine.getEngineServerClient(); + notificationService?.notify( + `Please release a new version of the project and create a new query from that to reduce the load time`, + ); let analysisResult: PlainObject; if ( cacheResult && @@ -189,9 +198,14 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu { enableCompression: true }, ); } - return V1_deserializeDataSpaceAnalysisResult( + return this.buildDataSpaceAnalytics( analysisResult, this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + graphReport, + pureGraph, + executionContext, + mappingPath, + projectInfo, ); } @@ -216,24 +230,18 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu } async buildDataSpaceAnalytics( - analytics: - | PlainObject - | V1_DataSpaceAnalysisResult, + analytics: PlainObject, plugins: PureProtocolProcessorPlugin[], graphReport?: GraphManagerOperationReport | undefined, pureGraph?: PureModel | undefined, - pmcd?: V1_PureModelContextData | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, projectInfo?: ProjectGAVCoordinates, ): Promise { - let analysisResult: V1_DataSpaceAnalysisResult; - if (analytics instanceof V1_DataSpaceAnalysisResult) { - analysisResult = analytics; - } else { - analysisResult = V1_deserializeDataSpaceAnalysisResult( - analytics, - plugins, - ); - } + const analysisResult = V1_deserializeDataSpaceAnalysisResult( + analytics, + plugins, + ); const result = new DataSpaceAnalysisResult(); result.name = analysisResult.name; result.package = analysisResult.package; @@ -334,6 +342,14 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu runtime.runtimeValue = new V1_EngineRuntime(); return runtime; }); + const mappingModelCoverageAnalysisResult = + analysisResult.executionContexts.find( + (value) => value.name === executionContext, + )?.mappingModelCoverageAnalysisResult ?? + analysisResult.executionContexts.find( + (value) => value.mapping === mappingPath, + )?.mappingModelCoverageAnalysisResult; + const pmcd = mappingModelCoverageAnalysisResult?.model; if (pmcd && projectInfo) { graphEntities = pmcd.elements .concat(mappingModels) @@ -342,7 +358,7 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu // so we could rid of it .filter((el) => !graph.getNullableElement(el.path, false)) .map((el) => this.graphManager.elementProtocolToEntity(el)); - await this.graphManager.buildGraph( + await this.graphManager.buildGraphForQuery( graph, graphEntities, ActionState.create(), @@ -354,7 +370,6 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu ), }, graphReport, - true, ); } else { // prepare the model context data diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts index 7d2c366c20c..182fab3b539 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts @@ -59,7 +59,7 @@ import { guaranteeNonNullable, guaranteeType, } from '@finos/legend-shared'; -import { action, flow, makeObservable, observable } from 'mobx'; +import { action, flow, flowResult, makeObservable, observable } from 'mobx'; import { renderDataSpaceQueryBuilderSetupPanelContent } from '../../components/query/DataSpaceQueryBuilder.js'; import { DataSpace, @@ -69,7 +69,10 @@ import { DATA_SPACE_ELEMENT_CLASSIFIER_PATH } from '../../graph-manager/protocol import { type DataSpaceInfo, extractDataSpaceInfo } from './DataSpaceInfo.js'; import { DataSpaceAdvancedSearchState } from './DataSpaceAdvancedSearchState.js'; import type { DataSpaceAnalysisResult } from '../../graph-manager/action/analytics/DataSpaceAnalysis.js'; -import type { QueryEditorStore } from '@finos/legend-application-query'; +import { + LEGEND_QUERY_APP_EVENT, + type QueryEditorStore, +} from '@finos/legend-application-query'; export const resolveUsableDataSpaceClasses = ( queryBuilderState: DataSpaceQueryBuilderState, @@ -92,7 +95,7 @@ export const resolveUsableDataSpaceClasses = ( ) { const compatibleClassPaths = queryBuilderState.explorerState.mappingModelCoverageAnalysisResult.mappedEntities.map( - (e) => e.classPath, + (e) => e.info?.classPath, ); const uniqueCompatibleClasses = compatibleClassPaths.filter( (val, index) => compatibleClassPaths.indexOf(val) === index, @@ -148,7 +151,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { readonly depotServerClient: DepotServerClient; readonly isAdvancedDataSpaceSearchEnabled: boolean; readonly loadDataSpacesState = ActionState.create(); - readonly onDataSpaceChange: (val: DataSpaceInfo) => void; + readonly onDataSpaceChange: (val: DataSpaceInfo) => Promise; readonly onExecutionContextChange?: | ((val: DataSpaceExecutionContext) => void) | undefined; @@ -174,7 +177,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { dataSpace: DataSpace, executionContext: DataSpaceExecutionContext, isLightGraphEnabled: boolean, - onDataSpaceChange: (val: DataSpaceInfo) => void, + onDataSpaceChange: (val: DataSpaceInfo) => Promise, isAdvancedDataSpaceSearchEnabled: boolean, dataSpaceAnalysisResult?: DataSpaceAnalysisResult | undefined, onExecutionContextChange?: @@ -324,96 +327,116 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { )?.mappingModelCoverageAnalysisResult; if (this.dataSpaceAnalysisResult && mappingModelCoverageAnalysisResult) { if (!isGraphBuildingNotRequired && editorStore) { - const stopWatch = new StopWatch(); - const graph = this.graphManagerState.createNewGraph(); + try { + const stopWatch = new StopWatch(); + const graph = this.graphManagerState.createNewGraph(); - const graph_buildReport = createGraphBuilderReport(); - // Create dummy mappings and runtimes - // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should - // simplify the model here or potentially refactor the backend analytics endpoint to return these as model - const mappingModels = uniq( - Array.from( - this.dataSpaceAnalysisResult.executionContextsIndex.values(), - ).map((context) => context.mapping), - ).map((m) => { - const _mapping = new V1_Mapping(); - const [packagePath, name] = resolvePackagePathAndElementName(m.path); - _mapping.package = packagePath; - _mapping.name = name; - return guaranteeType( - this.graphManagerState.graphManager, - V1_PureGraphManager, - ).elementProtocolToEntity(_mapping); - }); - const runtimeModels = uniq( - Array.from( - this.dataSpaceAnalysisResult.executionContextsIndex.values(), + const graph_buildReport = createGraphBuilderReport(); + // Create dummy mappings and runtimes + // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should + // simplify the model here or potentially refactor the backend analytics endpoint to return these as model + const mappingModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).map((context) => context.mapping), + ).map((m) => { + const _mapping = new V1_Mapping(); + const [packagePath, name] = resolvePackagePathAndElementName( + m.path, + ); + _mapping.package = packagePath; + _mapping.name = name; + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(_mapping); + }); + const runtimeModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ) + .map((context) => context.defaultRuntime) + .concat( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).flatMap((val) => val.compatibleRuntimes), + ), + ).map((r) => { + const runtime = new V1_PackageableRuntime(); + const [packagePath, name] = resolvePackagePathAndElementName( + r.path, + ); + runtime.package = packagePath; + runtime.name = name; + runtime.runtimeValue = new V1_EngineRuntime(); + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(runtime); + }); + const graphEntities = guaranteeNonNullable( + mappingModelCoverageAnalysisResult.entities, ) - .map((context) => context.defaultRuntime) - .concat( - Array.from( - this.dataSpaceAnalysisResult.executionContextsIndex.values(), - ).flatMap((val) => val.compatibleRuntimes), - ), - ).map((r) => { - const runtime = new V1_PackageableRuntime(); - const [packagePath, name] = resolvePackagePathAndElementName(r.path); - runtime.package = packagePath; - runtime.name = name; - runtime.runtimeValue = new V1_EngineRuntime(); - return guaranteeType( - this.graphManagerState.graphManager, - V1_PureGraphManager, - ).elementProtocolToEntity(runtime); - }); - const graphEntities = guaranteeNonNullable( - mappingModelCoverageAnalysisResult.entities, - ) - .concat(mappingModels) - .concat(runtimeModels) - // NOTE: if an element could be found in the graph already it means it comes from system - // so we could rid of it - .filter( - (el) => - !graph.getNullableElement(el.path, false) && - !el.path.startsWith('meta::'), + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter( + (el) => + !graph.getNullableElement(el.path, false) && + !el.path.startsWith('meta::'), + ); + await this.graphManagerState.graphManager.buildGraphForQuery( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + guaranteeNonNullable(this.projectInfo).groupId, + guaranteeNonNullable(this.projectInfo).artifactId, + resolveVersion( + guaranteeNonNullable(this.projectInfo).versionId, + ), + ), + }, + graph_buildReport, ); - await this.graphManagerState.graphManager.buildGraph( - graph, - graphEntities, - ActionState.create(), - { - origin: new LegendSDLC( - guaranteeNonNullable(this.projectInfo).groupId, - guaranteeNonNullable(this.projectInfo).artifactId, - resolveVersion(guaranteeNonNullable(this.projectInfo).versionId), - ), - }, - graph_buildReport, - true, - ); - this.graphManagerState.graph = graph; - const dependency_buildReport = createGraphBuilderReport(); - // report - stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); - const graphBuilderReportData = { - timings: - this.applicationStore.timeService.finalizeTimingsRecord(stopWatch), - dependencies: dependency_buildReport, - dependenciesCount: - this.graphManagerState.graph.dependencyManager.numberOfDependencies, - graph: graph_buildReport, - }; - editorStore.logBuildGraphMetrics(graphBuilderReportData); + this.graphManagerState.graph = graph; + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord( + stopWatch, + ), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager + .numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); - this.applicationStore.logService.info( - LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), - graphBuilderReportData, - ); + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } catch (error) { + assertErrorThrown(error); + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + + editorStore.graphManagerState.graph = + editorStore.graphManagerState.createNewGraph(); + await flowResult(editorStore.buildFullGraph()); + } } const compatibleClassPaths = mappingModelCoverageAnalysisResult.mappedEntities.map( - (e) => e.classPath, + (e) => e.info?.classPath, ); const uniqueCompatibleClasses = compatibleClassPaths.filter( (val, index) => compatibleClassPaths.indexOf(val) === index, diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts index 4291e2e224e..d0fb4a4c718 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts @@ -23,7 +23,6 @@ import { type Runtime, type Class, type RawLambda, - type V1_PureModelContextData, createGraphBuilderReport, GRAPH_MANAGER_EVENT, } from '@finos/legend-graph'; @@ -46,6 +45,7 @@ import { LogEvent, StopWatch, uuid, + type GeneratorFn, } from '@finos/legend-shared'; import { QUERY_PROFILE_PATH, @@ -64,6 +64,7 @@ import type { QueryBuilderState } from '@finos/legend-query-builder'; import type { ProjectGAVCoordinates } from '@finos/legend-storage'; import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; +import { flowResult } from 'mobx'; export const createQueryDataSpaceTaggedValue = ( dataSpacePath: string, @@ -130,7 +131,6 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { async initializeQueryBuilderState(): Promise { let dataSpaceAnalysisResult; - let dataSpaceAnalysisResultMetaModel; let isLightGraphEnabled = true; try { const stopWatch = new StopWatch(); @@ -138,6 +138,13 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { await this.depotServerClient.getProject(this.groupId, this.artifactId), ); this.initState.setMessage('Fetching dataspace analysis result'); + // initialize system + stopWatch.record(); + await this.graphManagerState.initializeSystem(); + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS); + + const graph_buildReport = createGraphBuilderReport(); + const dependency_buildReport = createGraphBuilderReport(); dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( this.graphManagerState.graphManager, ).analyzeDataSpaceCoverage( @@ -155,38 +162,18 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { this.dataSpacePath, this.depotServerClient, ), + undefined, + graph_buildReport, + this.graphManagerState.graph, + this.executionContext, + undefined, + this.getProjectInfo(), + this.applicationStore.notificationService, ); - const mappingPath = dataSpaceAnalysisResult?.executionContexts.find( - (e) => e.name === this.executionContext, + const mappingPath = dataSpaceAnalysisResult.executionContextsIndex.get( + this.executionContext, )?.mapping; - if (dataSpaceAnalysisResult && mappingPath) { - // initialize system - stopWatch.record(); - await this.graphManagerState.initializeSystem(); - stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS); - - // build graph - let pmcd: V1_PureModelContextData | undefined = undefined; - if (dataSpaceAnalysisResult) { - const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResult?.executionContexts.find( - (value) => value.mapping === mappingPath, - )?.mappingModelCoverageAnalysisResult; - pmcd = mappingModelCoverageAnalysisResult?.model; - } - const graph_buildReport = createGraphBuilderReport(); - dataSpaceAnalysisResultMetaModel = - await DSL_DataSpace_getGraphManagerExtension( - this.graphManagerState.graphManager, - ).buildDataSpaceAnalytics( - dataSpaceAnalysisResult, - this.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), - graph_buildReport, - this.graphManagerState.graph, - pmcd, - this.getProjectInfo(), - ); - const dependency_buildReport = createGraphBuilderReport(); + if (mappingPath) { // report stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); const graphBuilderReportData = { @@ -204,7 +191,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { } else { isLightGraphEnabled = false; this.graphManagerState.graph = this.graphManagerState.createNewGraph(); - await this.buildGraph(); + await flowResult(this.buildFullGraph()); } } catch (error) { this.applicationStore.logService.error( @@ -213,7 +200,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { ); this.graphManagerState.graph = this.graphManagerState.createNewGraph(); isLightGraphEnabled = false; - await this.buildGraph(); + await flowResult(this.buildFullGraph()); } const dataSpace = getDataSpace( this.dataSpacePath, @@ -242,7 +229,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { dataSpace, executionContext, isLightGraphEnabled, - (dataSpaceInfo: DataSpaceInfo) => { + async (dataSpaceInfo: DataSpaceInfo) => { if (dataSpaceInfo.defaultExecutionContext) { this.applicationStore.navigationService.navigator.goToLocation( generateDataSpaceQueryCreatorRoute( @@ -262,7 +249,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { } }, true, - dataSpaceAnalysisResultMetaModel, + dataSpaceAnalysisResult, (ec: DataSpaceExecutionContext) => { // runtime should already be set const runtimePointer = guaranteeType( @@ -325,7 +312,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { projectInfo, ); queryBuilderState.setExecutionContext(executionContext); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( executionContext, this, true, @@ -376,4 +363,8 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { }, }; } + + override *buildGraph(): GeneratorFn { + // do nothing + } } diff --git a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts index 4a3ddebeded..6d00ecf13ed 100644 --- a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts @@ -254,6 +254,17 @@ export abstract class AbstractPureGraphManager { report?: GraphManagerOperationReport, ): Promise; + /** + * Process entities and build the main graph for query. + */ + abstract buildGraphForQuery( + graph: PureModel, + entities: Entity[], + buildState: ActionState, + options?: GraphBuilderOptions, + report?: GraphManagerOperationReport, + ): Promise; + /** * Process entities and build the main graph. */ @@ -263,7 +274,6 @@ export abstract class AbstractPureGraphManager { buildState: ActionState, options?: GraphBuilderOptions, report?: GraphManagerOperationReport, - buildRequiredGraph?: boolean | undefined, ): Promise; /** diff --git a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts index 2857f19bf29..8d8ed66a75c 100644 --- a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts @@ -19,17 +19,35 @@ import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/ export type RawMappingModelCoverageAnalysisResult = object; +export class MappedEntityInfo { + readonly __PROPERTIES_INDEX = new Map(); + + classPath: string; + isRootEntity: boolean; + subClasses: string[]; + + constructor(classPath: string, isRootEntity: boolean, subClasses: string[]) { + this.isRootEntity = isRootEntity; + this.subClasses = subClasses; + this.classPath = classPath; + } +} + export class MappedEntity { readonly __PROPERTIES_INDEX = new Map(); path: string; - classPath: string; properties: MappedProperty[]; + info?: MappedEntityInfo | undefined; - constructor(path: string, classPath: string, properties: MappedProperty[]) { + constructor( + path: string, + properties: MappedProperty[], + info?: MappedEntityInfo | undefined, + ) { this.path = path; - this.classPath = classPath; this.properties = properties; + this.info = info; properties.forEach((property) => this.__PROPERTIES_INDEX.set(property.name, property), ); diff --git a/packages/legend-graph/src/graph-manager/action/query/Query.ts b/packages/legend-graph/src/graph-manager/action/query/Query.ts index 29c88c22115..af9ce1c11ba 100644 --- a/packages/legend-graph/src/graph-manager/action/query/Query.ts +++ b/packages/legend-graph/src/graph-manager/action/query/Query.ts @@ -14,10 +14,6 @@ * limitations under the License. */ -import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js'; -import type { PackageableElementReference } from '../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js'; -import type { PackageableRuntime } from '../../../graph/metamodel/pure/packageableElements/runtime/PackageableRuntime.js'; - export class QueryTaggedValue { profile!: string; tag!: string; @@ -40,8 +36,9 @@ export class Query { versionId!: string; groupId!: string; artifactId!: string; - mapping!: PackageableElementReference; - runtime!: PackageableElementReference; + // NOTE: Query can be built before the actual graph is built so we can't have the reference of metamodels here + mapping!: string; + runtime!: string; // We enforce a single owner, for collaboration on query, use Studio // if not owner is specified, any user can own the query // NOTE: the owner is managed automatically by the backend @@ -67,8 +64,6 @@ export class LightQuery { artifactId!: string; owner?: string | undefined; lastUpdatedAt?: number | undefined; - mapping?: string | PackageableElementReference | undefined; - taggedValues?: QueryTaggedValue[] | undefined; isCurrentUserQuery = false; } @@ -82,8 +77,6 @@ export const toLightQuery = (query: Query): LightQuery => { lightQuery.versionId = query.versionId; lightQuery.owner = query.owner; lightQuery.isCurrentUserQuery = query.isCurrentUserQuery; - lightQuery.mapping = query.mapping.value.path; - lightQuery.taggedValues = query.taggedValues; return lightQuery; }; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts index e8fa4aa7d2a..0138b4acaf4 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts @@ -797,13 +797,99 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { } } + async buildGraphForQuery( + graph: PureModel, + entities: Entity[], + buildState: ActionState, + options?: GraphBuilderOptions, + _report?: GraphManagerOperationReport, + ): Promise { + const stopWatch = new StopWatch(); + const report = _report ?? createGraphBuilderReport(); + buildState.reset(); + + try { + // deserialize + buildState.setMessage(`Deserializing elements...`); + const data = new V1_PureModelContextData(); + await V1_entitiesToPureModelContextData( + entities, + data, + this.pluginManager.getPureProtocolProcessorPlugins(), + this.subtypeInfo, + ); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS, + ); + + // prepare build inputs + const buildInputs: V1_PureGraphBuilderInput[] = [ + { + model: graph, + data: V1_indexPureModelContextData( + report, + data, + this.graphBuilderExtensions, + ), + }, + ]; + + // index + buildState.setMessage( + `Indexing ${report.elementCount.total} elements...`, + ); + await this.initializeAndIndexElements(graph, buildInputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS, + ); + // build types + buildState.setMessage(`Building domain models...`); + await this.buildTypes(graph, buildInputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS, + ); + + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS, + ); + + if (options?.origin) { + graph.setOrigin(options.origin); + } + + buildState.pass(); + + const totalTime = stopWatch.elapsed; + report.timings = { + ...Object.fromEntries(stopWatch.records), + [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime, + total: totalTime, + }; + } catch (error) { + assertErrorThrown(error); + this.logService.error( + LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), + error, + ); + buildState.fail(); + /** + * Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder + * But we might want to rethink this decision in the future and throw appropriate type of error + */ + throw error instanceof GraphBuilderError + ? error + : new GraphBuilderError(error); + } finally { + buildState.setMessage(undefined); + } + } + async buildGraph( graph: PureModel, entities: Entity[], buildState: ActionState, options?: GraphBuilderOptions, _report?: GraphManagerOperationReport, - buildRequiredGraph?: boolean | undefined, ): Promise { const stopWatch = new StopWatch(); const report = _report ?? createGraphBuilderReport(); @@ -837,25 +923,14 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ]; // build - if (!buildRequiredGraph) { - await this.buildGraphFromInputs( - graph, - buildInputs, - report, - stopWatch, - buildState, - options, - ); - } else { - await this.buildRequiredGraphFromInputs( - graph, - buildInputs, - report, - stopWatch, - buildState, - options, - ); - } + await this.buildGraphFromInputs( + graph, + buildInputs, + report, + stopWatch, + buildState, + options, + ); /** * For now, we delete the section index. We are able to read both resolved and unresolved element paths @@ -1142,32 +1217,6 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ); } - private async buildRequiredGraphFromInputs( - graph: PureModel, - inputs: V1_PureGraphBuilderInput[], - report: GraphManagerOperationReport, - stopWatch: StopWatch, - graphBuilderState: ActionState, - options?: GraphBuilderOptions, - ): Promise { - // index - graphBuilderState.setMessage( - `Indexing ${report.elementCount.total} elements...`, - ); - await this.initializeAndIndexElements(graph, inputs, options); - stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS); - // build types - graphBuilderState.setMessage(`Building domain models...`); - await this.buildTypes(graph, inputs, options); - stopWatch.record( - GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS, - ); - - stopWatch.record( - GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS, - ); - } - private async buildLightGraphFromInputs( graph: PureModel, inputs: V1_PureGraphBuilderInput[], diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts index b120b63841f..b7617df2454 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts @@ -32,7 +32,6 @@ import { V1_QueryParameterValue, } from './query/V1_Query.js'; import type { PureModel } from '../../../../../graph/PureModel.js'; -import { PackageableElementExplicitReference } from '../../../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js'; import { DEPRECATED__ServiceTestResult } from '../../../../../graph-manager/action/service/DEPRECATED__ServiceTestResult.js'; import type { V1_DEPRECATED__ServiceTestResult } from './service/V1_DEPRECATED__ServiceTestResult.js'; import type { V1_ServiceRegistrationResult } from './service/V1_ServiceRegistrationResult.js'; @@ -97,29 +96,10 @@ export const V1_buildLightQuery = ( protocol.artifactId, `Query 'artifactId' field is missing`, ); - metamodel.mapping = protocol.mapping; metamodel.owner = protocol.owner; metamodel.lastUpdatedAt = protocol.lastUpdatedAt; metamodel.isCurrentUserQuery = currentUserId !== undefined && protocol.owner === currentUserId; - // NOTE: we don't properly process tagged values and stereotypes for query - // because these profiles/tags/stereotypes can come from external systems. - metamodel.taggedValues = protocol.taggedValues?.map((taggedValueProtocol) => { - const taggedValue = new QueryTaggedValue(); - taggedValue.profile = guaranteeNonEmptyString( - taggedValueProtocol.tag.profile, - `Tagged value 'tag.profile' field is missing or empty`, - ); - taggedValue.tag = guaranteeNonEmptyString( - taggedValueProtocol.tag.value, - `Tagged value 'tag.value' field is missing or empty`, - ); - taggedValue.value = guaranteeNonEmptyString( - taggedValueProtocol.value, - `Tagged value 'value' field is missing or empty`, - ); - return taggedValue; - }); return metamodel; }; @@ -149,22 +129,8 @@ export const V1_buildQuery = ( protocol.artifactId, `Query 'artifactId' field is missing`, ); - metamodel.mapping = PackageableElementExplicitReference.create( - graph.getMapping( - guaranteeNonNullable( - protocol.mapping, - `Query 'mapping' field is missing`, - ), - ), - ); - metamodel.runtime = PackageableElementExplicitReference.create( - graph.getRuntime( - guaranteeNonNullable( - protocol.runtime, - `Query 'runtime' field is missing`, - ), - ), - ); + metamodel.mapping = protocol.mapping; + metamodel.runtime = protocol.runtime; metamodel.content = guaranteeNonNullable( protocol.content, `Query 'content' field is missing`, @@ -229,8 +195,8 @@ export const V1_transformQuery = (metamodel: Query): V1_Query => { protocol.versionId = metamodel.versionId; protocol.groupId = metamodel.groupId; protocol.artifactId = metamodel.artifactId; - protocol.mapping = metamodel.mapping.valueForSerialization ?? ''; - protocol.runtime = metamodel.runtime.valueForSerialization ?? ''; + protocol.mapping = metamodel.mapping; + protocol.runtime = metamodel.runtime; protocol.content = metamodel.content; protocol.owner = metamodel.owner; protocol.taggedValues = metamodel.taggedValues?.map((_taggedValue) => { diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts index cf3d1d36458..6a45611f233 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts @@ -33,6 +33,7 @@ import { EntityMappedProperty, EnumMappedProperty, MappedEntity, + MappedEntityInfo, MappedProperty, MappingModelCoverageAnalysisResult, } from '../../../../../../graph-manager/action/analytics/MappingModelCoverageAnalysis.js'; @@ -107,15 +108,31 @@ const V1_deserializeMappedProperty = ( } }; +class V1_MappedEntityInfo { + classPath!: string; + isRootEntity!: boolean; + subClasses: string[] = []; + + static readonly serialization = new SerializationFactory( + createModelSchema(V1_MappedEntityInfo, { + classPath: primitive(), + isRootEntity: primitive(), + subClasses: list(primitive()), + }), + ); +} + class V1_MappedEntity { path!: string; - classPath!: string; properties: V1_MappedProperty[] = []; + info?: V1_MappedEntityInfo | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappedEntity, { path: primitive(), - classPath: primitive(), + info: optional( + usingModelSchema(V1_MappedEntityInfo.serialization.schema), + ), properties: list( custom( (prop) => V1_serializeMappedProperty(prop), @@ -170,12 +187,23 @@ const buildMappedProperty = (protocol: V1_MappedProperty): MappedProperty => ? new EnumMappedProperty(protocol.name, protocol.enumPath) : new MappedProperty(protocol.name); -const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => - new MappedEntity( - protocol.path, +const buildMappedEntityInfo = ( + protocol: V1_MappedEntityInfo, +): MappedEntityInfo => + new MappedEntityInfo( protocol.classPath, + protocol.isRootEntity, + protocol.subClasses, + ); + +const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => { + const info = protocol.info ? buildMappedEntityInfo(protocol.info) : undefined; + return new MappedEntity( + protocol.path, protocol.properties.map((p) => buildMappedProperty(p)), + info, ); +}; export const V1_buildModelCoverageAnalysisResult = ( protocol: V1_MappingModelCoverageAnalysisResult,