From 75dc805f50b1b0f520d6ceb5d24b4dfb094e6b07 Mon Sep 17 00:00:00 2001 From: Gayathri Date: Thu, 17 Aug 2023 16:15:54 +0530 Subject: [PATCH 1/2] Use PMCD returned by mappinganalysis to build minimal graph for query --- .changeset/clean-toes-learn.md | 3 + .changeset/kind-rabbits-draw.md | 6 + .../stores/LegendQueryApplicationPlugin.ts | 11 +- .../src/stores/QueryEditorStore.ts | 39 +- ...DataSpace_LegendQueryApplicationPlugin.tsx | 424 +++++++++++------- .../query/DataSpaceQueryBuilder.tsx | 45 +- ...DSL_DataSpace_PureGraphManagerExtension.ts | 26 +- ...DSL_DataSpace_PureGraphManagerExtension.ts | 251 +++++++---- .../query/DataSpaceQueryBuilderState.ts | 162 ++++++- .../query/DataSpaceQueryCreatorStore.ts | 124 ++++- .../graph-manager/AbstractPureGraphManager.ts | 1 + .../analytics/MappingModelCoverageAnalysis.ts | 13 +- .../src/graph-manager/action/query/Query.ts | 4 + .../protocol/pure/v1/V1_PureGraphManager.ts | 56 ++- .../pure/v1/engine/V1_EngineHelper.ts | 19 + .../V1_MappingModelCoverageAnalysis.ts | 28 +- .../protocol/pure/v1/engine/query/V1_Query.ts | 4 + 17 files changed, 907 insertions(+), 309 deletions(-) create mode 100644 .changeset/clean-toes-learn.md create mode 100644 .changeset/kind-rabbits-draw.md diff --git a/.changeset/clean-toes-learn.md b/.changeset/clean-toes-learn.md new file mode 100644 index 0000000000..d78bbd6f64 --- /dev/null +++ b/.changeset/clean-toes-learn.md @@ -0,0 +1,3 @@ +--- +'@finos/legend-graph': patch +--- diff --git a/.changeset/kind-rabbits-draw.md b/.changeset/kind-rabbits-draw.md new file mode 100644 index 0000000000..457d23317c --- /dev/null +++ b/.changeset/kind-rabbits-draw.md @@ -0,0 +1,6 @@ +--- +'@finos/legend-extension-dsl-data-space': minor +'@finos/legend-application-query': minor +--- + +Use PMCD returned by mapping analysis to build minimal graph for query diff --git a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts index 1e90584b97..c54479278c 100644 --- a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts +++ b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts @@ -15,7 +15,6 @@ */ import { LegendApplicationPlugin } from '@finos/legend-application'; -import type { Query } from '@finos/legend-graph'; import type { QueryBuilderState } from '@finos/legend-query-builder'; import type React from 'react'; import type { LegendQueryPluginManager } from '../application/LegendQueryPluginManager.js'; @@ -52,10 +51,13 @@ export type QuerySetupActionConfiguration = { }; export type ExistingQueryEditorStateBuilder = ( - query: Query, editorStore: ExistingQueryEditorStore, ) => Promise; +export type ExistingQueryGraphBuilderChecker = ( + editorStore: ExistingQueryEditorStore, +) => boolean | undefined; + export type QueryEditorActionConfiguration = { key: string; renderer: ( @@ -98,6 +100,11 @@ export abstract class LegendQueryApplicationPlugin extends LegendApplicationPlug */ getExtraExistingQueryEditorStateBuilders?(): ExistingQueryEditorStateBuilder[]; + /** + * Get the list of existing query graph builder checkers + */ + getExtraExistingQueryGraphBuilderCheckers?(): ExistingQueryGraphBuilderChecker[]; + /** * Get the list of query editor action renderer configurations. */ diff --git a/packages/legend-application-query/src/stores/QueryEditorStore.ts b/packages/legend-application-query/src/stores/QueryEditorStore.ts index 1e0416e09e..c055e4bc70 100644 --- a/packages/legend-application-query/src/stores/QueryEditorStore.ts +++ b/packages/legend-application-query/src/stores/QueryEditorStore.ts @@ -381,6 +381,10 @@ export abstract class QueryEditorStore { // do nothing } + requiresGraphBuilding(): boolean { + return true; + } + async buildQueryForPersistence( query: Query, rawLambda: RawLambda, @@ -485,7 +489,9 @@ export abstract class QueryEditorStore { ); yield this.setUpEditorState(); - yield flowResult(this.buildGraph()); + if (this.requiresGraphBuilding()) { + yield flowResult(this.buildGraph()); + } this.queryBuilderState = (yield this.initializeQueryBuilderState( stopWatch, )) as QueryBuilderState; @@ -991,9 +997,24 @@ export class ExistingQueryEditorStore extends QueryEditorStore { ); } - async initializeQueryBuilderState( - stopWatch: StopWatch, - ): Promise { + override requiresGraphBuilding(): boolean { + const existingQueryGraphBuilderCheckers = + this.applicationStore.pluginManager + .getApplicationPlugins() + .flatMap( + (plugin) => + plugin.getExtraExistingQueryGraphBuilderCheckers?.() ?? [], + ); + for (const checker of existingQueryGraphBuilderCheckers) { + const isGraphBuildingRequired = checker(this); + if (isGraphBuildingRequired !== undefined) { + return isGraphBuildingRequired; + } + } + return true; + } + + async setupQuery(): Promise { const query = await this.graphManagerState.graphManager.getQuery( this.queryId, this.graphManagerState.graph, @@ -1003,6 +1024,11 @@ export class ExistingQueryEditorStore extends QueryEditorStore { this.applicationStore.userDataService, query.id, ); + } + + async initializeQueryBuilderState( + stopWatch: StopWatch, + ): Promise { let queryBuilderState: QueryBuilderState | undefined; const existingQueryEditorStateBuilders = this.applicationStore.pluginManager .getApplicationPlugins() @@ -1010,12 +1036,15 @@ export class ExistingQueryEditorStore extends QueryEditorStore { (plugin) => plugin.getExtraExistingQueryEditorStateBuilders?.() ?? [], ); for (const builder of existingQueryEditorStateBuilders) { - queryBuilderState = await builder(query, this); + queryBuilderState = await builder(this); if (queryBuilderState) { break; } } + await this.setupQuery(); + const query = guaranteeNonNullable(this.query); + // if no extension found, fall back to basic `class -> mapping -> runtime` mode queryBuilderState = queryBuilderState ?? diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx index c969f796ee..6e142a080f 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx @@ -17,7 +17,6 @@ import packageJson from '../../../package.json' assert { type: 'json' }; import { type QuerySetupActionConfiguration, - type ExistingQueryEditorStateBuilder, type ExistingQueryEditorStore, LegendQueryApplicationPlugin, generateExistingQueryEditorRoute, @@ -25,6 +24,8 @@ import { LegendQueryEventHelper, createViewProjectHandler, createViewSDLCProjectHandler, + type ExistingQueryEditorStateBuilder, + type ExistingQueryGraphBuilderChecker, } from '@finos/legend-application-query'; import { SquareIcon } from '@finos/legend-art'; import { @@ -38,22 +39,38 @@ import { } from '../../__lib__/query/DSL_DataSpace_LegendQueryNavigation.js'; import { DataSpaceQueryCreator } from './DataSpaceQueryCreator.js'; import { createQueryDataSpaceTaggedValue } from '../../stores/query/DataSpaceQueryCreatorStore.js'; -import { Query, isValidFullPath } from '@finos/legend-graph'; +import { + Query, + isValidFullPath, + GRAPH_MANAGER_EVENT, + type V1_PureModelContextData, + createGraphBuilderReport, +} from '@finos/legend-graph'; import { QUERY_PROFILE_PATH, QUERY_PROFILE_TAG_DATA_SPACE, } from '../../graph/DSL_DataSpace_MetaModelConst.js'; import { - DataSpaceQueryBuilderState, DataSpaceProjectInfo, + DataSpaceQueryBuilderState, } from '../../stores/query/DataSpaceQueryBuilderState.js'; import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { getOwnDataSpace } from '../../graph-manager/DSL_DataSpace_GraphManagerHelper.js'; -import { assertErrorThrown, LogEvent, uuid } from '@finos/legend-shared'; +import { + assertErrorThrown, + guaranteeNonNullable, + isString, + LogEvent, + StopWatch, + uuid, +} from '@finos/legend-shared'; import type { QueryBuilderState } from '@finos/legend-query-builder'; import { DataSpaceQuerySetup } from './DataSpaceQuerySetup.js'; import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js'; -import { StoreProjectData } from '@finos/legend-server-depot'; +import { + retrieveProjectEntitiesWithDependencies, + StoreProjectData, +} from '@finos/legend-server-depot'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin { @@ -101,185 +118,278 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli override getExtraExistingQueryEditorStateBuilders(): ExistingQueryEditorStateBuilder[] { return [ async ( - query: Query, editorStore: ExistingQueryEditorStore, ): Promise => { - const dataSpaceTaggedValue = query.taggedValues?.find( + const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( (taggedValue) => taggedValue.profile === QUERY_PROFILE_PATH && taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && isValidFullPath(taggedValue.value), ); - + let isLightGraphEnabled = true; if (dataSpaceTaggedValue) { const dataSpacePath = dataSpaceTaggedValue.value; - const dataSpace = getOwnDataSpace( - dataSpacePath, - editorStore.graphManagerState.graph, - ); - const mapping = query.mapping.value; - const matchingExecutionContext = dataSpace.executionContexts.find( - (ec) => ec.mapping.value === mapping, - ); - if (!matchingExecutionContext) { - // if a matching execution context is not found, it means this query is not - // properly created from a data space, therefore, we cannot support this case - return undefined; - } + const mappingPath = editorStore.lightQuery?.mapping; let dataSpaceAnalysisResult; - try { - const project = StoreProjectData.serialization.fromJson( - await editorStore.depotServerClient.getProject( - query.groupId, - query.artifactId, - ), - ); - dataSpaceAnalysisResult = - await DSL_DataSpace_getGraphManagerExtension( - editorStore.graphManagerState.graphManager, - ).retrieveDataSpaceAnalysisFromCache(() => - retrieveAnalyticsResultCache( - project, - query.versionId, - dataSpace.path, - editorStore.depotServerClient, + let dataSpaceAnalysisResultMetaModel; + const { groupId, artifactId, versionId } = + editorStore.getProjectInfo(); + if (dataSpacePath && isString(mappingPath)) { + try { + editorStore.initState.setMessage( + 'Fetching dataspace analysis result', + ); + const project = StoreProjectData.serialization.fromJson( + await editorStore.depotServerClient.getProject( + groupId, + artifactId, ), ); - } catch { - // do nothing - } - const projectInfo = new DataSpaceProjectInfo( - query.groupId, - query.artifactId, - query.versionId, - createViewProjectHandler(editorStore.applicationStore), - createViewSDLCProjectHandler( + dataSpaceAnalysisResult = + await DSL_DataSpace_getGraphManagerExtension( + editorStore.graphManagerState.graphManager, + ).analyzeDataSpaceCoverage( + dataSpacePath, + () => + retrieveProjectEntitiesWithDependencies( + project, + versionId, + editorStore.depotServerClient, + ), + () => + retrieveAnalyticsResultCache( + project, + versionId, + dataSpacePath, + editorStore.depotServerClient, + ), + ); + const stopWatch = new StopWatch(); + // initialize system + stopWatch.record(); + await editorStore.graphManagerState.initializeSystem(); + stopWatch.record( + GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS, + ); + + // build graph + let pmcd: V1_PureModelContextData | undefined = undefined; + if (dataSpaceAnalysisResult) { + const mappingModelCoverageAnalysisResult = + dataSpaceAnalysisResult?.executionContexts.find( + (value) => value.mapping === mappingPath, + )?.mappingModelCoverageAnalysisResult; + pmcd = mappingModelCoverageAnalysisResult?.model; + } + const graph_buildReport = createGraphBuilderReport(); + dataSpaceAnalysisResultMetaModel = + await DSL_DataSpace_getGraphManagerExtension( + editorStore.graphManagerState.graphManager, + ).buildDataSpaceAnalytics( + dataSpaceAnalysisResult, + editorStore.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + graph_buildReport, + editorStore.graphManagerState.graph, + pmcd, + editorStore.getProjectInfo(), + ); + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + editorStore.applicationStore.timeService.finalizeTimingsRecord( + stopWatch, + ), + dependencies: dependency_buildReport, + dependenciesCount: + editorStore.graphManagerState.graph.dependencyManager + .numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); + + editorStore.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } catch (error) { + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + isLightGraphEnabled = false; + editorStore.graphManagerState.graph = + editorStore.graphManagerState.createNewGraph(); + await editorStore.buildGraph(); + } + + const dataSpace = getOwnDataSpace( + dataSpacePath, + editorStore.graphManagerState.graph, + ); + const mapping = + editorStore.graphManagerState.graph.getMapping(mappingPath); + const matchingExecutionContext = dataSpace.executionContexts.find( + (ec) => ec.mapping.value === mapping, + ); + if (!matchingExecutionContext) { + // if a matching execution context is not found, it means this query is not + // properly created from a data space, therefore, we cannot support this case + return undefined; + } + await editorStore.setupQuery(); + const query = guaranteeNonNullable(editorStore.query); + const projectInfo = new DataSpaceProjectInfo( + query.groupId, + query.artifactId, + query.versionId, + createViewProjectHandler(editorStore.applicationStore), + createViewSDLCProjectHandler( + editorStore.applicationStore, + editorStore.depotServerClient, + ), + ); + const dataSpaceQueryBuilderState = new DataSpaceQueryBuilderState( editorStore.applicationStore, + editorStore.graphManagerState, editorStore.depotServerClient, - ), - ); - const dataSpaceQueryBuilderState = new DataSpaceQueryBuilderState( - editorStore.applicationStore, - editorStore.graphManagerState, - editorStore.depotServerClient, - dataSpace, - matchingExecutionContext, - (dataSpaceInfo: DataSpaceInfo) => { - if (dataSpaceInfo.defaultExecutionContext) { - const createQuery = async (): Promise => { - // prepare the new query to save - const _query = new Query(); - _query.name = query.name; - _query.id = query.id; - _query.groupId = query.groupId; - _query.artifactId = query.artifactId; - _query.versionId = query.versionId; - _query.mapping = query.mapping; - _query.runtime = query.runtime; - _query.taggedValues = [ - createQueryDataSpaceTaggedValue(dataSpaceInfo.path), - ].concat( - (query.taggedValues ?? []).filter( - (taggedValue) => taggedValue !== dataSpaceTaggedValue, - ), - ); - _query.stereotypes = query.stereotypes; - _query.content = query.content; - _query.owner = query.owner; - _query.lastUpdatedAt = query.lastUpdatedAt; + dataSpace, + matchingExecutionContext, + isLightGraphEnabled, + (dataSpaceInfo: DataSpaceInfo) => { + if (dataSpaceInfo.defaultExecutionContext) { + const createQuery = async (): Promise => { + // prepare the new query to save + const _query = new Query(); + _query.name = query.name; + _query.id = query.id; + _query.groupId = query.groupId; + _query.artifactId = query.artifactId; + _query.versionId = query.versionId; + _query.mapping = query.mapping; + _query.runtime = query.runtime; + _query.taggedValues = [ + createQueryDataSpaceTaggedValue(dataSpaceInfo.path), + ].concat( + (query.taggedValues ?? []).filter( + (taggedValue) => taggedValue !== dataSpaceTaggedValue, + ), + ); + _query.stereotypes = query.stereotypes; + _query.content = query.content; + _query.owner = query.owner; + _query.lastUpdatedAt = query.lastUpdatedAt; - try { - if (!query.isCurrentUserQuery) { - _query.id = uuid(); - const newQuery = - await editorStore.graphManagerState.graphManager.createQuery( + try { + if (!query.isCurrentUserQuery) { + _query.id = uuid(); + const newQuery = + await editorStore.graphManagerState.graphManager.createQuery( + _query, + editorStore.graphManagerState.graph, + ); + editorStore.applicationStore.notificationService.notifySuccess( + `Successfully created query!`, + ); + LegendQueryEventHelper.notify_QueryCreateSucceeded( + editorStore.applicationStore.eventService, + { queryId: newQuery.id }, + ); + editorStore.applicationStore.navigationService.navigator.goToLocation( + generateExistingQueryEditorRoute(newQuery.id), + ); + } else { + await editorStore.graphManagerState.graphManager.updateQuery( _query, editorStore.graphManagerState.graph, ); - editorStore.applicationStore.notificationService.notifySuccess( - `Successfully created query!`, - ); - LegendQueryEventHelper.notify_QueryCreateSucceeded( - editorStore.applicationStore.eventService, - { queryId: newQuery.id }, - ); - editorStore.applicationStore.navigationService.navigator.goToLocation( - generateExistingQueryEditorRoute(newQuery.id), - ); - } else { - await editorStore.graphManagerState.graphManager.updateQuery( - _query, - editorStore.graphManagerState.graph, + editorStore.applicationStore.notificationService.notifySuccess( + `Successfully updated query!`, + ); + editorStore.applicationStore.navigationService.navigator.reload(); + } + } catch (error) { + assertErrorThrown(error); + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, ); - editorStore.applicationStore.notificationService.notifySuccess( - `Successfully updated query!`, + editorStore.applicationStore.notificationService.notifyError( + error, ); - editorStore.applicationStore.navigationService.navigator.reload(); } - } catch (error) { - assertErrorThrown(error); - editorStore.applicationStore.logService.error( - LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), - error, - ); - editorStore.applicationStore.notificationService.notifyError( - error, - ); - } - }; + }; - editorStore.applicationStore.alertService.setActionAlertInfo({ - message: `To change the data space associated with this query, you need to ${ - query.isCurrentUserQuery - ? 'update the query' - : 'create a new query' - } to proceed`, - type: ActionAlertType.CAUTION, - actions: [ - { - label: query.isCurrentUserQuery - ? 'Update query' - : 'Create new query', - type: ActionAlertActionType.PROCEED_WITH_CAUTION, - handler: () => { - createQuery().catch( - editorStore.applicationStore.alertUnhandledError, - ); + editorStore.applicationStore.alertService.setActionAlertInfo({ + message: `To change the data space associated with this query, you need to ${ + query.isCurrentUserQuery + ? 'update the query' + : 'create a new query' + } to proceed`, + type: ActionAlertType.CAUTION, + actions: [ + { + label: query.isCurrentUserQuery + ? 'Update query' + : 'Create new query', + type: ActionAlertActionType.PROCEED_WITH_CAUTION, + handler: () => { + createQuery().catch( + editorStore.applicationStore.alertUnhandledError, + ); + }, }, - }, - { - label: 'Abort', - type: ActionAlertActionType.PROCEED, - default: true, - }, - ], - }); - } else { - editorStore.applicationStore.notificationService.notifyWarning( - `Can't switch data space: default execution context not specified`, - ); - } - }, - true, - dataSpaceAnalysisResult, - undefined, - undefined, - undefined, - projectInfo, - editorStore.applicationStore.config.options.queryBuilderConfig, - ); - const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResult?.executionContextsIndex.get( - matchingExecutionContext.name, - )?.mappingModelCoverageAnalysisResult; - if (mappingModelCoverageAnalysisResult) { - dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult = - mappingModelCoverageAnalysisResult; + { + label: 'Abort', + type: ActionAlertActionType.PROCEED, + default: true, + }, + ], + }); + } else { + editorStore.applicationStore.notificationService.notifyWarning( + `Can't switch data space: default execution context not specified`, + ); + } + }, + true, + dataSpaceAnalysisResultMetaModel, + undefined, + undefined, + undefined, + projectInfo, + ); + const mappingModelCoverageAnalysisResult = + dataSpaceAnalysisResultMetaModel?.executionContextsIndex.get( + matchingExecutionContext.name, + )?.mappingModelCoverageAnalysisResult; + if (mappingModelCoverageAnalysisResult) { + dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult = + mappingModelCoverageAnalysisResult; + } + return dataSpaceQueryBuilderState; } - return dataSpaceQueryBuilderState; } return undefined; }, ]; } + + override getExtraExistingQueryGraphBuilderCheckers(): ExistingQueryGraphBuilderChecker[] { + return [ + (editorStore: ExistingQueryEditorStore): boolean | undefined => { + const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( + (taggedValue) => + taggedValue.profile === QUERY_PROFILE_PATH && + taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && + isValidFullPath(taggedValue.value), + ); + if (dataSpaceTaggedValue) { + return false; + } + return true; + }, + ]; + } } diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx index 4a1f702b2d..04713bf831 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx @@ -44,11 +44,16 @@ import { getMappingCompatibleRuntimes, PackageableElementExplicitReference, RuntimePointer, + type PackageableRuntime, } from '@finos/legend-graph'; import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { generateGAVCoordinates } from '@finos/legend-storage'; import { useEffect, useMemo, useState } from 'react'; -import { debounce, guaranteeType } from '@finos/legend-shared'; +import { + debounce, + guaranteeType, + guaranteeNonNullable, +} from '@finos/legend-shared'; import { flowResult } from 'mobx'; import { DataSpace, @@ -57,6 +62,7 @@ import { import { DataSpaceIcon } from '../DSL_DataSpace_Icon.js'; import { DataSpaceAdvancedSearchModal } from './DataSpaceAdvancedSearchModal.js'; import type { EditorStore } from '@finos/legend-application-studio'; +import { useQueryEditorStore } from '@finos/legend-application-query'; export type DataSpaceOption = { label: string; @@ -105,6 +111,25 @@ export const formatDataSpaceOptionLabel = ( ); +const resolveExecutionContextRuntimes = ( + queryBuilderState: DataSpaceQueryBuilderState, +): PackageableRuntime[] => { + if (queryBuilderState.dataSpaceAnalysisResult) { + const executionContext = Array.from( + queryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).find( + (e) => + e.mapping.path === + queryBuilderState.executionContext.mapping.value.path, + ); + return guaranteeNonNullable(executionContext).compatibleRuntimes; + } + return getMappingCompatibleRuntimes( + queryBuilderState.executionContext.mapping.value, + queryBuilderState.graphManagerState.usableRuntimes, + ); +}; + type ExecutionContextOption = { label: string; value: DataSpaceExecutionContext; @@ -129,6 +154,7 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( (props: { queryBuilderState: DataSpaceQueryBuilderState }) => { const { queryBuilderState } = props; const applicationStore = useApplicationStore(); + const editorStore = useQueryEditorStore(); const [dataSpaceSearchText, setDataSpaceSearchText] = useState(''); // data space @@ -187,15 +213,15 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( return; } queryBuilderState.setExecutionContext(option.value); - queryBuilderState.propagateExecutionContextChange(option.value); + queryBuilderState.propagateExecutionContextChange( + option.value, + editorStore, + ); queryBuilderState.onExecutionContextChange?.(option.value); }; // runtime - const runtimeOptions = getMappingCompatibleRuntimes( - queryBuilderState.executionContext.mapping.value, - queryBuilderState.graphManagerState.usableRuntimes, - ) + const runtimeOptions = resolveExecutionContextRuntimes(queryBuilderState) .map( (rt) => new RuntimePointer(PackageableElementExplicitReference.create(rt)), @@ -225,11 +251,7 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( }); // class - const classes = resolveUsableDataSpaceClasses( - queryBuilderState.dataSpace, - queryBuilderState.executionContext.mapping.value, - queryBuilderState.graphManagerState, - ); + const classes = resolveUsableDataSpaceClasses(queryBuilderState); useEffect(() => { flowResult(queryBuilderState.loadDataSpaces('')).catch( @@ -407,6 +429,7 @@ export const queryDataSpace = async ( editorStore.depotServerClient, dataSpace, dataSpace.defaultExecutionContext, + false, (dataSpaceInfo: DataSpaceInfo) => { queryBuilderState.dataSpace = guaranteeType( queryBuilderState.graphManagerState.graph.getElement( diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts index 48a9d17992..a833323433 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts @@ -17,14 +17,19 @@ import { type AbstractPureGraphManager, AbstractPureGraphManagerExtension, + type PureProtocolProcessorPlugin, + type V1_PureModelContextData, + type PureModel, + type GraphManagerOperationReport, } from '@finos/legend-graph'; -import type { Entity } from '@finos/legend-storage'; +import type { Entity, ProjectGAVCoordinates } from '@finos/legend-storage'; import { guaranteeNonNullable, type ActionState, type PlainObject, } from '@finos/legend-shared'; import type { DataSpaceAnalysisResult } from '../../action/analytics/DataSpaceAnalysis.js'; +import type { V1_DataSpaceAnalysisResult } from './v1/engine/analytics/V1_DataSpaceAnalysis.js'; export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPureGraphManagerExtension { abstract analyzeDataSpace( @@ -34,10 +39,23 @@ export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPu actionState?: ActionState, ): Promise; - abstract retrieveDataSpaceAnalysisFromCache( - cacheRetriever: () => Promise>, + abstract analyzeDataSpaceCoverage( + dataSpacePath: string, + entitiesRetriever: () => Promise, + cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise; + ): Promise; + + abstract buildDataSpaceAnalytics( + analytics: + | PlainObject + | V1_DataSpaceAnalysisResult, + plugins: PureProtocolProcessorPlugin[], + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + pmcd?: V1_PureModelContextData | undefined, + projectInfo?: ProjectGAVCoordinates, + ): Promise; } export const DSL_DataSpace_getGraphManagerExtension = ( diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts index 03b3a2f59b..8f67084c1a 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts @@ -31,8 +31,11 @@ import { V1_buildDatasetSpecification, type PureProtocolProcessorPlugin, V1_buildModelCoverageAnalysisResult, + type V1_PureModelContextData, + type GraphManagerOperationReport, + LegendSDLC, } from '@finos/legend-graph'; -import type { Entity } from '@finos/legend-storage'; +import type { Entity, ProjectGAVCoordinates } from '@finos/legend-storage'; import { ActionState, assertErrorThrown, @@ -73,7 +76,7 @@ import { } from '../../../action/analytics/DataSpaceAnalysis.js'; import { DSL_DataSpace_PureGraphManagerExtension } from '../DSL_DataSpace_PureGraphManagerExtension.js'; import { - type V1_DataSpaceAnalysisResult, + V1_DataSpaceAnalysisResult, V1_DataSpaceAssociationDocumentationEntry, V1_DataSpaceClassDocumentationEntry, V1_DataSpaceEnumerationDocumentationEntry, @@ -83,6 +86,7 @@ import { V1_DataSpaceMultiExecutionServiceExecutableInfo, } from './engine/analytics/V1_DataSpaceAnalysis.js'; import { getDiagram } from '@finos/legend-extension-dsl-diagram/graph'; +import { resolveVersion } from '@finos/legend-server-depot'; const ANALYZE_DATA_SPACE_TRACE = 'analyze data space'; const TEMPORARY__TDS_SAMPLE_VALUES__DELIMETER = '-- e.g.'; @@ -141,20 +145,54 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu ); } - async retrieveDataSpaceAnalysisFromCache( - cacheRetriever: () => Promise>, + async analyzeDataSpaceCoverage( + dataSpacePath: string, + entitiesRetriever: () => Promise, + cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise { - const cacheResult = await this.fetchDataSpaceAnalysisFromCache( - cacheRetriever, - actionState, - ); - return cacheResult - ? this.buildDataSpaceAnalytics( - cacheResult, - this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), - ) + ): Promise { + const cacheResult = cacheRetriever + ? await this.fetchDataSpaceAnalysisFromCache(cacheRetriever, actionState) : undefined; + const engineClient = this.graphManager.engine.getEngineServerClient(); + let analysisResult: PlainObject; + if ( + cacheResult && + V1_deserializeDataSpaceAnalysisResult( + cacheResult, + this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + ).executionContexts.every( + (e) => e.mappingModelCoverageAnalysisResult.model !== undefined, + ) + ) { + analysisResult = cacheResult; + } else { + actionState?.setMessage('Fetching project entities and dependencies...'); + const entities = await entitiesRetriever(); + actionState?.setMessage('Analyzing data space...'); + analysisResult = await engineClient.postWithTracing< + PlainObject + >( + engineClient.getTraceData(ANALYZE_DATA_SPACE_TRACE), + `${engineClient._pure()}/analytics/dataSpace/coverage`, + { + clientVersion: V1_PureGraphManager.DEV_PROTOCOL_VERSION, + dataSpace: dataSpacePath, + model: { + _type: V1_PureModelContextType.DATA, + elements: entities.map((entity) => entity.content), + }, + }, + {}, + undefined, + undefined, + { enableCompression: true }, + ); + } + return V1_deserializeDataSpaceAnalysisResult( + analysisResult, + this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + ); } private async fetchDataSpaceAnalysisFromCache( @@ -177,11 +215,25 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu return cacheResult; } - private async buildDataSpaceAnalytics( - json: PlainObject, + async buildDataSpaceAnalytics( + analytics: + | PlainObject + | V1_DataSpaceAnalysisResult, plugins: PureProtocolProcessorPlugin[], + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + pmcd?: V1_PureModelContextData | undefined, + projectInfo?: ProjectGAVCoordinates, ): Promise { - const analysisResult = V1_deserializeDataSpaceAnalysisResult(json, plugins); + let analysisResult: V1_DataSpaceAnalysisResult; + if (analytics instanceof V1_DataSpaceAnalysisResult) { + analysisResult = analytics; + } else { + analysisResult = V1_deserializeDataSpaceAnalysisResult( + analytics, + plugins, + ); + } const result = new DataSpaceAnalysisResult(); result.name = analysisResult.name; result.package = analysisResult.package; @@ -228,25 +280,32 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu // NOTE: we will relax the check and not throw here for unknown support info type } - // create an empty graph - const extensionElementClasses = this.graphManager.pluginManager - .getPureGraphPlugins() - .flatMap((plugin) => plugin.getExtraPureGraphExtensionClasses?.() ?? []); - const systemModel = new SystemModel(extensionElementClasses); - const coreModel = new CoreModel(extensionElementClasses); - await this.graphManager.buildSystem( - coreModel, - systemModel, - ActionState.create(), - {}, - ); - systemModel.initializeAutoImports(); - const graph = new PureModel( - coreModel, - systemModel, - this.graphManager.pluginManager.getPureGraphPlugins(), - ); - + let graphEntities; + let graph: PureModel; + if (pureGraph) { + graph = pureGraph; + } else { + // create an empty graph + const extensionElementClasses = this.graphManager.pluginManager + .getPureGraphPlugins() + .flatMap( + (plugin) => plugin.getExtraPureGraphExtensionClasses?.() ?? [], + ); + const systemModel = new SystemModel(extensionElementClasses); + const coreModel = new CoreModel(extensionElementClasses); + await this.graphManager.buildSystem( + coreModel, + systemModel, + ActionState.create(), + {}, + ); + systemModel.initializeAutoImports(); + graph = new PureModel( + coreModel, + systemModel, + this.graphManager.pluginManager.getPureGraphPlugins(), + ); + } // Create dummy mappings and runtimes // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should // simplify the model here or potentially refactor the backend analytics endpoint to return these as model @@ -275,56 +334,79 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu runtime.runtimeValue = new V1_EngineRuntime(); return runtime; }); - - // prepare the model context data - const graphEntities = analysisResult.model.elements - // NOTE: this is a temporary hack to fix a problem with data space analytics - // where the classes for properties are not properly surveyed - // We need to wait for the actual fix in backend to be merged and released - // See https://github.com/finos/legend-engine/pull/836 - .concat( - uniq( - analysisResult.model.elements.flatMap((element) => { - if (element instanceof V1_Class) { - return element.derivedProperties - .map((prop) => prop.returnType) - .concat(element.properties.map((prop) => prop.type)); - } - return []; - }), - ) - // make sure to not include types already returned by the analysis - .filter( - (path) => - !analysisResult.model.elements - .map((el) => el.path) - .includes(path), + if (pmcd && projectInfo) { + graphEntities = pmcd.elements + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter((el) => !graph.getNullableElement(el.path, false)) + .map((el) => this.graphManager.elementProtocolToEntity(el)); + await this.graphManager.buildGraph( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + projectInfo.groupId, + projectInfo.artifactId, + resolveVersion(projectInfo.versionId), + ), + }, + graphReport, + true, + ); + } else { + // prepare the model context data + graphEntities = analysisResult.model.elements + // NOTE: this is a temporary hack to fix a problem with data space analytics + // where the classes for properties are not properly surveyed + // We need to wait for the actual fix in backend to be merged and released + // See https://github.com/finos/legend-engine/pull/836 + .concat( + uniq( + analysisResult.model.elements.flatMap((element) => { + if (element instanceof V1_Class) { + return element.derivedProperties + .map((prop) => prop.returnType) + .concat(element.properties.map((prop) => prop.type)); + } + return []; + }), ) - .map((path) => { - const [pkgPath, name] = resolvePackagePathAndElementName(path); - if (!pkgPath) { - // exclude package-less elements (i.e. primitive types) - return undefined; - } - const _class = new V1_Class(); - _class.name = name; - _class.package = pkgPath; - return _class; - }) - .filter(isNonNullable), - ) - .concat(mappingModels) - .concat(runtimeModels) - // NOTE: if an element could be found in the graph already it means it comes from system - // so we could rid of it - .filter((el) => !graph.getNullableElement(el.path, false)) - .map((el) => this.graphManager.elementProtocolToEntity(el)); + // make sure to not include types already returned by the analysis + .filter( + (path) => + !analysisResult.model.elements + .map((el) => el.path) + .includes(path), + ) + .map((path) => { + const [pkgPath, name] = resolvePackagePathAndElementName(path); + if (!pkgPath) { + // exclude package-less elements (i.e. primitive types) + return undefined; + } + const _class = new V1_Class(); + _class.name = name; + _class.package = pkgPath; + return _class; + }) + .filter(isNonNullable), + ) + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter((el) => !graph.getNullableElement(el.path, false)) + .map((el) => this.graphManager.elementProtocolToEntity(el)); - await this.graphManager.buildGraph( - graph, - graphEntities, - ActionState.create(), - ); + await this.graphManager.buildGraph( + graph, + graphEntities, + ActionState.create(), + ); + } // execution context result.executionContextsIndex = new Map< @@ -344,6 +426,7 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu contextAnalysisResult.mappingModelCoverageAnalysisResult = V1_buildModelCoverageAnalysisResult( context.mappingModelCoverageAnalysisResult, + this.graphManager, contextAnalysisResult.mapping, ); contextAnalysisResult.compatibleRuntimes = context.compatibleRuntimes.map( diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts index 6a1a189e1a..7d2c366c20 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts @@ -29,12 +29,20 @@ import { RuntimePointer, type Runtime, Class, - type Mapping, getDescendantsOfPackage, Package, + createGraphBuilderReport, + GRAPH_MANAGER_EVENT, + LegendSDLC, + V1_EngineRuntime, + V1_Mapping, + V1_PackageableRuntime, + V1_PureGraphManager, + resolvePackagePathAndElementName, } from '@finos/legend-graph'; import { DepotScope, + resolveVersion, SNAPSHOT_VERSION_ALIAS, type DepotServerClient, type StoredEntity, @@ -46,6 +54,10 @@ import { getNullableFirstEntry, filterByType, uniq, + StopWatch, + LogEvent, + guaranteeNonNullable, + guaranteeType, } from '@finos/legend-shared'; import { action, flow, makeObservable, observable } from 'mobx'; import { renderDataSpaceQueryBuilderSetupPanelContent } from '../../components/query/DataSpaceQueryBuilder.js'; @@ -57,12 +69,14 @@ import { DATA_SPACE_ELEMENT_CLASSIFIER_PATH } from '../../graph-manager/protocol import { type DataSpaceInfo, extractDataSpaceInfo } from './DataSpaceInfo.js'; import { DataSpaceAdvancedSearchState } from './DataSpaceAdvancedSearchState.js'; import type { DataSpaceAnalysisResult } from '../../graph-manager/action/analytics/DataSpaceAnalysis.js'; +import type { QueryEditorStore } from '@finos/legend-application-query'; export const resolveUsableDataSpaceClasses = ( - dataSpace: DataSpace, - mapping: Mapping, - graphManagerState: GraphManagerState, + queryBuilderState: DataSpaceQueryBuilderState, ): Class[] => { + const dataSpace = queryBuilderState.dataSpace; + const mapping = queryBuilderState.executionContext.mapping.value; + const graphManagerState = queryBuilderState.graphManagerState; if (dataSpace.elements?.length) { const dataSpaceElements = dataSpace.elements.map((ep) => ep.element.value); return uniq([ @@ -73,6 +87,19 @@ export const resolveUsableDataSpaceClasses = ( .flat() .filter(filterByType(Class)), ]); + } else if ( + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult + ) { + const compatibleClassPaths = + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult.mappedEntities.map( + (e) => e.classPath, + ); + const uniqueCompatibleClasses = compatibleClassPaths.filter( + (val, index) => compatibleClassPaths.indexOf(val) === index, + ); + return graphManagerState.graph.classes.filter((c) => + uniqueCompatibleClasses.includes(c.path), + ); } return getMappingCompatibleClasses(mapping, graphManagerState.usableClasses); }; @@ -138,6 +165,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { dataSpaces: DataSpaceInfo[] = []; showRuntimeSelector = false; advancedSearchState?: DataSpaceAdvancedSearchState | undefined; + isLightGraphEnabled!: boolean; constructor( applicationStore: GenericLegendApplicationStore, @@ -145,6 +173,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { depotServerClient: DepotServerClient, dataSpace: DataSpace, executionContext: DataSpaceExecutionContext, + isLightGraphEnabled: boolean, onDataSpaceChange: (val: DataSpaceInfo) => void, isAdvancedDataSpaceSearchEnabled: boolean, dataSpaceAnalysisResult?: DataSpaceAnalysisResult | undefined, @@ -161,11 +190,13 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { makeObservable(this, { dataSpaces: observable, executionContext: observable, + isLightGraphEnabled: observable, showRuntimeSelector: observable, advancedSearchState: observable, showAdvancedSearchPanel: action, hideAdvancedSearchPanel: action, setExecutionContext: action, + setIsLightGraphEnabled: action, setShowRuntimeSelector: action, loadDataSpaces: flow, }); @@ -174,6 +205,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { this.dataSpace = dataSpace; this.executionContext = executionContext; this.projectInfo = projectInfo; + this.isLightGraphEnabled = isLightGraphEnabled; this.onDataSpaceChange = onDataSpaceChange; this.onExecutionContextChange = onExecutionContextChange; this.onRuntimeChange = onRuntimeChange; @@ -188,6 +220,10 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { : 'query-builder__setup__data-space'; } + setIsLightGraphEnabled(val: boolean): void { + this.isLightGraphEnabled = val; + } + showAdvancedSearchPanel(): void { if (this.projectInfo && this.isAdvancedDataSpaceSearchEnabled) { this.advancedSearchState = new DataSpaceAdvancedSearchState( @@ -275,26 +311,125 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { * - If no class is chosen, try to choose a compatible class * - If the chosen class is compatible with the new selected execution context mapping, do nothing, otherwise, try to choose a compatible class */ - propagateExecutionContextChange( + async propagateExecutionContextChange( executionContext: DataSpaceExecutionContext, - ): void { + editorStore?: QueryEditorStore, + isGraphBuildingNotRequired?: boolean, + ): Promise { const mapping = executionContext.mapping.value; - this.changeMapping(mapping); + let compatibleClasses; const mappingModelCoverageAnalysisResult = this.dataSpaceAnalysisResult?.executionContextsIndex.get( executionContext.name, )?.mappingModelCoverageAnalysisResult; - if (mappingModelCoverageAnalysisResult) { + if (this.dataSpaceAnalysisResult && mappingModelCoverageAnalysisResult) { + if (!isGraphBuildingNotRequired && editorStore) { + const stopWatch = new StopWatch(); + const graph = this.graphManagerState.createNewGraph(); + + const graph_buildReport = createGraphBuilderReport(); + // Create dummy mappings and runtimes + // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should + // simplify the model here or potentially refactor the backend analytics endpoint to return these as model + const mappingModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).map((context) => context.mapping), + ).map((m) => { + const _mapping = new V1_Mapping(); + const [packagePath, name] = resolvePackagePathAndElementName(m.path); + _mapping.package = packagePath; + _mapping.name = name; + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(_mapping); + }); + const runtimeModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ) + .map((context) => context.defaultRuntime) + .concat( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).flatMap((val) => val.compatibleRuntimes), + ), + ).map((r) => { + const runtime = new V1_PackageableRuntime(); + const [packagePath, name] = resolvePackagePathAndElementName(r.path); + runtime.package = packagePath; + runtime.name = name; + runtime.runtimeValue = new V1_EngineRuntime(); + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(runtime); + }); + const graphEntities = guaranteeNonNullable( + mappingModelCoverageAnalysisResult.entities, + ) + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter( + (el) => + !graph.getNullableElement(el.path, false) && + !el.path.startsWith('meta::'), + ); + await this.graphManagerState.graphManager.buildGraph( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + guaranteeNonNullable(this.projectInfo).groupId, + guaranteeNonNullable(this.projectInfo).artifactId, + resolveVersion(guaranteeNonNullable(this.projectInfo).versionId), + ), + }, + graph_buildReport, + true, + ); + this.graphManagerState.graph = graph; + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord(stopWatch), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager.numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); + + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } + const compatibleClassPaths = + mappingModelCoverageAnalysisResult.mappedEntities.map( + (e) => e.classPath, + ); + const uniqueCompatibleClasses = compatibleClassPaths.filter( + (val, index) => compatibleClassPaths.indexOf(val) === index, + ); + compatibleClasses = this.graphManagerState.graph.classes.filter((c) => + uniqueCompatibleClasses.includes(c.path), + ); this.explorerState.mappingModelCoverageAnalysisResult = mappingModelCoverageAnalysisResult; + } else { + compatibleClasses = resolveUsableDataSpaceClasses(this); } + this.changeMapping(mapping); + this.changeRuntime(new RuntimePointer(executionContext.defaultRuntime)); - const compatibleClasses = resolveUsableDataSpaceClasses( - this.dataSpace, - mapping, - this.graphManagerState, - ); // if there is no chosen class or the chosen one is not compatible // with the mapping then pick a compatible class if possible if (!this.class || !compatibleClasses.includes(this.class)) { @@ -303,5 +438,6 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { this.changeClass(possibleNewClass); } } + this.explorerState.refreshTreeData(); } } diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts index 579d752d36..4291e2e224 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts @@ -23,6 +23,9 @@ import { type Runtime, type Class, type RawLambda, + type V1_PureModelContextData, + createGraphBuilderReport, + GRAPH_MANAGER_EVENT, } from '@finos/legend-graph'; import { QueryEditorStore, @@ -30,14 +33,18 @@ import { type LegendQueryApplicationStore, createViewProjectHandler, createViewSDLCProjectHandler, + LEGEND_QUERY_APP_EVENT, } from '@finos/legend-application-query'; import { type DepotServerClient, StoreProjectData, + retrieveProjectEntitiesWithDependencies, } from '@finos/legend-server-depot'; import { guaranteeNonNullable, guaranteeType, + LogEvent, + StopWatch, uuid, } from '@finos/legend-shared'; import { @@ -117,7 +124,97 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { }; } + override requiresGraphBuilding(): boolean { + return false; + } + async initializeQueryBuilderState(): Promise { + let dataSpaceAnalysisResult; + let dataSpaceAnalysisResultMetaModel; + let isLightGraphEnabled = true; + try { + const stopWatch = new StopWatch(); + const project = StoreProjectData.serialization.fromJson( + await this.depotServerClient.getProject(this.groupId, this.artifactId), + ); + this.initState.setMessage('Fetching dataspace analysis result'); + dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( + this.graphManagerState.graphManager, + ).analyzeDataSpaceCoverage( + this.dataSpacePath, + () => + retrieveProjectEntitiesWithDependencies( + project, + this.versionId, + this.depotServerClient, + ), + () => + retrieveAnalyticsResultCache( + project, + this.versionId, + this.dataSpacePath, + this.depotServerClient, + ), + ); + const mappingPath = dataSpaceAnalysisResult?.executionContexts.find( + (e) => e.name === this.executionContext, + )?.mapping; + if (dataSpaceAnalysisResult && mappingPath) { + // initialize system + stopWatch.record(); + await this.graphManagerState.initializeSystem(); + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS); + + // build graph + let pmcd: V1_PureModelContextData | undefined = undefined; + if (dataSpaceAnalysisResult) { + const mappingModelCoverageAnalysisResult = + dataSpaceAnalysisResult?.executionContexts.find( + (value) => value.mapping === mappingPath, + )?.mappingModelCoverageAnalysisResult; + pmcd = mappingModelCoverageAnalysisResult?.model; + } + const graph_buildReport = createGraphBuilderReport(); + dataSpaceAnalysisResultMetaModel = + await DSL_DataSpace_getGraphManagerExtension( + this.graphManagerState.graphManager, + ).buildDataSpaceAnalytics( + dataSpaceAnalysisResult, + this.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + graph_buildReport, + this.graphManagerState.graph, + pmcd, + this.getProjectInfo(), + ); + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord(stopWatch), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager.numberOfDependencies, + graph: graph_buildReport, + }; + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } else { + isLightGraphEnabled = false; + this.graphManagerState.graph = this.graphManagerState.createNewGraph(); + await this.buildGraph(); + } + } catch (error) { + this.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + this.graphManagerState.graph = this.graphManagerState.createNewGraph(); + isLightGraphEnabled = false; + await this.buildGraph(); + } const dataSpace = getDataSpace( this.dataSpacePath, this.graphManagerState.graph, @@ -128,24 +225,6 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { ), `Can't find execution context '${this.executionContext}'`, ); - let dataSpaceAnalysisResult; - try { - const project = StoreProjectData.serialization.fromJson( - await this.depotServerClient.getProject(this.groupId, this.artifactId), - ); - dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( - this.graphManagerState.graphManager, - ).retrieveDataSpaceAnalysisFromCache(() => - retrieveAnalyticsResultCache( - project, - this.versionId, - dataSpace.path, - this.depotServerClient, - ), - ); - } catch { - // do nothing - } const projectInfo = new DataSpaceProjectInfo( this.groupId, this.artifactId, @@ -162,6 +241,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { this.depotServerClient, dataSpace, executionContext, + isLightGraphEnabled, (dataSpaceInfo: DataSpaceInfo) => { if (dataSpaceInfo.defaultExecutionContext) { this.applicationStore.navigationService.navigator.goToLocation( @@ -182,7 +262,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { } }, true, - dataSpaceAnalysisResult, + dataSpaceAnalysisResultMetaModel, (ec: DataSpaceExecutionContext) => { // runtime should already be set const runtimePointer = guaranteeType( @@ -245,7 +325,11 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { projectInfo, ); queryBuilderState.setExecutionContext(executionContext); - queryBuilderState.propagateExecutionContextChange(executionContext); + queryBuilderState.propagateExecutionContextChange( + executionContext, + this, + true, + ); // set runtime if already chosen if (this.runtimePath) { diff --git a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts index 1ac8a1956f..4a3ddebede 100644 --- a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts @@ -263,6 +263,7 @@ export abstract class AbstractPureGraphManager { buildState: ActionState, options?: GraphBuilderOptions, report?: GraphManagerOperationReport, + buildRequiredGraph?: boolean | undefined, ): Promise; /** diff --git a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts index 38808ac627..2857f19bf2 100644 --- a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type { Entity } from '@finos/legend-storage'; import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js'; export type RawMappingModelCoverageAnalysisResult = object; @@ -22,10 +23,12 @@ export class MappedEntity { readonly __PROPERTIES_INDEX = new Map(); path: string; + classPath: string; properties: MappedProperty[]; - constructor(path: string, properties: MappedProperty[]) { + constructor(path: string, classPath: string, properties: MappedProperty[]) { this.path = path; + this.classPath = classPath; this.properties = properties; properties.forEach((property) => this.__PROPERTIES_INDEX.set(property.name, property), @@ -73,12 +76,18 @@ export class MappingModelCoverageAnalysisResult { readonly mapping: Mapping; mappedEntities: MappedEntity[]; + entities?: Entity[] | undefined; - constructor(mappedEntities: MappedEntity[], mapping: Mapping) { + constructor( + mappedEntities: MappedEntity[], + mapping: Mapping, + entities?: Entity[] | undefined, + ) { this.mappedEntities = mappedEntities; this.mapping = mapping; mappedEntities.forEach((entity) => this.__ENTITIES_INDEX.set(entity.path, entity), ); + this.entities = entities; } } diff --git a/packages/legend-graph/src/graph-manager/action/query/Query.ts b/packages/legend-graph/src/graph-manager/action/query/Query.ts index a168113618..29c88c2211 100644 --- a/packages/legend-graph/src/graph-manager/action/query/Query.ts +++ b/packages/legend-graph/src/graph-manager/action/query/Query.ts @@ -67,6 +67,8 @@ export class LightQuery { artifactId!: string; owner?: string | undefined; lastUpdatedAt?: number | undefined; + mapping?: string | PackageableElementReference | undefined; + taggedValues?: QueryTaggedValue[] | undefined; isCurrentUserQuery = false; } @@ -80,6 +82,8 @@ export const toLightQuery = (query: Query): LightQuery => { lightQuery.versionId = query.versionId; lightQuery.owner = query.owner; lightQuery.isCurrentUserQuery = query.isCurrentUserQuery; + lightQuery.mapping = query.mapping.value.path; + lightQuery.taggedValues = query.taggedValues; return lightQuery; }; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts index 2dafd9d28d..e8fa4aa7d2 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts @@ -803,6 +803,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { buildState: ActionState, options?: GraphBuilderOptions, _report?: GraphManagerOperationReport, + buildRequiredGraph?: boolean | undefined, ): Promise { const stopWatch = new StopWatch(); const report = _report ?? createGraphBuilderReport(); @@ -836,14 +837,25 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ]; // build - await this.buildGraphFromInputs( - graph, - buildInputs, - report, - stopWatch, - buildState, - options, - ); + if (!buildRequiredGraph) { + await this.buildGraphFromInputs( + graph, + buildInputs, + report, + stopWatch, + buildState, + options, + ); + } else { + await this.buildRequiredGraphFromInputs( + graph, + buildInputs, + report, + stopWatch, + buildState, + options, + ); + } /** * For now, we delete the section index. We are able to read both resolved and unresolved element paths @@ -1130,6 +1142,32 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ); } + private async buildRequiredGraphFromInputs( + graph: PureModel, + inputs: V1_PureGraphBuilderInput[], + report: GraphManagerOperationReport, + stopWatch: StopWatch, + graphBuilderState: ActionState, + options?: GraphBuilderOptions, + ): Promise { + // index + graphBuilderState.setMessage( + `Indexing ${report.elementCount.total} elements...`, + ); + await this.initializeAndIndexElements(graph, inputs, options); + stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS); + // build types + graphBuilderState.setMessage(`Building domain models...`); + await this.buildTypes(graph, inputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS, + ); + + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS, + ); + } + private async buildLightGraphFromInputs( graph: PureModel, inputs: V1_PureGraphBuilderInput[], @@ -2962,6 +3000,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { : this.getFullGraphModelData(graph); return V1_buildModelCoverageAnalysisResult( await this.engine.analyzeMappingModelCoverage(input), + this, mapping, ); } @@ -2975,6 +3014,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { V1_MappingModelCoverageAnalysisResult, input as PlainObject, ), + this, mapping, ); } diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts index b341b2f8f4..b120b63841 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts @@ -97,10 +97,29 @@ export const V1_buildLightQuery = ( protocol.artifactId, `Query 'artifactId' field is missing`, ); + metamodel.mapping = protocol.mapping; metamodel.owner = protocol.owner; metamodel.lastUpdatedAt = protocol.lastUpdatedAt; metamodel.isCurrentUserQuery = currentUserId !== undefined && protocol.owner === currentUserId; + // NOTE: we don't properly process tagged values and stereotypes for query + // because these profiles/tags/stereotypes can come from external systems. + metamodel.taggedValues = protocol.taggedValues?.map((taggedValueProtocol) => { + const taggedValue = new QueryTaggedValue(); + taggedValue.profile = guaranteeNonEmptyString( + taggedValueProtocol.tag.profile, + `Tagged value 'tag.profile' field is missing or empty`, + ); + taggedValue.tag = guaranteeNonEmptyString( + taggedValueProtocol.tag.value, + `Tagged value 'tag.value' field is missing or empty`, + ); + taggedValue.value = guaranteeNonEmptyString( + taggedValueProtocol.value, + `Tagged value 'value' field is missing or empty`, + ); + return taggedValue; + }); return metamodel; }; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts index 52228332c6..cf3d1d3645 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts @@ -37,7 +37,13 @@ import { MappingModelCoverageAnalysisResult, } from '../../../../../../graph-manager/action/analytics/MappingModelCoverageAnalysis.js'; import type { V1_PureModelContext } from '../../model/context/V1_PureModelContext.js'; -import { V1_pureModelContextPropSchema } from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js'; +import { + V1_deserializePureModelContextData, + V1_pureModelContextPropSchema, + V1_serializePureModelContextData, +} from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js'; +import type { V1_PureModelContextData } from '../../model/context/V1_PureModelContextData.js'; +import type { V1_PureGraphManager } from '../../V1_PureGraphManager.js'; enum V1_MappedPropertyType { ENUM = 'enum', @@ -103,11 +109,13 @@ const V1_deserializeMappedProperty = ( class V1_MappedEntity { path!: string; + classPath!: string; properties: V1_MappedProperty[] = []; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappedEntity, { path: primitive(), + classPath: primitive(), properties: list( custom( (prop) => V1_serializeMappedProperty(prop), @@ -134,12 +142,19 @@ export class V1_MappingModelCoverageAnalysisInput { export class V1_MappingModelCoverageAnalysisResult { mappedEntities: V1_MappedEntity[] = []; + model?: V1_PureModelContextData | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappingModelCoverageAnalysisResult, { mappedEntities: list( usingModelSchema(V1_MappedEntity.serialization.schema), ), + model: optional( + custom( + (val) => V1_serializePureModelContextData(val), + (val) => V1_deserializePureModelContextData(val), + ), + ), }), ); } @@ -158,14 +173,21 @@ const buildMappedProperty = (protocol: V1_MappedProperty): MappedProperty => const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => new MappedEntity( protocol.path, + protocol.classPath, protocol.properties.map((p) => buildMappedProperty(p)), ); export const V1_buildModelCoverageAnalysisResult = ( protocol: V1_MappingModelCoverageAnalysisResult, + graphManager: V1_PureGraphManager, mapping: Mapping, -): MappingModelCoverageAnalysisResult => - new MappingModelCoverageAnalysisResult( +): MappingModelCoverageAnalysisResult => { + const entities = protocol.model?.elements.map((el) => + graphManager.elementProtocolToEntity(el), + ); + return new MappingModelCoverageAnalysisResult( protocol.mappedEntities.map((p) => buildMappedEntity(p)), mapping, + entities, ); +}; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts index 3d2e9bbd28..94e21fc17f 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts @@ -87,6 +87,8 @@ export class V1_LightQuery { artifactId!: string; versionId!: string; lastUpdatedAt?: number | undefined; + mapping?: string | undefined; + taggedValues?: V1_TaggedValue[] | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_Query, { @@ -94,8 +96,10 @@ export class V1_LightQuery { id: primitive(), groupId: primitive(), lastUpdatedAt: optional(primitive()), + mapping: optional(primitive()), name: primitive(), owner: optional(primitive()), + taggedValues: optional(list(usingModelSchema(V1_taggedValueModelSchema))), versionId: primitive(), }), { From 9be683bfa6c9e9add250016b3637a1da1875e229 Mon Sep 17 00:00:00 2001 From: Gayathri Date: Mon, 25 Sep 2023 18:35:31 +0530 Subject: [PATCH 2/2] minor changes to the query load process --- .changeset/clean-toes-learn.md | 1 + .../QueryEditorComponentTestUtils.tsx | 7 +- .../stores/LegendQueryApplicationPlugin.ts | 13 +- .../src/stores/QueryEditorStore.ts | 67 +++--- .../editor/editor-group/FunctionEditor.tsx | 3 +- .../DEPRECATED__MappingTestEditor.tsx | 2 +- .../MappingExecutionBuilder.tsx | 2 +- .../mapping-editor/MappingTestableEditor.tsx | 2 +- .../ServiceExecutionQueryEditor.tsx | 4 +- .../uml-editor/ClassQueryBuilder.tsx | 2 +- .../editor/EmbeddedQueryBuilderState.ts | 5 +- ...DataSpace_LegendQueryApplicationPlugin.tsx | 99 ++++---- .../query/DataSpaceQueryBuilder.tsx | 20 +- ...DSL_DataSpace_PureGraphManagerExtension.ts | 15 +- ...L_DataSpace_PureProtocolProcessorPlugin.ts | 46 ++-- ...DSL_DataSpace_PureGraphManagerExtension.ts | 53 +++-- .../query/DataSpaceQueryBuilderState.ts | 219 ++++++++++-------- .../query/DataSpaceQueryCreatorStore.ts | 65 +++--- .../graph-manager/AbstractPureGraphManager.ts | 12 +- .../analytics/MappingModelCoverageAnalysis.ts | 24 +- .../src/graph-manager/action/query/Query.ts | 13 +- .../protocol/pure/v1/V1_PureGraphManager.ts | 141 +++++++---- .../pure/v1/engine/V1_EngineHelper.ts | 42 +--- .../V1_MappingModelCoverageAnalysis.ts | 38 ++- 24 files changed, 481 insertions(+), 414 deletions(-) diff --git a/.changeset/clean-toes-learn.md b/.changeset/clean-toes-learn.md index d78bbd6f64..7b87f06420 100644 --- a/.changeset/clean-toes-learn.md +++ b/.changeset/clean-toes-learn.md @@ -1,3 +1,4 @@ --- '@finos/legend-graph': patch +'@finos/legend-application-studio': patch --- diff --git a/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx b/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx index e8f1e769e5..e7a2f9024c 100644 --- a/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx +++ b/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx @@ -22,7 +22,6 @@ import { Query, LightQuery, RawLambda, - PackageableElementExplicitReference, type RawMappingModelCoverageAnalysisResult, } from '@finos/legend-graph'; import { DepotServerClient } from '@finos/legend-server-depot'; @@ -128,10 +127,8 @@ export const TEST__setUpQueryEditor = async ( query.owner = lightQuery.owner; query.isCurrentUserQuery = lightQuery.isCurrentUserQuery; const _mapping = graphManagerState.graph.getMapping(mappingPath); - query.mapping = PackageableElementExplicitReference.create(_mapping); - query.runtime = PackageableElementExplicitReference.create( - graphManagerState.graph.getRuntime(runtimePath), - ); + query.mapping = mappingPath; + query.runtime = runtimePath; query.content = 'some content'; createSpy( diff --git a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts index c54479278c..f8d127a771 100644 --- a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts +++ b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts @@ -16,6 +16,7 @@ import { LegendApplicationPlugin } from '@finos/legend-application'; import type { QueryBuilderState } from '@finos/legend-query-builder'; +import type { GeneratorFn } from '@finos/legend-shared'; import type React from 'react'; import type { LegendQueryPluginManager } from '../application/LegendQueryPluginManager.js'; import type { @@ -23,6 +24,7 @@ import type { QueryEditorStore, } from './QueryEditorStore.js'; import type { QuerySetupLandingPageStore } from './QuerySetupStore.js'; +import type { Query } from '@finos/legend-graph'; export enum QuerySetupActionTag { PRODUCTIONIZATION = 'Productionization', @@ -51,12 +53,13 @@ export type QuerySetupActionConfiguration = { }; export type ExistingQueryEditorStateBuilder = ( + query: Query, editorStore: ExistingQueryEditorStore, ) => Promise; -export type ExistingQueryGraphBuilderChecker = ( - editorStore: ExistingQueryEditorStore, -) => boolean | undefined; +export type QueryGraphBuilderGetter = ( + editorStore: QueryEditorStore, +) => ((editorStore: QueryEditorStore) => GeneratorFn) | undefined; export type QueryEditorActionConfiguration = { key: string; @@ -101,9 +104,9 @@ export abstract class LegendQueryApplicationPlugin extends LegendApplicationPlug getExtraExistingQueryEditorStateBuilders?(): ExistingQueryEditorStateBuilder[]; /** - * Get the list of existing query graph builder checkers + * Get the list of query graph builders */ - getExtraExistingQueryGraphBuilderCheckers?(): ExistingQueryGraphBuilderChecker[]; + getExtraQueryGraphBuilderGetters?(): QueryGraphBuilderGetter[]; /** * Get the list of query editor action renderer configurations. diff --git a/packages/legend-application-query/src/stores/QueryEditorStore.ts b/packages/legend-application-query/src/stores/QueryEditorStore.ts index c055e4bc70..27edf88c24 100644 --- a/packages/legend-application-query/src/stores/QueryEditorStore.ts +++ b/packages/legend-application-query/src/stores/QueryEditorStore.ts @@ -293,6 +293,7 @@ export abstract class QueryEditorStore { setExistingQueryName: action, initialize: flow, buildGraph: flow, + buildFullGraph: flow, searchExistingQueryName: flow, }); @@ -405,10 +406,9 @@ export abstract class QueryEditorStore { RuntimePointer, 'Query runtime must be of type runtime pointer', ); - query.mapping = PackageableElementExplicitReference.create( - this.queryBuilderState.executionContextState.mapping, - ); - query.runtime = runtimeValue.packageableRuntime; + (query.mapping = + this.queryBuilderState.executionContextState.mapping.path), + (query.runtime = runtimeValue.packageableRuntime.value.path); query.content = await this.graphManagerState.graphManager.lambdaToPureCode(rawLambda); config?.decorator?.(query); @@ -489,9 +489,8 @@ export abstract class QueryEditorStore { ); yield this.setUpEditorState(); - if (this.requiresGraphBuilding()) { - yield flowResult(this.buildGraph()); - } + yield flowResult(this.buildGraph()); + this.queryBuilderState = (yield this.initializeQueryBuilderState( stopWatch, )) as QueryBuilderState; @@ -543,7 +542,7 @@ export abstract class QueryEditorStore { ); } - *buildGraph(): GeneratorFn { + *buildFullGraph(): GeneratorFn { const stopWatch = new StopWatch(); const { groupId, artifactId, versionId } = this.getProjectInfo(); @@ -632,6 +631,20 @@ export abstract class QueryEditorStore { graphBuilderReportData, ); } + + *buildGraph(): GeneratorFn { + const queryGraphBuilderGetters = this.applicationStore.pluginManager + .getApplicationPlugins() + .flatMap((plugin) => plugin.getExtraQueryGraphBuilderGetters?.() ?? []); + for (const getter of queryGraphBuilderGetters) { + const builderFunction = getter(this); + if (builderFunction) { + yield flowResult(builderFunction(this)); + return; + } + } + yield flowResult(this.buildFullGraph()); + } } export class MappingQueryCreatorStore extends QueryEditorStore { @@ -992,34 +1005,12 @@ export class ExistingQueryEditorStore extends QueryEditorStore { } override async setUpEditorState(): Promise { - this.setLightQuery( - await this.graphManagerState.graphManager.getLightQuery(this.queryId), - ); - } - - override requiresGraphBuilding(): boolean { - const existingQueryGraphBuilderCheckers = - this.applicationStore.pluginManager - .getApplicationPlugins() - .flatMap( - (plugin) => - plugin.getExtraExistingQueryGraphBuilderCheckers?.() ?? [], - ); - for (const checker of existingQueryGraphBuilderCheckers) { - const isGraphBuildingRequired = checker(this); - if (isGraphBuildingRequired !== undefined) { - return isGraphBuildingRequired; - } - } - return true; - } - - async setupQuery(): Promise { const query = await this.graphManagerState.graphManager.getQuery( this.queryId, this.graphManagerState.graph, ); this.setQuery(query); + this.setLightQuery(toLightQuery(query)); LegendQueryUserDataHelper.addRecentlyViewedQuery( this.applicationStore.userDataService, query.id, @@ -1029,6 +1020,7 @@ export class ExistingQueryEditorStore extends QueryEditorStore { async initializeQueryBuilderState( stopWatch: StopWatch, ): Promise { + const query = guaranteeNonNullable(this.query); let queryBuilderState: QueryBuilderState | undefined; const existingQueryEditorStateBuilders = this.applicationStore.pluginManager .getApplicationPlugins() @@ -1036,15 +1028,12 @@ export class ExistingQueryEditorStore extends QueryEditorStore { (plugin) => plugin.getExtraExistingQueryEditorStateBuilders?.() ?? [], ); for (const builder of existingQueryEditorStateBuilders) { - queryBuilderState = await builder(this); + queryBuilderState = await builder(query, this); if (queryBuilderState) { break; } } - await this.setupQuery(); - const query = guaranteeNonNullable(this.query); - // if no extension found, fall back to basic `class -> mapping -> runtime` mode queryBuilderState = queryBuilderState ?? @@ -1054,11 +1043,11 @@ export class ExistingQueryEditorStore extends QueryEditorStore { this.applicationStore.config.options.queryBuilderConfig, ); - queryBuilderState.executionContextState.setMapping(query.mapping.value); + const mapping = this.graphManagerState.graph.getMapping(query.mapping); + const runtime = this.graphManagerState.graph.getRuntime(query.runtime); + queryBuilderState.executionContextState.setMapping(mapping); queryBuilderState.executionContextState.setRuntimeValue( - new RuntimePointer( - PackageableElementExplicitReference.create(query.runtime.value), - ), + new RuntimePointer(PackageableElementExplicitReference.create(runtime)), ); // leverage initialization of query builder state to ensure we handle unsupported queries diff --git a/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx index e5e7e47587..e107d39dbe 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx @@ -1105,8 +1105,7 @@ export const FunctionEditor = observer(() => { ); await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => - functionQueryBuilderState, + setupQueryBuilderState: async () => functionQueryBuilderState, actionConfigs: [ { key: 'save-query-btn', diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx index f56f46469f..f4f9328fb6 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx @@ -112,7 +112,7 @@ const MappingTestQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx index 092f2fc345..6355d15cff 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx @@ -205,7 +205,7 @@ const MappingExecutionQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx index bea58ebc0c..b369c79373 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx @@ -589,7 +589,7 @@ const MappingTestSuiteQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx index 60aeb28758..2cc4ccb740 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx @@ -125,7 +125,7 @@ export const ServiceExecutionQueryEditor = observer( executionState.selectedExecutionContextState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new ServiceQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, @@ -459,7 +459,7 @@ export const queryService = async ( : undefined; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new ServiceQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx b/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx index ccd8c83326..6bf11b1058 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx @@ -375,7 +375,7 @@ export const queryClass = async ( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: () => { + setupQueryBuilderState: async () => { const queryBuilderState = new ClassQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts b/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts index 03581d7bd7..7b5f5ee2eb 100644 --- a/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts +++ b/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts @@ -30,7 +30,7 @@ type EmbeddedQueryBuilderActionConfiguration = { }; type EmbeddedQueryBuilderConfiguration = { - setupQueryBuilderState: () => QueryBuilderState; + setupQueryBuilderState: () => Promise; disableCompile?: boolean | undefined; actionConfigs: EmbeddedQueryBuilderActionConfiguration[]; }; @@ -100,7 +100,8 @@ export class EmbeddedQueryBuilderState { } } if (!this.editorStore.graphState.error) { - this.queryBuilderState = config.setupQueryBuilderState(); + this.queryBuilderState = + (yield config.setupQueryBuilderState()) as QueryBuilderState; this.actionConfigs = config.actionConfigs; this.editorStore.applicationStore.layoutService.setBackdropContainerElementID( QUERY_BUILDER_COMPONENT_ELEMENT_ID.BACKDROP_CONTAINER, diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx index 6e142a080f..677b749ae1 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx @@ -17,7 +17,7 @@ import packageJson from '../../../package.json' assert { type: 'json' }; import { type QuerySetupActionConfiguration, - type ExistingQueryEditorStore, + ExistingQueryEditorStore, LegendQueryApplicationPlugin, generateExistingQueryEditorRoute, LEGEND_QUERY_APP_EVENT, @@ -25,7 +25,8 @@ import { createViewProjectHandler, createViewSDLCProjectHandler, type ExistingQueryEditorStateBuilder, - type ExistingQueryGraphBuilderChecker, + type QueryGraphBuilderGetter, + type QueryEditorStore, } from '@finos/legend-application-query'; import { SquareIcon } from '@finos/legend-art'; import { @@ -43,7 +44,6 @@ import { Query, isValidFullPath, GRAPH_MANAGER_EVENT, - type V1_PureModelContextData, createGraphBuilderReport, } from '@finos/legend-graph'; import { @@ -58,11 +58,11 @@ import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { getOwnDataSpace } from '../../graph-manager/DSL_DataSpace_GraphManagerHelper.js'; import { assertErrorThrown, - guaranteeNonNullable, isString, LogEvent, StopWatch, uuid, + type GeneratorFn, } from '@finos/legend-shared'; import type { QueryBuilderState } from '@finos/legend-query-builder'; import { DataSpaceQuerySetup } from './DataSpaceQuerySetup.js'; @@ -72,6 +72,11 @@ import { StoreProjectData, } from '@finos/legend-server-depot'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; +import { flowResult } from 'mobx'; + +function* buildGraph(): GeneratorFn { + // do nothing +} export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin { constructor() { @@ -118,9 +123,10 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli override getExtraExistingQueryEditorStateBuilders(): ExistingQueryEditorStateBuilder[] { return [ async ( + query: Query, editorStore: ExistingQueryEditorStore, ): Promise => { - const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( + const dataSpaceTaggedValue = query.taggedValues?.find( (taggedValue) => taggedValue.profile === QUERY_PROFILE_PATH && taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && @@ -129,11 +135,10 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli let isLightGraphEnabled = true; if (dataSpaceTaggedValue) { const dataSpacePath = dataSpaceTaggedValue.value; - const mappingPath = editorStore.lightQuery?.mapping; - let dataSpaceAnalysisResult; - let dataSpaceAnalysisResultMetaModel; + const mappingPath = query.mapping; const { groupId, artifactId, versionId } = editorStore.getProjectInfo(); + let dataSpaceAnalysisResult; if (dataSpacePath && isString(mappingPath)) { try { editorStore.initState.setMessage( @@ -145,6 +150,15 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli artifactId, ), ); + const graph_buildReport = createGraphBuilderReport(); + const stopWatch = new StopWatch(); + // initialize system + stopWatch.record(); + await editorStore.graphManagerState.initializeSystem(); + stopWatch.record( + GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS, + ); + const dependency_buildReport = createGraphBuilderReport(); dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( editorStore.graphManagerState.graphManager, @@ -163,37 +177,15 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli dataSpacePath, editorStore.depotServerClient, ), - ); - const stopWatch = new StopWatch(); - // initialize system - stopWatch.record(); - await editorStore.graphManagerState.initializeSystem(); - stopWatch.record( - GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS, - ); - - // build graph - let pmcd: V1_PureModelContextData | undefined = undefined; - if (dataSpaceAnalysisResult) { - const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResult?.executionContexts.find( - (value) => value.mapping === mappingPath, - )?.mappingModelCoverageAnalysisResult; - pmcd = mappingModelCoverageAnalysisResult?.model; - } - const graph_buildReport = createGraphBuilderReport(); - dataSpaceAnalysisResultMetaModel = - await DSL_DataSpace_getGraphManagerExtension( - editorStore.graphManagerState.graphManager, - ).buildDataSpaceAnalytics( - dataSpaceAnalysisResult, - editorStore.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + undefined, graph_buildReport, editorStore.graphManagerState.graph, - pmcd, + undefined, + mappingPath, editorStore.getProjectInfo(), + editorStore.applicationStore.notificationService, ); - const dependency_buildReport = createGraphBuilderReport(); + // report stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); const graphBuilderReportData = { @@ -221,7 +213,7 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli isLightGraphEnabled = false; editorStore.graphManagerState.graph = editorStore.graphManagerState.createNewGraph(); - await editorStore.buildGraph(); + await flowResult(editorStore.buildFullGraph()); } const dataSpace = getOwnDataSpace( @@ -238,8 +230,6 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli // properly created from a data space, therefore, we cannot support this case return undefined; } - await editorStore.setupQuery(); - const query = guaranteeNonNullable(editorStore.query); const projectInfo = new DataSpaceProjectInfo( query.groupId, query.artifactId, @@ -257,7 +247,7 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli dataSpace, matchingExecutionContext, isLightGraphEnabled, - (dataSpaceInfo: DataSpaceInfo) => { + async (dataSpaceInfo: DataSpaceInfo) => { if (dataSpaceInfo.defaultExecutionContext) { const createQuery = async (): Promise => { // prepare the new query to save @@ -354,14 +344,14 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli } }, true, - dataSpaceAnalysisResultMetaModel, + dataSpaceAnalysisResult, undefined, undefined, undefined, projectInfo, ); const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResultMetaModel?.executionContextsIndex.get( + dataSpaceAnalysisResult?.executionContextsIndex.get( matchingExecutionContext.name, )?.mappingModelCoverageAnalysisResult; if (mappingModelCoverageAnalysisResult) { @@ -376,19 +366,24 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli ]; } - override getExtraExistingQueryGraphBuilderCheckers(): ExistingQueryGraphBuilderChecker[] { + override getExtraQueryGraphBuilderGetters(): QueryGraphBuilderGetter[] { return [ - (editorStore: ExistingQueryEditorStore): boolean | undefined => { - const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( - (taggedValue) => - taggedValue.profile === QUERY_PROFILE_PATH && - taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && - isValidFullPath(taggedValue.value), - ); - if (dataSpaceTaggedValue) { - return false; + ( + editorStore: QueryEditorStore, + ): ((editorStore: QueryEditorStore) => GeneratorFn) | undefined => { + if (editorStore instanceof ExistingQueryEditorStore) { + const query = editorStore.query; + const dataSpaceTaggedValue = query?.taggedValues?.find( + (taggedValue) => + taggedValue.profile === QUERY_PROFILE_PATH && + taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && + isValidFullPath(taggedValue.value), + ); + if (dataSpaceTaggedValue) { + return buildGraph; + } } - return true; + return undefined; }, ]; } diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx index 04713bf831..dd5ac9b5a4 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx @@ -174,8 +174,10 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( queryBuilderState.dataSpace.defaultExecutionContext.name, }, }; - const onDataSpaceOptionChange = (option: DataSpaceOption): void => { - queryBuilderState.onDataSpaceChange(option.value); + const onDataSpaceOptionChange = async ( + option: DataSpaceOption, + ): Promise => { + await queryBuilderState.onDataSpaceChange(option.value); }; // data space search text @@ -206,14 +208,14 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( const selectedExecutionContextOption = buildExecutionContextOption( queryBuilderState.executionContext, ); - const onExecutionContextOptionChange = ( + const onExecutionContextOptionChange = async ( option: ExecutionContextOption, - ): void => { + ): Promise => { if (option.value === queryBuilderState.executionContext) { return; } queryBuilderState.setExecutionContext(option.value); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( option.value, editorStore, ); @@ -422,7 +424,7 @@ export const queryDataSpace = async ( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: () => { + setupQueryBuilderState: async () => { const queryBuilderState = new DataSpaceQueryBuilderState( editorStore.applicationStore, editorStore.graphManagerState, @@ -430,7 +432,7 @@ export const queryDataSpace = async ( dataSpace, dataSpace.defaultExecutionContext, false, - (dataSpaceInfo: DataSpaceInfo) => { + async (dataSpaceInfo: DataSpaceInfo) => { queryBuilderState.dataSpace = guaranteeType( queryBuilderState.graphManagerState.graph.getElement( dataSpaceInfo.path, @@ -440,7 +442,7 @@ export const queryDataSpace = async ( queryBuilderState.setExecutionContext( queryBuilderState.dataSpace.defaultExecutionContext, ); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( queryBuilderState.dataSpace.defaultExecutionContext, ); }, @@ -455,7 +457,7 @@ export const queryDataSpace = async ( queryBuilderState.setExecutionContext( dataSpace.defaultExecutionContext, ); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( dataSpace.defaultExecutionContext, ); return queryBuilderState; diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts index a833323433..bdb0538dd7 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts @@ -17,8 +17,6 @@ import { type AbstractPureGraphManager, AbstractPureGraphManagerExtension, - type PureProtocolProcessorPlugin, - type V1_PureModelContextData, type PureModel, type GraphManagerOperationReport, } from '@finos/legend-graph'; @@ -29,7 +27,7 @@ import { type PlainObject, } from '@finos/legend-shared'; import type { DataSpaceAnalysisResult } from '../../action/analytics/DataSpaceAnalysis.js'; -import type { V1_DataSpaceAnalysisResult } from './v1/engine/analytics/V1_DataSpaceAnalysis.js'; +import type { NotificationService } from '@finos/legend-application'; export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPureGraphManagerExtension { abstract analyzeDataSpace( @@ -44,17 +42,12 @@ export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPu entitiesRetriever: () => Promise, cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise; - - abstract buildDataSpaceAnalytics( - analytics: - | PlainObject - | V1_DataSpaceAnalysisResult, - plugins: PureProtocolProcessorPlugin[], graphReport?: GraphManagerOperationReport | undefined, pureGraph?: PureModel | undefined, - pmcd?: V1_PureModelContextData | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, projectInfo?: ProjectGAVCoordinates, + notificationService?: NotificationService | undefined, ): Promise; } diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts index ba05ff2db9..e380cba695 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts @@ -144,6 +144,29 @@ export class DSL_DataSpace_PureProtocolProcessorPlugin ); return execContext; }); + if (elementProtocol.elements) { + element.elements = elementProtocol.elements.map((pointer) => { + const elementReference = context.resolveElement( + pointer.path, + true, + ); + if ( + elementReference.value instanceof Package || + elementReference.value instanceof Class || + elementReference.value instanceof Enumeration || + elementReference.value instanceof Association + ) { + const elementPointer = new DataSpaceElementPointer(); + elementPointer.element = + elementReference as unknown as PackageableElementReference; + elementPointer.exclude = pointer.exclude; + return elementPointer; + } + throw new UnsupportedOperationError( + `Can't find data space element (only allow packages, classes, enumerations, and associations) '${pointer.path}'`, + ); + }); + } element.defaultExecutionContext = guaranteeNonNullable( element.executionContexts.find( (execContext) => @@ -180,29 +203,6 @@ export class DSL_DataSpace_PureProtocolProcessorPlugin .filter(isNonNullable); element.title = elementProtocol.title; element.description = elementProtocol.description; - if (elementProtocol.elements) { - element.elements = elementProtocol.elements.map((pointer) => { - const elementReference = context.resolveElement( - pointer.path, - true, - ); - if ( - elementReference.value instanceof Package || - elementReference.value instanceof Class || - elementReference.value instanceof Enumeration || - elementReference.value instanceof Association - ) { - const elementPointer = new DataSpaceElementPointer(); - elementPointer.element = - elementReference as unknown as PackageableElementReference; - elementPointer.exclude = pointer.exclude; - return elementPointer; - } - throw new UnsupportedOperationError( - `Can't find data space element (only allow packages, classes, enumerations, and associations) '${pointer.path}'`, - ); - }); - } if (elementProtocol.executables) { element.executables = elementProtocol.executables.map( (executableProtocol) => { diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts index 8f67084c1a..f6821942ef 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts @@ -31,7 +31,6 @@ import { V1_buildDatasetSpecification, type PureProtocolProcessorPlugin, V1_buildModelCoverageAnalysisResult, - type V1_PureModelContextData, type GraphManagerOperationReport, LegendSDLC, } from '@finos/legend-graph'; @@ -76,7 +75,7 @@ import { } from '../../../action/analytics/DataSpaceAnalysis.js'; import { DSL_DataSpace_PureGraphManagerExtension } from '../DSL_DataSpace_PureGraphManagerExtension.js'; import { - V1_DataSpaceAnalysisResult, + type V1_DataSpaceAnalysisResult, V1_DataSpaceAssociationDocumentationEntry, V1_DataSpaceClassDocumentationEntry, V1_DataSpaceEnumerationDocumentationEntry, @@ -87,6 +86,7 @@ import { } from './engine/analytics/V1_DataSpaceAnalysis.js'; import { getDiagram } from '@finos/legend-extension-dsl-diagram/graph'; import { resolveVersion } from '@finos/legend-server-depot'; +import type { NotificationService } from '@finos/legend-application'; const ANALYZE_DATA_SPACE_TRACE = 'analyze data space'; const TEMPORARY__TDS_SAMPLE_VALUES__DELIMETER = '-- e.g.'; @@ -150,11 +150,20 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu entitiesRetriever: () => Promise, cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise { + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, + projectInfo?: ProjectGAVCoordinates, + notificationService?: NotificationService | undefined, + ): Promise { const cacheResult = cacheRetriever ? await this.fetchDataSpaceAnalysisFromCache(cacheRetriever, actionState) : undefined; const engineClient = this.graphManager.engine.getEngineServerClient(); + notificationService?.notify( + `Please release a new version of the project and create a new query from that to reduce the load time`, + ); let analysisResult: PlainObject; if ( cacheResult && @@ -189,9 +198,14 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu { enableCompression: true }, ); } - return V1_deserializeDataSpaceAnalysisResult( + return this.buildDataSpaceAnalytics( analysisResult, this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + graphReport, + pureGraph, + executionContext, + mappingPath, + projectInfo, ); } @@ -216,24 +230,18 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu } async buildDataSpaceAnalytics( - analytics: - | PlainObject - | V1_DataSpaceAnalysisResult, + analytics: PlainObject, plugins: PureProtocolProcessorPlugin[], graphReport?: GraphManagerOperationReport | undefined, pureGraph?: PureModel | undefined, - pmcd?: V1_PureModelContextData | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, projectInfo?: ProjectGAVCoordinates, ): Promise { - let analysisResult: V1_DataSpaceAnalysisResult; - if (analytics instanceof V1_DataSpaceAnalysisResult) { - analysisResult = analytics; - } else { - analysisResult = V1_deserializeDataSpaceAnalysisResult( - analytics, - plugins, - ); - } + const analysisResult = V1_deserializeDataSpaceAnalysisResult( + analytics, + plugins, + ); const result = new DataSpaceAnalysisResult(); result.name = analysisResult.name; result.package = analysisResult.package; @@ -334,6 +342,14 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu runtime.runtimeValue = new V1_EngineRuntime(); return runtime; }); + const mappingModelCoverageAnalysisResult = + analysisResult.executionContexts.find( + (value) => value.name === executionContext, + )?.mappingModelCoverageAnalysisResult ?? + analysisResult.executionContexts.find( + (value) => value.mapping === mappingPath, + )?.mappingModelCoverageAnalysisResult; + const pmcd = mappingModelCoverageAnalysisResult?.model; if (pmcd && projectInfo) { graphEntities = pmcd.elements .concat(mappingModels) @@ -342,7 +358,7 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu // so we could rid of it .filter((el) => !graph.getNullableElement(el.path, false)) .map((el) => this.graphManager.elementProtocolToEntity(el)); - await this.graphManager.buildGraph( + await this.graphManager.buildGraphForQuery( graph, graphEntities, ActionState.create(), @@ -354,7 +370,6 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu ), }, graphReport, - true, ); } else { // prepare the model context data diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts index 7d2c366c20..55ebb2c196 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts @@ -59,7 +59,7 @@ import { guaranteeNonNullable, guaranteeType, } from '@finos/legend-shared'; -import { action, flow, makeObservable, observable } from 'mobx'; +import { action, flow, flowResult, makeObservable, observable } from 'mobx'; import { renderDataSpaceQueryBuilderSetupPanelContent } from '../../components/query/DataSpaceQueryBuilder.js'; import { DataSpace, @@ -69,7 +69,10 @@ import { DATA_SPACE_ELEMENT_CLASSIFIER_PATH } from '../../graph-manager/protocol import { type DataSpaceInfo, extractDataSpaceInfo } from './DataSpaceInfo.js'; import { DataSpaceAdvancedSearchState } from './DataSpaceAdvancedSearchState.js'; import type { DataSpaceAnalysisResult } from '../../graph-manager/action/analytics/DataSpaceAnalysis.js'; -import type { QueryEditorStore } from '@finos/legend-application-query'; +import { + LEGEND_QUERY_APP_EVENT, + type QueryEditorStore, +} from '@finos/legend-application-query'; export const resolveUsableDataSpaceClasses = ( queryBuilderState: DataSpaceQueryBuilderState, @@ -88,11 +91,16 @@ export const resolveUsableDataSpaceClasses = ( .filter(filterByType(Class)), ]); } else if ( - queryBuilderState.explorerState.mappingModelCoverageAnalysisResult + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult && + // This check is to make sure that we have `info` field present in `MappedEntity` which + // contains information about the mapped class path + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult.mappedEntities.some( + (m) => m.info !== undefined, + ) ) { const compatibleClassPaths = queryBuilderState.explorerState.mappingModelCoverageAnalysisResult.mappedEntities.map( - (e) => e.classPath, + (e) => e.info?.classPath, ); const uniqueCompatibleClasses = compatibleClassPaths.filter( (val, index) => compatibleClassPaths.indexOf(val) === index, @@ -148,7 +156,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { readonly depotServerClient: DepotServerClient; readonly isAdvancedDataSpaceSearchEnabled: boolean; readonly loadDataSpacesState = ActionState.create(); - readonly onDataSpaceChange: (val: DataSpaceInfo) => void; + readonly onDataSpaceChange: (val: DataSpaceInfo) => Promise; readonly onExecutionContextChange?: | ((val: DataSpaceExecutionContext) => void) | undefined; @@ -174,7 +182,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { dataSpace: DataSpace, executionContext: DataSpaceExecutionContext, isLightGraphEnabled: boolean, - onDataSpaceChange: (val: DataSpaceInfo) => void, + onDataSpaceChange: (val: DataSpaceInfo) => Promise, isAdvancedDataSpaceSearchEnabled: boolean, dataSpaceAnalysisResult?: DataSpaceAnalysisResult | undefined, onExecutionContextChange?: @@ -317,115 +325,124 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { isGraphBuildingNotRequired?: boolean, ): Promise { const mapping = executionContext.mapping.value; - let compatibleClasses; const mappingModelCoverageAnalysisResult = this.dataSpaceAnalysisResult?.executionContextsIndex.get( executionContext.name, )?.mappingModelCoverageAnalysisResult; if (this.dataSpaceAnalysisResult && mappingModelCoverageAnalysisResult) { if (!isGraphBuildingNotRequired && editorStore) { - const stopWatch = new StopWatch(); - const graph = this.graphManagerState.createNewGraph(); + try { + const stopWatch = new StopWatch(); + const graph = this.graphManagerState.createNewGraph(); - const graph_buildReport = createGraphBuilderReport(); - // Create dummy mappings and runtimes - // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should - // simplify the model here or potentially refactor the backend analytics endpoint to return these as model - const mappingModels = uniq( - Array.from( - this.dataSpaceAnalysisResult.executionContextsIndex.values(), - ).map((context) => context.mapping), - ).map((m) => { - const _mapping = new V1_Mapping(); - const [packagePath, name] = resolvePackagePathAndElementName(m.path); - _mapping.package = packagePath; - _mapping.name = name; - return guaranteeType( - this.graphManagerState.graphManager, - V1_PureGraphManager, - ).elementProtocolToEntity(_mapping); - }); - const runtimeModels = uniq( - Array.from( - this.dataSpaceAnalysisResult.executionContextsIndex.values(), + const graph_buildReport = createGraphBuilderReport(); + // Create dummy mappings and runtimes + // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should + // simplify the model here or potentially refactor the backend analytics endpoint to return these as model + const mappingModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).map((context) => context.mapping), + ).map((m) => { + const _mapping = new V1_Mapping(); + const [packagePath, name] = resolvePackagePathAndElementName( + m.path, + ); + _mapping.package = packagePath; + _mapping.name = name; + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(_mapping); + }); + const runtimeModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ) + .map((context) => context.defaultRuntime) + .concat( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).flatMap((val) => val.compatibleRuntimes), + ), + ).map((r) => { + const runtime = new V1_PackageableRuntime(); + const [packagePath, name] = resolvePackagePathAndElementName( + r.path, + ); + runtime.package = packagePath; + runtime.name = name; + runtime.runtimeValue = new V1_EngineRuntime(); + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(runtime); + }); + const graphEntities = guaranteeNonNullable( + mappingModelCoverageAnalysisResult.entities, ) - .map((context) => context.defaultRuntime) - .concat( - Array.from( - this.dataSpaceAnalysisResult.executionContextsIndex.values(), - ).flatMap((val) => val.compatibleRuntimes), - ), - ).map((r) => { - const runtime = new V1_PackageableRuntime(); - const [packagePath, name] = resolvePackagePathAndElementName(r.path); - runtime.package = packagePath; - runtime.name = name; - runtime.runtimeValue = new V1_EngineRuntime(); - return guaranteeType( - this.graphManagerState.graphManager, - V1_PureGraphManager, - ).elementProtocolToEntity(runtime); - }); - const graphEntities = guaranteeNonNullable( - mappingModelCoverageAnalysisResult.entities, - ) - .concat(mappingModels) - .concat(runtimeModels) - // NOTE: if an element could be found in the graph already it means it comes from system - // so we could rid of it - .filter( - (el) => - !graph.getNullableElement(el.path, false) && - !el.path.startsWith('meta::'), + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter( + (el) => + !graph.getNullableElement(el.path, false) && + !el.path.startsWith('meta::'), + ); + await this.graphManagerState.graphManager.buildGraphForQuery( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + guaranteeNonNullable(this.projectInfo).groupId, + guaranteeNonNullable(this.projectInfo).artifactId, + resolveVersion( + guaranteeNonNullable(this.projectInfo).versionId, + ), + ), + }, + graph_buildReport, ); - await this.graphManagerState.graphManager.buildGraph( - graph, - graphEntities, - ActionState.create(), - { - origin: new LegendSDLC( - guaranteeNonNullable(this.projectInfo).groupId, - guaranteeNonNullable(this.projectInfo).artifactId, - resolveVersion(guaranteeNonNullable(this.projectInfo).versionId), - ), - }, - graph_buildReport, - true, - ); - this.graphManagerState.graph = graph; - const dependency_buildReport = createGraphBuilderReport(); - // report - stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); - const graphBuilderReportData = { - timings: - this.applicationStore.timeService.finalizeTimingsRecord(stopWatch), - dependencies: dependency_buildReport, - dependenciesCount: - this.graphManagerState.graph.dependencyManager.numberOfDependencies, - graph: graph_buildReport, - }; - editorStore.logBuildGraphMetrics(graphBuilderReportData); + this.graphManagerState.graph = graph; + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord( + stopWatch, + ), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager + .numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); - this.applicationStore.logService.info( - LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), - graphBuilderReportData, - ); + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } catch (error) { + assertErrorThrown(error); + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + + editorStore.graphManagerState.graph = + editorStore.graphManagerState.createNewGraph(); + await flowResult(editorStore.buildFullGraph()); + } } - const compatibleClassPaths = - mappingModelCoverageAnalysisResult.mappedEntities.map( - (e) => e.classPath, - ); - const uniqueCompatibleClasses = compatibleClassPaths.filter( - (val, index) => compatibleClassPaths.indexOf(val) === index, - ); - compatibleClasses = this.graphManagerState.graph.classes.filter((c) => - uniqueCompatibleClasses.includes(c.path), - ); this.explorerState.mappingModelCoverageAnalysisResult = mappingModelCoverageAnalysisResult; - } else { - compatibleClasses = resolveUsableDataSpaceClasses(this); } + const compatibleClasses = resolveUsableDataSpaceClasses(this); + this.changeMapping(mapping); this.changeRuntime(new RuntimePointer(executionContext.defaultRuntime)); diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts index 4291e2e224..d0fb4a4c71 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts @@ -23,7 +23,6 @@ import { type Runtime, type Class, type RawLambda, - type V1_PureModelContextData, createGraphBuilderReport, GRAPH_MANAGER_EVENT, } from '@finos/legend-graph'; @@ -46,6 +45,7 @@ import { LogEvent, StopWatch, uuid, + type GeneratorFn, } from '@finos/legend-shared'; import { QUERY_PROFILE_PATH, @@ -64,6 +64,7 @@ import type { QueryBuilderState } from '@finos/legend-query-builder'; import type { ProjectGAVCoordinates } from '@finos/legend-storage'; import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; +import { flowResult } from 'mobx'; export const createQueryDataSpaceTaggedValue = ( dataSpacePath: string, @@ -130,7 +131,6 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { async initializeQueryBuilderState(): Promise { let dataSpaceAnalysisResult; - let dataSpaceAnalysisResultMetaModel; let isLightGraphEnabled = true; try { const stopWatch = new StopWatch(); @@ -138,6 +138,13 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { await this.depotServerClient.getProject(this.groupId, this.artifactId), ); this.initState.setMessage('Fetching dataspace analysis result'); + // initialize system + stopWatch.record(); + await this.graphManagerState.initializeSystem(); + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS); + + const graph_buildReport = createGraphBuilderReport(); + const dependency_buildReport = createGraphBuilderReport(); dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( this.graphManagerState.graphManager, ).analyzeDataSpaceCoverage( @@ -155,38 +162,18 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { this.dataSpacePath, this.depotServerClient, ), + undefined, + graph_buildReport, + this.graphManagerState.graph, + this.executionContext, + undefined, + this.getProjectInfo(), + this.applicationStore.notificationService, ); - const mappingPath = dataSpaceAnalysisResult?.executionContexts.find( - (e) => e.name === this.executionContext, + const mappingPath = dataSpaceAnalysisResult.executionContextsIndex.get( + this.executionContext, )?.mapping; - if (dataSpaceAnalysisResult && mappingPath) { - // initialize system - stopWatch.record(); - await this.graphManagerState.initializeSystem(); - stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS); - - // build graph - let pmcd: V1_PureModelContextData | undefined = undefined; - if (dataSpaceAnalysisResult) { - const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResult?.executionContexts.find( - (value) => value.mapping === mappingPath, - )?.mappingModelCoverageAnalysisResult; - pmcd = mappingModelCoverageAnalysisResult?.model; - } - const graph_buildReport = createGraphBuilderReport(); - dataSpaceAnalysisResultMetaModel = - await DSL_DataSpace_getGraphManagerExtension( - this.graphManagerState.graphManager, - ).buildDataSpaceAnalytics( - dataSpaceAnalysisResult, - this.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), - graph_buildReport, - this.graphManagerState.graph, - pmcd, - this.getProjectInfo(), - ); - const dependency_buildReport = createGraphBuilderReport(); + if (mappingPath) { // report stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); const graphBuilderReportData = { @@ -204,7 +191,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { } else { isLightGraphEnabled = false; this.graphManagerState.graph = this.graphManagerState.createNewGraph(); - await this.buildGraph(); + await flowResult(this.buildFullGraph()); } } catch (error) { this.applicationStore.logService.error( @@ -213,7 +200,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { ); this.graphManagerState.graph = this.graphManagerState.createNewGraph(); isLightGraphEnabled = false; - await this.buildGraph(); + await flowResult(this.buildFullGraph()); } const dataSpace = getDataSpace( this.dataSpacePath, @@ -242,7 +229,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { dataSpace, executionContext, isLightGraphEnabled, - (dataSpaceInfo: DataSpaceInfo) => { + async (dataSpaceInfo: DataSpaceInfo) => { if (dataSpaceInfo.defaultExecutionContext) { this.applicationStore.navigationService.navigator.goToLocation( generateDataSpaceQueryCreatorRoute( @@ -262,7 +249,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { } }, true, - dataSpaceAnalysisResultMetaModel, + dataSpaceAnalysisResult, (ec: DataSpaceExecutionContext) => { // runtime should already be set const runtimePointer = guaranteeType( @@ -325,7 +312,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { projectInfo, ); queryBuilderState.setExecutionContext(executionContext); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( executionContext, this, true, @@ -376,4 +363,8 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { }, }; } + + override *buildGraph(): GeneratorFn { + // do nothing + } } diff --git a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts index 4a3ddebede..6d00ecf13e 100644 --- a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts @@ -254,6 +254,17 @@ export abstract class AbstractPureGraphManager { report?: GraphManagerOperationReport, ): Promise; + /** + * Process entities and build the main graph for query. + */ + abstract buildGraphForQuery( + graph: PureModel, + entities: Entity[], + buildState: ActionState, + options?: GraphBuilderOptions, + report?: GraphManagerOperationReport, + ): Promise; + /** * Process entities and build the main graph. */ @@ -263,7 +274,6 @@ export abstract class AbstractPureGraphManager { buildState: ActionState, options?: GraphBuilderOptions, report?: GraphManagerOperationReport, - buildRequiredGraph?: boolean | undefined, ): Promise; /** diff --git a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts index 2857f19bf2..8d8ed66a75 100644 --- a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts @@ -19,17 +19,35 @@ import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/ export type RawMappingModelCoverageAnalysisResult = object; +export class MappedEntityInfo { + readonly __PROPERTIES_INDEX = new Map(); + + classPath: string; + isRootEntity: boolean; + subClasses: string[]; + + constructor(classPath: string, isRootEntity: boolean, subClasses: string[]) { + this.isRootEntity = isRootEntity; + this.subClasses = subClasses; + this.classPath = classPath; + } +} + export class MappedEntity { readonly __PROPERTIES_INDEX = new Map(); path: string; - classPath: string; properties: MappedProperty[]; + info?: MappedEntityInfo | undefined; - constructor(path: string, classPath: string, properties: MappedProperty[]) { + constructor( + path: string, + properties: MappedProperty[], + info?: MappedEntityInfo | undefined, + ) { this.path = path; - this.classPath = classPath; this.properties = properties; + this.info = info; properties.forEach((property) => this.__PROPERTIES_INDEX.set(property.name, property), ); diff --git a/packages/legend-graph/src/graph-manager/action/query/Query.ts b/packages/legend-graph/src/graph-manager/action/query/Query.ts index 29c88c2211..af9ce1c11b 100644 --- a/packages/legend-graph/src/graph-manager/action/query/Query.ts +++ b/packages/legend-graph/src/graph-manager/action/query/Query.ts @@ -14,10 +14,6 @@ * limitations under the License. */ -import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js'; -import type { PackageableElementReference } from '../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js'; -import type { PackageableRuntime } from '../../../graph/metamodel/pure/packageableElements/runtime/PackageableRuntime.js'; - export class QueryTaggedValue { profile!: string; tag!: string; @@ -40,8 +36,9 @@ export class Query { versionId!: string; groupId!: string; artifactId!: string; - mapping!: PackageableElementReference; - runtime!: PackageableElementReference; + // NOTE: Query can be built before the actual graph is built so we can't have the reference of metamodels here + mapping!: string; + runtime!: string; // We enforce a single owner, for collaboration on query, use Studio // if not owner is specified, any user can own the query // NOTE: the owner is managed automatically by the backend @@ -67,8 +64,6 @@ export class LightQuery { artifactId!: string; owner?: string | undefined; lastUpdatedAt?: number | undefined; - mapping?: string | PackageableElementReference | undefined; - taggedValues?: QueryTaggedValue[] | undefined; isCurrentUserQuery = false; } @@ -82,8 +77,6 @@ export const toLightQuery = (query: Query): LightQuery => { lightQuery.versionId = query.versionId; lightQuery.owner = query.owner; lightQuery.isCurrentUserQuery = query.isCurrentUserQuery; - lightQuery.mapping = query.mapping.value.path; - lightQuery.taggedValues = query.taggedValues; return lightQuery; }; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts index e8fa4aa7d2..0138b4acaf 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts @@ -797,13 +797,99 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { } } + async buildGraphForQuery( + graph: PureModel, + entities: Entity[], + buildState: ActionState, + options?: GraphBuilderOptions, + _report?: GraphManagerOperationReport, + ): Promise { + const stopWatch = new StopWatch(); + const report = _report ?? createGraphBuilderReport(); + buildState.reset(); + + try { + // deserialize + buildState.setMessage(`Deserializing elements...`); + const data = new V1_PureModelContextData(); + await V1_entitiesToPureModelContextData( + entities, + data, + this.pluginManager.getPureProtocolProcessorPlugins(), + this.subtypeInfo, + ); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS, + ); + + // prepare build inputs + const buildInputs: V1_PureGraphBuilderInput[] = [ + { + model: graph, + data: V1_indexPureModelContextData( + report, + data, + this.graphBuilderExtensions, + ), + }, + ]; + + // index + buildState.setMessage( + `Indexing ${report.elementCount.total} elements...`, + ); + await this.initializeAndIndexElements(graph, buildInputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS, + ); + // build types + buildState.setMessage(`Building domain models...`); + await this.buildTypes(graph, buildInputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS, + ); + + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS, + ); + + if (options?.origin) { + graph.setOrigin(options.origin); + } + + buildState.pass(); + + const totalTime = stopWatch.elapsed; + report.timings = { + ...Object.fromEntries(stopWatch.records), + [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime, + total: totalTime, + }; + } catch (error) { + assertErrorThrown(error); + this.logService.error( + LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), + error, + ); + buildState.fail(); + /** + * Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder + * But we might want to rethink this decision in the future and throw appropriate type of error + */ + throw error instanceof GraphBuilderError + ? error + : new GraphBuilderError(error); + } finally { + buildState.setMessage(undefined); + } + } + async buildGraph( graph: PureModel, entities: Entity[], buildState: ActionState, options?: GraphBuilderOptions, _report?: GraphManagerOperationReport, - buildRequiredGraph?: boolean | undefined, ): Promise { const stopWatch = new StopWatch(); const report = _report ?? createGraphBuilderReport(); @@ -837,25 +923,14 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ]; // build - if (!buildRequiredGraph) { - await this.buildGraphFromInputs( - graph, - buildInputs, - report, - stopWatch, - buildState, - options, - ); - } else { - await this.buildRequiredGraphFromInputs( - graph, - buildInputs, - report, - stopWatch, - buildState, - options, - ); - } + await this.buildGraphFromInputs( + graph, + buildInputs, + report, + stopWatch, + buildState, + options, + ); /** * For now, we delete the section index. We are able to read both resolved and unresolved element paths @@ -1142,32 +1217,6 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ); } - private async buildRequiredGraphFromInputs( - graph: PureModel, - inputs: V1_PureGraphBuilderInput[], - report: GraphManagerOperationReport, - stopWatch: StopWatch, - graphBuilderState: ActionState, - options?: GraphBuilderOptions, - ): Promise { - // index - graphBuilderState.setMessage( - `Indexing ${report.elementCount.total} elements...`, - ); - await this.initializeAndIndexElements(graph, inputs, options); - stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS); - // build types - graphBuilderState.setMessage(`Building domain models...`); - await this.buildTypes(graph, inputs, options); - stopWatch.record( - GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS, - ); - - stopWatch.record( - GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS, - ); - } - private async buildLightGraphFromInputs( graph: PureModel, inputs: V1_PureGraphBuilderInput[], diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts index b120b63841..b7617df245 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts @@ -32,7 +32,6 @@ import { V1_QueryParameterValue, } from './query/V1_Query.js'; import type { PureModel } from '../../../../../graph/PureModel.js'; -import { PackageableElementExplicitReference } from '../../../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js'; import { DEPRECATED__ServiceTestResult } from '../../../../../graph-manager/action/service/DEPRECATED__ServiceTestResult.js'; import type { V1_DEPRECATED__ServiceTestResult } from './service/V1_DEPRECATED__ServiceTestResult.js'; import type { V1_ServiceRegistrationResult } from './service/V1_ServiceRegistrationResult.js'; @@ -97,29 +96,10 @@ export const V1_buildLightQuery = ( protocol.artifactId, `Query 'artifactId' field is missing`, ); - metamodel.mapping = protocol.mapping; metamodel.owner = protocol.owner; metamodel.lastUpdatedAt = protocol.lastUpdatedAt; metamodel.isCurrentUserQuery = currentUserId !== undefined && protocol.owner === currentUserId; - // NOTE: we don't properly process tagged values and stereotypes for query - // because these profiles/tags/stereotypes can come from external systems. - metamodel.taggedValues = protocol.taggedValues?.map((taggedValueProtocol) => { - const taggedValue = new QueryTaggedValue(); - taggedValue.profile = guaranteeNonEmptyString( - taggedValueProtocol.tag.profile, - `Tagged value 'tag.profile' field is missing or empty`, - ); - taggedValue.tag = guaranteeNonEmptyString( - taggedValueProtocol.tag.value, - `Tagged value 'tag.value' field is missing or empty`, - ); - taggedValue.value = guaranteeNonEmptyString( - taggedValueProtocol.value, - `Tagged value 'value' field is missing or empty`, - ); - return taggedValue; - }); return metamodel; }; @@ -149,22 +129,8 @@ export const V1_buildQuery = ( protocol.artifactId, `Query 'artifactId' field is missing`, ); - metamodel.mapping = PackageableElementExplicitReference.create( - graph.getMapping( - guaranteeNonNullable( - protocol.mapping, - `Query 'mapping' field is missing`, - ), - ), - ); - metamodel.runtime = PackageableElementExplicitReference.create( - graph.getRuntime( - guaranteeNonNullable( - protocol.runtime, - `Query 'runtime' field is missing`, - ), - ), - ); + metamodel.mapping = protocol.mapping; + metamodel.runtime = protocol.runtime; metamodel.content = guaranteeNonNullable( protocol.content, `Query 'content' field is missing`, @@ -229,8 +195,8 @@ export const V1_transformQuery = (metamodel: Query): V1_Query => { protocol.versionId = metamodel.versionId; protocol.groupId = metamodel.groupId; protocol.artifactId = metamodel.artifactId; - protocol.mapping = metamodel.mapping.valueForSerialization ?? ''; - protocol.runtime = metamodel.runtime.valueForSerialization ?? ''; + protocol.mapping = metamodel.mapping; + protocol.runtime = metamodel.runtime; protocol.content = metamodel.content; protocol.owner = metamodel.owner; protocol.taggedValues = metamodel.taggedValues?.map((_taggedValue) => { diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts index cf3d1d3645..6a45611f23 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts @@ -33,6 +33,7 @@ import { EntityMappedProperty, EnumMappedProperty, MappedEntity, + MappedEntityInfo, MappedProperty, MappingModelCoverageAnalysisResult, } from '../../../../../../graph-manager/action/analytics/MappingModelCoverageAnalysis.js'; @@ -107,15 +108,31 @@ const V1_deserializeMappedProperty = ( } }; +class V1_MappedEntityInfo { + classPath!: string; + isRootEntity!: boolean; + subClasses: string[] = []; + + static readonly serialization = new SerializationFactory( + createModelSchema(V1_MappedEntityInfo, { + classPath: primitive(), + isRootEntity: primitive(), + subClasses: list(primitive()), + }), + ); +} + class V1_MappedEntity { path!: string; - classPath!: string; properties: V1_MappedProperty[] = []; + info?: V1_MappedEntityInfo | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappedEntity, { path: primitive(), - classPath: primitive(), + info: optional( + usingModelSchema(V1_MappedEntityInfo.serialization.schema), + ), properties: list( custom( (prop) => V1_serializeMappedProperty(prop), @@ -170,12 +187,23 @@ const buildMappedProperty = (protocol: V1_MappedProperty): MappedProperty => ? new EnumMappedProperty(protocol.name, protocol.enumPath) : new MappedProperty(protocol.name); -const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => - new MappedEntity( - protocol.path, +const buildMappedEntityInfo = ( + protocol: V1_MappedEntityInfo, +): MappedEntityInfo => + new MappedEntityInfo( protocol.classPath, + protocol.isRootEntity, + protocol.subClasses, + ); + +const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => { + const info = protocol.info ? buildMappedEntityInfo(protocol.info) : undefined; + return new MappedEntity( + protocol.path, protocol.properties.map((p) => buildMappedProperty(p)), + info, ); +}; export const V1_buildModelCoverageAnalysisResult = ( protocol: V1_MappingModelCoverageAnalysisResult,