From 128d017e2c8eb75d9439a3d09c6739bc0c552938 Mon Sep 17 00:00:00 2001 From: Dmitriy L <97888955+wolfeeeg@users.noreply.github.com> Date: Mon, 23 Jan 2023 19:49:02 +0300 Subject: [PATCH] feat(api-gateway, server-core): Added dataSources method (#5789) --- packages/cubejs-api-gateway/src/gateway.ts | 30 +++++++++ .../cubejs-api-gateway/test/index.test.ts | 1 + packages/cubejs-api-gateway/test/mocks.ts | 8 ++- .../src/core/CompilerApi.js | 48 +++++++++++++ .../test/unit/index.test.ts | 67 ++++++++++++++++++- 5 files changed, 152 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index a653b8f2286ed..4a5b5100b5593 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -366,6 +366,18 @@ class ApiGateway { }); } ); + + app.get( + '/cubejs-system/v1/data-sources', + jsonParser, + systemMiddlewares, + async (req: Request, res: Response) => { + await this.dataSources({ + context: req.context, + res: this.resToResultFn(res), + }); + } + ); } app.get('/readyz', guestMiddlewares, cachedHandler(this.readiness)); @@ -1005,6 +1017,24 @@ class ApiGateway { }); } } + + protected async dataSources({ context, res }: { context?: RequestContext, res: ResponseResultFn }) { + const requestStarted = new Date(); + + try { + const orchestratorApi = await this.getAdapterApi(context); + const { dataSources } = await this.getCompilerApi(context).dataSources(orchestratorApi); + + res({ dataSources }); + } catch (e) { + this.handleError({ + e, + context, + res, + requestStarted, + }); + } + } protected async sqlRunner({ query, context, res }: QueryRequest) { const requestStarted = new Date(); diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index a5e87829bfe1a..ae255fd859bf1 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -592,6 +592,7 @@ describe('API Gateway', () => { } }] }, + { route: 'data-sources', successResult: { dataSources: ['default'] } }, ]; testConfigs.forEach((config) => { diff --git a/packages/cubejs-api-gateway/test/mocks.ts b/packages/cubejs-api-gateway/test/mocks.ts index d5aaf30d3b8bc..739638a032aa5 100644 --- a/packages/cubejs-api-gateway/test/mocks.ts +++ b/packages/cubejs-api-gateway/test/mocks.ts @@ -139,7 +139,13 @@ export const compilerApi = jest.fn().mockImplementation(() => ({ async preAggregations() { return preAggregationsResultFactory(); - } + }, + + async dataSources() { + return { + dataSources: ['default'] + }; + }, })); export class RefreshSchedulerMock { diff --git a/packages/cubejs-server-core/src/core/CompilerApi.js b/packages/cubejs-server-core/src/core/CompilerApi.js index 63f883170a0db..d5c1f6be21bb5 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.js +++ b/packages/cubejs-server-core/src/core/CompilerApi.js @@ -196,6 +196,54 @@ export class CompilerApi { }; } + async dataSources(orchestratorApi) { + let compilerVersion = ( + this.schemaVersion && await this.schemaVersion() || + 'default_schema_version' + ); + + if (typeof compilerVersion === 'object') { + compilerVersion = JSON.stringify(compilerVersion); + } + + let { compilers } = this; + if (!compilers || this.compilerVersion !== compilerVersion) { + compilers = await compile(this.repository, { + allowNodeRequire: this.allowNodeRequire, + compileContext: this.compileContext, + allowJsDuplicatePropsInSchema: this.allowJsDuplicatePropsInSchema, + standalone: this.standalone, + }); + } + + const { cubeEvaluator } = await compilers; + + let dataSources = await Promise.all( + cubeEvaluator + .cubeNames() + .map( + async (cube) => cubeEvaluator.cubeFromPath(cube).dataSource ?? 'default' + ) + ); + + dataSources = [...new Set(dataSources)]; + + dataSources = await Promise.all( + dataSources.map(async (dataSource) => { + try { + await orchestratorApi.driverFactory(dataSource); + return dataSource; + } catch (err) { + return null; + } + }) + ); + + return { + dataSources: dataSources.filter((source) => source), + }; + } + canUsePreAggregationForTransformedQuery(transformedQuery, refs) { return PreAggregations.canUsePreAggregationForTransformedQueryFn(transformedQuery, refs); } diff --git a/packages/cubejs-server-core/test/unit/index.test.ts b/packages/cubejs-server-core/test/unit/index.test.ts index 365065a487846..8098ff2ef7469 100644 --- a/packages/cubejs-server-core/test/unit/index.test.ts +++ b/packages/cubejs-server-core/test/unit/index.test.ts @@ -39,7 +39,7 @@ cube('Bar', { type: 'count' } }, - + dimensions: { time: { sql: 'timestamp', @@ -57,6 +57,28 @@ const repositoryWithoutContent: SchemaFileRepository = { dataSchemaFiles: () => Promise.resolve([{ fileName: 'main.js', content: '' }]), }; +const repositoryWithDataSource: SchemaFileRepository = { + localPath: () => __dirname, + dataSchemaFiles: () => Promise.resolve([{ fileName: 'main.js', content: ` +cube('Bar', { + sql: 'select * from bar', + + measures: { + count: { + type: 'count' + } + }, + dimensions: { + time: { + sql: 'timestamp', + type: 'time' + } + }, + dataSource: 'main' +}); +` }]), +}; + describe('index.test', () => { beforeEach(() => { delete process.env.CUBEJS_EXT_DB_TYPE; @@ -366,6 +388,49 @@ describe('index.test', () => { expect(metaConfigExtendedSpy).toHaveBeenCalled(); metaConfigExtendedSpy.mockClear(); }); + + test('CompilerApi dataSources default', async () => { + const dataSources = await compilerApi.dataSources({ + driverFactory: jest.fn(async () => true) + }); + + expect(dataSources).toHaveProperty('dataSources'); + expect(dataSources.dataSources).toEqual([]); + }); + }); + + describe('CompilerApi dataSources method', () => { + const logger = jest.fn(() => {}); + const compilerApi = new CompilerApi( + repositoryWithDataSource, + async () => 'mysql', + { logger } + ); + + const dataSourcesSpy = jest.spyOn(compilerApi, 'dataSources'); + test('CompilerApi dataSources', async () => { + const dataSources = await compilerApi.dataSources({ + driverFactory: jest.fn(async () => true) + }); + + expect(dataSources).toHaveProperty('dataSources'); + expect(dataSources.dataSources).toEqual(['main']); + expect(dataSourcesSpy).toHaveBeenCalled(); + dataSourcesSpy.mockClear(); + }); + + test('CompilerApi dataSources with driverFactory error', async () => { + const dataSources = await compilerApi.dataSources({ + driverFactory: jest.fn(async () => { + throw new Error('Some driverFactory error'); + }) + }); + + expect(dataSources).toHaveProperty('dataSources'); + expect(dataSources.dataSources).toEqual([]); + expect(dataSourcesSpy).toHaveBeenCalled(); + dataSourcesSpy.mockClear(); + }); }); test('Should create instance of CubejsServerCore, dbType from process.env.CUBEJS_DB_TYPE', () => {