From 7e4b67de6d5a1666ce4c9df3a79b3aecc36baacf Mon Sep 17 00:00:00 2001 From: Lu Yu Date: Tue, 20 Feb 2024 19:09:33 -0800 Subject: [PATCH] Add api registry and allow it to be added into client config in data source plugin (#5895) (#5906) * add api registry and allow it to be added into client config * add changelog * add documentation for multi data source plugin api registry * change to resolve promise before calling getQueryClient --------- Signed-off-by: Lu Yu --- CHANGELOG.md | 1 + .../client_management_design.md | 26 ++++++++++++++++++- .../server/client/client_config.ts | 4 ++- .../server/client/configure_client.test.ts | 4 +++ .../server/client/configure_client.ts | 15 +++++++++-- .../server/legacy/client_config.ts | 4 ++- .../legacy/configure_legacy_client.test.ts | 3 +++ .../server/legacy/configure_legacy_client.ts | 14 ++++++++-- src/plugins/data_source/server/plugin.ts | 17 ++++++++++-- .../custom_api_schema_registry.test.ts | 20 ++++++++++++++ .../custom_api_schema_registry.ts | 19 ++++++++++++++ .../server/schema_registry/index.ts | 6 +++++ src/plugins/data_source/server/types.ts | 5 ++++ 13 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 src/plugins/data_source/server/schema_registry/custom_api_schema_registry.test.ts create mode 100644 src/plugins/data_source/server/schema_registry/custom_api_schema_registry.ts create mode 100644 src/plugins/data_source/server/schema_registry/index.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4126a36f105b..3de0726d56ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Multiple Datasource] Add datasource picker to import saved object flyout when multiple data source is enabled ([#5781](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5781)) - [Multiple Datasource] Add interfaces to register add-on authentication method from plug-in module ([#5851](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5851)) - [Multiple Datasource] Able to Hide "Local Cluster" option from datasource DropDown ([#5827](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5827)) +- [Multiple Datasource] Add api registry and allow it to be added into client config in data source plugin ([#5895](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5895)) ### 🐛 Bug Fixes diff --git a/docs/multi-datasource/client_management_design.md b/docs/multi-datasource/client_management_design.md index c83b254fe289..6c5befdb143f 100644 --- a/docs/multi-datasource/client_management_design.md +++ b/docs/multi-datasource/client_management_design.md @@ -10,6 +10,8 @@ This design is part of the OpenSearch Dashboards multi data source project [[RFC 2. How to expose data source clients to callers through clean interfaces? 3. How to maintain backwards compatibility if user turn off this feature? 4. How to manage multiple clients/connection efficiently, and not consume all the memory? +5. Where should we implement the core logic? +6. How to register custom API schema and add into the client during initialization? ## 2. Requirements @@ -87,6 +89,20 @@ Current `opensearch service` exists in core. The module we'll implement has simi 2. We don't mess up with OpenSearch Dashboards core, since this is an experimental feature, the potential risk of breaking existing behavior will be lowered if we use plugin. Worst case, user could just uninstall the plugin. 3. Complexity wise, it's about the same amount of work. +**6.How to register custom API schema and add into the client during initialization?** +Currently, OpenSearch Dashboards plugins uses the following to initialize instance of Cluster client and register the custom API schema via the plugins configuration option. +```ts +core.opensearch.legacy.createClient( + 'exampleName', + { + plugins: [ExamplePlugin], + } + ); +``` +The downside of this approach is the schema is defined inside the plugin and there is no centralized registry for the schema making it not easy to access. This will be resolved by implementing a centralized API schema registry, and consumers can add data source plugin as dependency and be able to consume all the registered schema, eg. `dataSource.registerCustomApiSchema(sqlPlugin)`. + +The schema will be added into the client configuration when multi data source client is initiated. + ### 4.1 Data Source Plugin Create a data source plugin that only has server side code, to hold most core logic of data source feature. Including data service, crypto service, and client management. A plugin will have all setup, start and stop as lifecycle. @@ -146,12 +162,20 @@ context.core.opensearch.legacy.client.callAsCurrentUser; context.core.opensearch.client.asCurrentUser; ``` -Since deprecating legacy client could be a bigger scope of project, multiple data source feature still need to implement a substitute for it as for now. Implementation should be done in a way that's decoupled with data source client as much as possible, for easier deprecation. Similar to [opensearch legacy service](https://github.com/opensearch-project/OpenSearch-Dashboards/tree/main/src/core/server/opensearch/legacy) in core. +Since deprecating legacy client could be a bigger scope of project, multiple data source feature still need to implement a substitute for it as for now. Implementation should be done in a way that's decoupled with data source client as much as possible, for easier deprecation. Similar to [opensearch legacy service](https://github.com/opensearch-project/OpenSearch-Dashboards/tree/main/src/core/server/opensearch/legacy) in core. See how to intialize the data source client below: ```ts context.dataSource.opensearch.legacy.getClient(dataSourceId); ``` +If using Legacy cluster client with asScoped and callAsCurrentUser, the following is the equivalent when using data source client: +```ts +//legacy cluster client +const response = client.asScoped(request).callAsCurrentUser(format, params); +//equivalent when using data source client instead +const response = client.callAPI(format, params); +``` + ### 4.3 Register datasource client to core context This is for plugin to access data source client via request handler. For example, by `core.client.search(params)`. It’s a very common use case for plugin to access cluster while handling request. In fact data plugin uses it in its search module to get client, and I’ll talk about it in details in next section. diff --git a/src/plugins/data_source/server/client/client_config.ts b/src/plugins/data_source/server/client/client_config.ts index 5de71fe5dc24..955ce3bacf5b 100644 --- a/src/plugins/data_source/server/client/client_config.ts +++ b/src/plugins/data_source/server/client/client_config.ts @@ -15,7 +15,8 @@ import { DataSourcePluginConfigType } from '../../config'; export function parseClientOptions( // TODO: will use client configs, that comes from a merge result of user config and default opensearch client config, config: DataSourcePluginConfigType, - endpoint: string + endpoint: string, + registeredSchema: any[] ): ClientOptions { const clientOptions: ClientOptions = { node: endpoint, @@ -23,6 +24,7 @@ export function parseClientOptions( requestCert: true, rejectUnauthorized: true, }, + plugins: registeredSchema, }; return clientOptions; diff --git a/src/plugins/data_source/server/client/configure_client.test.ts b/src/plugins/data_source/server/client/configure_client.test.ts index 32aacb5ead72..fa4b820a912e 100644 --- a/src/plugins/data_source/server/client/configure_client.test.ts +++ b/src/plugins/data_source/server/client/configure_client.test.ts @@ -22,6 +22,7 @@ import { opensearchClientMock } from '../../../../core/server/opensearch/client/ import { cryptographyServiceSetupMock } from '../cryptography_service.mocks'; import { CryptographyServiceSetup } from '../cryptography_service'; import { DataSourceClientParams } from '../types'; +import { CustomApiSchemaRegistry } from '../schema_registry'; const DATA_SOURCE_ID = 'a54b76ec86771ee865a0f74a305dfff8'; @@ -38,12 +39,14 @@ describe('configureClient', () => { let dataSourceClientParams: DataSourceClientParams; let usernamePasswordAuthContent: UsernamePasswordTypedContent; let sigV4AuthContent: SigV4Content; + let customApiSchemaRegistry: CustomApiSchemaRegistry; beforeEach(() => { dsClient = opensearchClientMock.createInternalClient(); logger = loggingSystemMock.createLogger(); savedObjectsMock = savedObjectsClientMock.create(); cryptographyMock = cryptographyServiceSetupMock.create(); + customApiSchemaRegistry = new CustomApiSchemaRegistry(); config = { enabled: true, @@ -95,6 +98,7 @@ describe('configureClient', () => { dataSourceId: DATA_SOURCE_ID, savedObjects: savedObjectsMock, cryptography: cryptographyMock, + customApiSchemaRegistryPromise: Promise.resolve(customApiSchemaRegistry), }; ClientMock.mockImplementation(() => dsClient); diff --git a/src/plugins/data_source/server/client/configure_client.ts b/src/plugins/data_source/server/client/configure_client.ts index 8cc835980c2a..6083a8527708 100644 --- a/src/plugins/data_source/server/client/configure_client.ts +++ b/src/plugins/data_source/server/client/configure_client.ts @@ -29,7 +29,13 @@ import { } from './configure_client_utils'; export const configureClient = async ( - { dataSourceId, savedObjects, cryptography, testClientDataSourceAttr }: DataSourceClientParams, + { + dataSourceId, + savedObjects, + cryptography, + testClientDataSourceAttr, + customApiSchemaRegistryPromise, + }: DataSourceClientParams, openSearchClientPoolSetup: OpenSearchClientPoolSetup, config: DataSourcePluginConfigType, logger: Logger @@ -64,10 +70,13 @@ export const configureClient = async ( dataSourceId ) as Client; + const registeredSchema = (await customApiSchemaRegistryPromise).getAll(); + return await getQueryClient( dataSource, openSearchClientPoolSetup.addClientToPool, config, + registeredSchema, cryptography, rootClient, dataSourceId, @@ -87,6 +96,7 @@ export const configureClient = async ( * * @param rootClient root client for the given data source. * @param dataSourceAttr data source saved object attributes + * @param registeredSchema registered API schema * @param cryptography cryptography service for password encryption / decryption * @param config data source config * @param addClientToPool function to add client to client pool @@ -98,6 +108,7 @@ const getQueryClient = async ( dataSourceAttr: DataSourceAttributes, addClientToPool: (endpoint: string, authType: AuthType, client: Client | LegacyClient) => void, config: DataSourcePluginConfigType, + registeredSchema: any[], cryptography?: CryptographyServiceSetup, rootClient?: Client, dataSourceId?: string, @@ -107,7 +118,7 @@ const getQueryClient = async ( auth: { type }, endpoint, } = dataSourceAttr; - const clientOptions = parseClientOptions(config, endpoint); + const clientOptions = parseClientOptions(config, endpoint, registeredSchema); const cacheKey = generateCacheKey(dataSourceAttr, dataSourceId); switch (type) { diff --git a/src/plugins/data_source/server/legacy/client_config.ts b/src/plugins/data_source/server/legacy/client_config.ts index d9b1cc704e3a..eed052cf245d 100644 --- a/src/plugins/data_source/server/legacy/client_config.ts +++ b/src/plugins/data_source/server/legacy/client_config.ts @@ -15,13 +15,15 @@ import { DataSourcePluginConfigType } from '../../config'; export function parseClientOptions( // TODO: will use client configs, that comes from a merge result of user config and default legacy client config, config: DataSourcePluginConfigType, - endpoint: string + endpoint: string, + registeredSchema: any[] ): ConfigOptions { const configOptions: ConfigOptions = { host: endpoint, ssl: { rejectUnauthorized: true, }, + plugins: registeredSchema, }; return configOptions; diff --git a/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts b/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts index 59c110d06dc5..f5cae1307f5a 100644 --- a/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts +++ b/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts @@ -15,6 +15,7 @@ import { OpenSearchClientPoolSetup } from '../client'; import { ConfigOptions } from 'elasticsearch'; import { ClientMock, parseClientOptionsMock } from './configure_legacy_client.test.mocks'; import { configureLegacyClient } from './configure_legacy_client'; +import { CustomApiSchemaRegistry } from '../schema_registry'; const DATA_SOURCE_ID = 'a54b76ec86771ee865a0f74a305dfff8'; @@ -35,6 +36,7 @@ describe('configureLegacyClient', () => { }; let dataSourceClientParams: DataSourceClientParams; let callApiParams: LegacyClientCallAPIParams; + const customApiSchemaRegistry = new CustomApiSchemaRegistry(); const mockResponse = { data: 'ping' }; @@ -98,6 +100,7 @@ describe('configureLegacyClient', () => { dataSourceId: DATA_SOURCE_ID, savedObjects: savedObjectsMock, cryptography: cryptographyMock, + customApiSchemaRegistryPromise: Promise.resolve(customApiSchemaRegistry), }; ClientMock.mockImplementation(() => mockOpenSearchClientInstance); diff --git a/src/plugins/data_source/server/legacy/configure_legacy_client.ts b/src/plugins/data_source/server/legacy/configure_legacy_client.ts index 8c02233950ee..1a7685b3b540 100644 --- a/src/plugins/data_source/server/legacy/configure_legacy_client.ts +++ b/src/plugins/data_source/server/legacy/configure_legacy_client.ts @@ -36,7 +36,12 @@ import { } from '../client/configure_client_utils'; export const configureLegacyClient = async ( - { dataSourceId, savedObjects, cryptography }: DataSourceClientParams, + { + dataSourceId, + savedObjects, + cryptography, + customApiSchemaRegistryPromise, + }: DataSourceClientParams, callApiParams: LegacyClientCallAPIParams, openSearchClientPoolSetup: OpenSearchClientPoolSetup, config: DataSourcePluginConfigType, @@ -50,12 +55,15 @@ export const configureLegacyClient = async ( dataSourceId ) as LegacyClient; + const registeredSchema = (await customApiSchemaRegistryPromise).getAll(); + return await getQueryClient( dataSourceAttr, cryptography, callApiParams, openSearchClientPoolSetup.addClientToPool, config, + registeredSchema, rootClient, dataSourceId ); @@ -75,6 +83,7 @@ export const configureLegacyClient = async ( * @param dataSourceAttr data source saved object attributes * @param cryptography cryptography service for password encryption / decryption * @param config data source config + * @param registeredSchema registered API schema * @param addClientToPool function to add client to client pool * @param dataSourceId id of data source saved Object * @returns child client. @@ -85,6 +94,7 @@ const getQueryClient = async ( { endpoint, clientParams, options }: LegacyClientCallAPIParams, addClientToPool: (endpoint: string, authType: AuthType, client: Client | LegacyClient) => void, config: DataSourcePluginConfigType, + registeredSchema: any[], rootClient?: LegacyClient, dataSourceId?: string ) => { @@ -92,7 +102,7 @@ const getQueryClient = async ( auth: { type }, endpoint: nodeUrl, } = dataSourceAttr; - const clientOptions = parseClientOptions(config, nodeUrl); + const clientOptions = parseClientOptions(config, nodeUrl, registeredSchema); const cacheKey = generateCacheKey(dataSourceAttr, dataSourceId); switch (type) { diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 5eaafc290000..6bccfbfad662 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -31,6 +31,7 @@ import { ensureRawRequest } from '../../../../src/core/server/http/router'; import { createDataSourceError } from './lib/error'; import { registerTestConnectionRoute } from './routes/test_connection'; import { AuthenticationMethodRegistery, IAuthenticationMethodRegistery } from './auth_registry'; +import { CustomApiSchemaRegistry } from './schema_registry'; export class DataSourcePlugin implements Plugin { private readonly logger: Logger; @@ -39,6 +40,7 @@ export class DataSourcePlugin implements Plugin; private started = false; private authMethodsRegistry = new AuthenticationMethodRegistery(); + private customApiSchemaRegistry = new CustomApiSchemaRegistry(); constructor(private initializerContext: PluginInitializerContext) { this.logger = this.initializerContext.logger.get(); @@ -104,6 +106,11 @@ export class DataSourcePlugin implements Plugin { + const dataSourcePluginStart = selfStart as DataSourcePluginStart; + return dataSourcePluginStart.getCustomApiSchemaRegistry(); + }); + // Register data source plugin context to route handler context core.http.registerRouteHandlerContext( 'dataSource', @@ -112,7 +119,8 @@ export class DataSourcePlugin implements Plugin createDataSourceError(e), registerCredentialProvider, + registerCustomApiSchema: (schema: any) => this.customApiSchemaRegistry.register(schema), }; } @@ -143,6 +152,7 @@ export class DataSourcePlugin implements Plugin this.authMethodsRegistry, + getCustomApiSchemaRegistry: () => this.customApiSchemaRegistry, }; } @@ -155,7 +165,8 @@ export class DataSourcePlugin implements Plugin, - authRegistryPromise: Promise + authRegistryPromise: Promise, + customApiSchemaRegistryPromise: Promise ): IContextProvider, 'dataSource'> => { return (context, req) => { return { @@ -169,6 +180,7 @@ export class DataSourcePlugin implements Plugin { + let registry: CustomApiSchemaRegistry; + + beforeEach(() => { + registry = new CustomApiSchemaRegistry(); + }); + + it('allows to register and get api schema', () => { + const sqlPlugin = () => {}; + registry.register(sqlPlugin); + expect(registry.getAll()).toEqual([sqlPlugin]); + }); +}); diff --git a/src/plugins/data_source/server/schema_registry/custom_api_schema_registry.ts b/src/plugins/data_source/server/schema_registry/custom_api_schema_registry.ts new file mode 100644 index 000000000000..ef9cd3b1e633 --- /dev/null +++ b/src/plugins/data_source/server/schema_registry/custom_api_schema_registry.ts @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +export class CustomApiSchemaRegistry { + private readonly schemaRegistry: any[]; + + constructor() { + this.schemaRegistry = new Array(); + } + + public register(schema: any) { + this.schemaRegistry.push(schema); + } + + public getAll(): any[] { + return this.schemaRegistry; + } +} diff --git a/src/plugins/data_source/server/schema_registry/index.ts b/src/plugins/data_source/server/schema_registry/index.ts new file mode 100644 index 000000000000..67a60bc90433 --- /dev/null +++ b/src/plugins/data_source/server/schema_registry/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { CustomApiSchemaRegistry } from './custom_api_schema_registry'; diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index 146a28eb8cfa..9bd70b142d8b 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -19,6 +19,7 @@ import { import { CryptographyServiceSetup } from './cryptography_service'; import { DataSourceError } from './lib/error'; import { IAuthenticationMethodRegistery } from './auth_registry'; +import { CustomApiSchemaRegistry } from './schema_registry'; export interface LegacyClientCallAPIParams { endpoint: string; @@ -34,6 +35,8 @@ export interface DataSourceClientParams { dataSourceId?: string; // required when creating test client testClientDataSourceAttr?: DataSourceAttributes; + // custom API schema registry promise, required for getting registered custom API schema + customApiSchemaRegistryPromise: Promise; } export interface DataSourceCredentialsProviderOptions { @@ -77,8 +80,10 @@ declare module 'src/core/server' { export interface DataSourcePluginSetup { createDataSourceError: (err: any) => DataSourceError; registerCredentialProvider: (method: AuthenticationMethod) => void; + registerCustomApiSchema: (schema: any) => void; } export interface DataSourcePluginStart { getAuthenticationMethodRegistery: () => IAuthenticationMethodRegistery; + getCustomApiSchemaRegistry: () => CustomApiSchemaRegistry; }