From 282d8acbcdb2b483d406fdbee6a83be4e411887b Mon Sep 17 00:00:00 2001 From: Yulia Cech Date: Tue, 15 Feb 2022 14:08:17 +0100 Subject: [PATCH 1/2] [Snapshot & Restore] Fix repository types on Cloud --- .../snapshot_restore/common/constants.ts | 9 +++-- .../server/routes/api/repositories.test.ts | 22 ++++++++---- .../server/routes/api/repositories.ts | 13 +++++-- .../apis/management/snapshot_restore/index.ts | 1 + .../snapshot_restore/repositories.ts | 34 +++++++++++++++++++ 5 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts diff --git a/x-pack/plugins/snapshot_restore/common/constants.ts b/x-pack/plugins/snapshot_restore/common/constants.ts index c4b64bb9395f8..d8b159fd01e96 100644 --- a/x-pack/plugins/snapshot_restore/common/constants.ts +++ b/x-pack/plugins/snapshot_restore/common/constants.ts @@ -35,12 +35,15 @@ export enum REPOSITORY_TYPES { } // Deliberately do not include `source` as a default repository since we treat it as a flag -export const DEFAULT_REPOSITORY_TYPES: RepositoryType[] = [ +export const ON_PREM_REPOSITORY_TYPES: RepositoryType[] = [ + REPOSITORY_TYPES.fs, + REPOSITORY_TYPES.url, +]; + +export const MODULE_REPOSITORY_TYPES: RepositoryType[] = [ REPOSITORY_TYPES.azure, REPOSITORY_TYPES.gcs, REPOSITORY_TYPES.s3, - REPOSITORY_TYPES.fs, - REPOSITORY_TYPES.url, ]; export const PLUGIN_REPOSITORY_TYPES: RepositoryType[] = [REPOSITORY_TYPES.hdfs]; diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts index 844be90f9b842..91e42a2b919cf 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { DEFAULT_REPOSITORY_TYPES, REPOSITORY_PLUGINS_MAP } from '../../../common'; +import { + ON_PREM_REPOSITORY_TYPES, + MODULE_REPOSITORY_TYPES, + REPOSITORY_PLUGINS_MAP, +} from '../../../common'; import { addBasePath } from '../helpers'; import { registerRepositoriesRoutes } from './repositories'; import { RouterMock, routeDependencies, RequestMock } from '../../test/helpers'; @@ -253,14 +257,14 @@ describe('[Snapshot and Restore API Routes] Repositories', () => { path: addBasePath('repository_types'), }; - it('returns default types if no repository plugins returned from ES', async () => { + it('returns module types and on-prem types if no repository plugins returned from ES', async () => { nodesInfoFn.mockResolvedValue({ nodes: { testNodeId: { plugins: [] } } }); - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES]; + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); }); - it('returns default types with any repository plugins returned from ES', async () => { + it('returns module types and on-prem types with any repository plugins returned from ES', async () => { const pluginNames = Object.keys(REPOSITORY_PLUGINS_MAP); const pluginTypes = Object.entries(REPOSITORY_PLUGINS_MAP).map(([key, value]) => value); @@ -269,7 +273,11 @@ describe('[Snapshot and Restore API Routes] Repositories', () => { }; nodesInfoFn.mockResolvedValue(mockEsResponse); - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES, ...pluginTypes]; + const expectedResponse = [ + ...MODULE_REPOSITORY_TYPES, + ...ON_PREM_REPOSITORY_TYPES, + ...pluginTypes, + ]; await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); }); @@ -280,7 +288,7 @@ describe('[Snapshot and Restore API Routes] Repositories', () => { }; nodesInfoFn.mockResolvedValue(mockEsResponse); - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES]; + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); }); @@ -296,7 +304,7 @@ describe('[Snapshot and Restore API Routes] Repositories', () => { }; nodesInfoFn.mockResolvedValue(mockEsResponse); - const expectedResponse = [...DEFAULT_REPOSITORY_TYPES]; + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); }); diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts index 22c4d4a2b3e91..4666870133f1f 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts @@ -12,7 +12,11 @@ import type { PluginStats, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { DEFAULT_REPOSITORY_TYPES, REPOSITORY_PLUGINS_MAP } from '../../../common'; +import { + ON_PREM_REPOSITORY_TYPES, + REPOSITORY_PLUGINS_MAP, + MODULE_REPOSITORY_TYPES, +} from '../../../common'; import { Repository, RepositoryType } from '../../../common/types'; import { RouteDependencies } from '../../types'; import { addBasePath } from '../helpers'; @@ -154,8 +158,11 @@ export function registerRepositoriesRoutes({ { path: addBasePath('repository_types'), validate: false }, license.guardApiRoute(async (ctx, req, res) => { const { client: clusterClient } = ctx.core.elasticsearch; - // In ECE/ESS, do not enable the default types - const types: RepositoryType[] = isCloudEnabled ? [] : [...DEFAULT_REPOSITORY_TYPES]; + // module repo types are available everywhere out of the box + // on-prem repo types are not available on Cloud + const types: RepositoryType[] = isCloudEnabled + ? [...MODULE_REPOSITORY_TYPES] + : [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; try { const { nodes } = await clusterClient.asCurrentUser.nodes.info({ diff --git a/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts b/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts index db5dbc9735e66..debfb683cd883 100644 --- a/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts +++ b/x-pack/test/api_integration/apis/management/snapshot_restore/index.ts @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('Snapshot and Restore', () => { loadTestFile(require.resolve('./policies')); loadTestFile(require.resolve('./snapshots')); + loadTestFile(require.resolve('./repositories')); }); } diff --git a/x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts b/x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts new file mode 100644 index 0000000000000..982f32faf73ce --- /dev/null +++ b/x-pack/test/api_integration/apis/management/snapshot_restore/repositories.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +const API_BASE_PATH = '/api/snapshot_restore'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const deployment = getService('deployment'); + + describe('snapshot repositories', function () { + describe('repository types', () => { + it('returns a list of default repository types', async () => { + const { body } = await supertest.get(`${API_BASE_PATH}/repository_types`).expect(200); + + const isCloud = await deployment.isCloud(); + if (isCloud) { + // on Cloud there are only module repo types + expect(body).to.eql(['azure', 'gcs', 's3']); + } else { + // on prem there are module repo types and file system and url repo types + expect(body).to.eql(['azure', 'gcs', 's3', 'fs', 'url']); + } + }); + }); + }); +} From ec6c256a00fa55220f445950b4208a67531a8086 Mon Sep 17 00:00:00 2001 From: Yulia Cech Date: Wed, 16 Feb 2022 16:16:01 +0100 Subject: [PATCH 2/2] [Snapshot & Restore] Move repo types tests into `on-prem` describe block --- .../server/routes/api/repositories.test.ts | 99 ++++++++++--------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts index 91e42a2b919cf..f29a84beaecf8 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts @@ -257,56 +257,59 @@ describe('[Snapshot and Restore API Routes] Repositories', () => { path: addBasePath('repository_types'), }; - it('returns module types and on-prem types if no repository plugins returned from ES', async () => { - nodesInfoFn.mockResolvedValue({ nodes: { testNodeId: { plugins: [] } } }); - - const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); - }); - - it('returns module types and on-prem types with any repository plugins returned from ES', async () => { - const pluginNames = Object.keys(REPOSITORY_PLUGINS_MAP); - const pluginTypes = Object.entries(REPOSITORY_PLUGINS_MAP).map(([key, value]) => value); - - const mockEsResponse = { - nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, - }; - nodesInfoFn.mockResolvedValue(mockEsResponse); - - const expectedResponse = [ - ...MODULE_REPOSITORY_TYPES, - ...ON_PREM_REPOSITORY_TYPES, - ...pluginTypes, - ]; - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); - }); - - it(`doesn't return non-repository plugins returned from ES`, async () => { - const pluginNames = ['foo-plugin', 'bar-plugin']; - const mockEsResponse = { - nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, - }; - nodesInfoFn.mockResolvedValue(mockEsResponse); - - const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; - - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); - }); - - it(`doesn't return repository plugins that are not installed on all nodes`, async () => { - const dataNodePlugins = ['repository-hdfs']; - const masterNodePlugins: string[] = []; - const mockEsResponse = { - nodes: { - dataNode: { plugins: [...dataNodePlugins.map((key) => ({ name: key }))] }, - masterNode: { plugins: [...masterNodePlugins.map((key) => ({ name: key }))] }, - }, - }; - nodesInfoFn.mockResolvedValue(mockEsResponse); + // TODO add Cloud specific tests for repo types + describe('on prem', () => { + it('returns module types and on-prem types if no repository plugins returned from ES', async () => { + nodesInfoFn.mockResolvedValue({ nodes: { testNodeId: { plugins: [] } } }); + + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); + + it('returns module types and on-prem types with any repository plugins returned from ES', async () => { + const pluginNames = Object.keys(REPOSITORY_PLUGINS_MAP); + const pluginTypes = Object.entries(REPOSITORY_PLUGINS_MAP).map(([key, value]) => value); + + const mockEsResponse = { + nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, + }; + nodesInfoFn.mockResolvedValue(mockEsResponse); + + const expectedResponse = [ + ...MODULE_REPOSITORY_TYPES, + ...ON_PREM_REPOSITORY_TYPES, + ...pluginTypes, + ]; + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); + + it(`doesn't return non-repository plugins returned from ES`, async () => { + const pluginNames = ['foo-plugin', 'bar-plugin']; + const mockEsResponse = { + nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } }, + }; + nodesInfoFn.mockResolvedValue(mockEsResponse); + + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; + + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); + + it(`doesn't return repository plugins that are not installed on all nodes`, async () => { + const dataNodePlugins = ['repository-hdfs']; + const masterNodePlugins: string[] = []; + const mockEsResponse = { + nodes: { + dataNode: { plugins: [...dataNodePlugins.map((key) => ({ name: key }))] }, + masterNode: { plugins: [...masterNodePlugins.map((key) => ({ name: key }))] }, + }, + }; + nodesInfoFn.mockResolvedValue(mockEsResponse); - const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; + const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES]; - await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); + }); }); it('should throw if ES error', async () => {