Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Snapshot & Restore] Fix repository types on Cloud #125656

Merged
merged 3 commits into from
Feb 16, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions x-pack/plugins/snapshot_restore/common/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,15 @@ export enum REPOSITORY_TYPES {
}

// Deliberately do not include `source` as a default repository since we treat it as a flag
export const DEFAULT_REPOSITORY_TYPES: RepositoryType[] = [
export const ON_PREM_REPOSITORY_TYPES: RepositoryType[] = [
REPOSITORY_TYPES.fs,
REPOSITORY_TYPES.url,
];

export const MODULE_REPOSITORY_TYPES: RepositoryType[] = [
REPOSITORY_TYPES.azure,
REPOSITORY_TYPES.gcs,
REPOSITORY_TYPES.s3,
REPOSITORY_TYPES.fs,
REPOSITORY_TYPES.url,
];

export const PLUGIN_REPOSITORY_TYPES: RepositoryType[] = [REPOSITORY_TYPES.hdfs];
Expand Down
101 changes: 56 additions & 45 deletions x-pack/plugins/snapshot_restore/server/routes/api/repositories.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@
* 2.0.
*/

import { DEFAULT_REPOSITORY_TYPES, REPOSITORY_PLUGINS_MAP } from '../../../common';
import {
ON_PREM_REPOSITORY_TYPES,
MODULE_REPOSITORY_TYPES,
REPOSITORY_PLUGINS_MAP,
} from '../../../common';
import { addBasePath } from '../helpers';
import { registerRepositoriesRoutes } from './repositories';
import { RouterMock, routeDependencies, RequestMock } from '../../test/helpers';
Expand Down Expand Up @@ -253,52 +257,59 @@ describe('[Snapshot and Restore API Routes] Repositories', () => {
path: addBasePath('repository_types'),
};

it('returns default types if no repository plugins returned from ES', async () => {
nodesInfoFn.mockResolvedValue({ nodes: { testNodeId: { plugins: [] } } });

const expectedResponse = [...DEFAULT_REPOSITORY_TYPES];
await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
});

it('returns default types with any repository plugins returned from ES', async () => {
const pluginNames = Object.keys(REPOSITORY_PLUGINS_MAP);
const pluginTypes = Object.entries(REPOSITORY_PLUGINS_MAP).map(([key, value]) => value);

const mockEsResponse = {
nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } },
};
nodesInfoFn.mockResolvedValue(mockEsResponse);

const expectedResponse = [...DEFAULT_REPOSITORY_TYPES, ...pluginTypes];
await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
});

it(`doesn't return non-repository plugins returned from ES`, async () => {
const pluginNames = ['foo-plugin', 'bar-plugin'];
const mockEsResponse = {
nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } },
};
nodesInfoFn.mockResolvedValue(mockEsResponse);

const expectedResponse = [...DEFAULT_REPOSITORY_TYPES];

await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
});

it(`doesn't return repository plugins that are not installed on all nodes`, async () => {
const dataNodePlugins = ['repository-hdfs'];
const masterNodePlugins: string[] = [];
const mockEsResponse = {
nodes: {
dataNode: { plugins: [...dataNodePlugins.map((key) => ({ name: key }))] },
masterNode: { plugins: [...masterNodePlugins.map((key) => ({ name: key }))] },
},
};
nodesInfoFn.mockResolvedValue(mockEsResponse);
// TODO add Cloud specific tests for repo types
describe('on prem', () => {
it('returns module types and on-prem types if no repository plugins returned from ES', async () => {
nodesInfoFn.mockResolvedValue({ nodes: { testNodeId: { plugins: [] } } });

const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES];
await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
});

it('returns module types and on-prem types with any repository plugins returned from ES', async () => {
const pluginNames = Object.keys(REPOSITORY_PLUGINS_MAP);
const pluginTypes = Object.entries(REPOSITORY_PLUGINS_MAP).map(([key, value]) => value);

const mockEsResponse = {
nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } },
};
nodesInfoFn.mockResolvedValue(mockEsResponse);

const expectedResponse = [
...MODULE_REPOSITORY_TYPES,
...ON_PREM_REPOSITORY_TYPES,
...pluginTypes,
];
await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
});

it(`doesn't return non-repository plugins returned from ES`, async () => {
const pluginNames = ['foo-plugin', 'bar-plugin'];
const mockEsResponse = {
nodes: { testNodeId: { plugins: [...pluginNames.map((key) => ({ name: key }))] } },
};
nodesInfoFn.mockResolvedValue(mockEsResponse);

const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES];

await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
});

it(`doesn't return repository plugins that are not installed on all nodes`, async () => {
const dataNodePlugins = ['repository-hdfs'];
const masterNodePlugins: string[] = [];
const mockEsResponse = {
nodes: {
dataNode: { plugins: [...dataNodePlugins.map((key) => ({ name: key }))] },
masterNode: { plugins: [...masterNodePlugins.map((key) => ({ name: key }))] },
},
};
nodesInfoFn.mockResolvedValue(mockEsResponse);

const expectedResponse = [...DEFAULT_REPOSITORY_TYPES];
const expectedResponse = [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES];

await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse });
});
});

it('should throw if ES error', async () => {
Expand Down
13 changes: 10 additions & 3 deletions x-pack/plugins/snapshot_restore/server/routes/api/repositories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ import type {
PluginStats,
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';

import { DEFAULT_REPOSITORY_TYPES, REPOSITORY_PLUGINS_MAP } from '../../../common';
import {
ON_PREM_REPOSITORY_TYPES,
REPOSITORY_PLUGINS_MAP,
MODULE_REPOSITORY_TYPES,
} from '../../../common';
import { Repository, RepositoryType } from '../../../common/types';
import { RouteDependencies } from '../../types';
import { addBasePath } from '../helpers';
Expand Down Expand Up @@ -154,8 +158,11 @@ export function registerRepositoriesRoutes({
{ path: addBasePath('repository_types'), validate: false },
license.guardApiRoute(async (ctx, req, res) => {
const { client: clusterClient } = ctx.core.elasticsearch;
// In ECE/ESS, do not enable the default types
const types: RepositoryType[] = isCloudEnabled ? [] : [...DEFAULT_REPOSITORY_TYPES];
// module repo types are available everywhere out of the box
// on-prem repo types are not available on Cloud
const types: RepositoryType[] = isCloudEnabled
? [...MODULE_REPOSITORY_TYPES]
: [...MODULE_REPOSITORY_TYPES, ...ON_PREM_REPOSITORY_TYPES];

try {
const { nodes } = await clusterClient.asCurrentUser.nodes.info({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
describe('Snapshot and Restore', () => {
loadTestFile(require.resolve('./policies'));
loadTestFile(require.resolve('./snapshots'));
loadTestFile(require.resolve('./repositories'));
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from '@kbn/expect';

import { FtrProviderContext } from '../../../ftr_provider_context';

const API_BASE_PATH = '/api/snapshot_restore';

export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const deployment = getService('deployment');

describe('snapshot repositories', function () {
describe('repository types', () => {
it('returns a list of default repository types', async () => {
const { body } = await supertest.get(`${API_BASE_PATH}/repository_types`).expect(200);

const isCloud = await deployment.isCloud();
if (isCloud) {
// on Cloud there are only module repo types
expect(body).to.eql(['azure', 'gcs', 's3']);
} else {
// on prem there are module repo types and file system and url repo types
expect(body).to.eql(['azure', 'gcs', 's3', 'fs', 'url']);
}
});
});
});
}