From f5191760dd9f5de4f5df6ee3538e87608cdff196 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 17 Jul 2020 18:44:57 -0400 Subject: [PATCH] [Security Solution][Endpoint][Exceptions] Only write manifest to policy when there are changes (#72000) (#72344) * Refactor security_solution policy creation callback - part 1 * Fix manifest dispatch * Change how dispatches are performed * simplify manifest types * Remove unused mock * Fix tests * one place to construct artifact ids * fixing linter exceptions * Add tests for stable hashes * Additional testing and type cleanup * Remove unnecessary log * Minor fixup * jsdoc * type fixup * Additional type adjustments --- x-pack/plugins/ingest_manager/common/mocks.ts | 87 ++++++ .../common/endpoint/schema/common.ts | 5 + .../common/endpoint/schema/manifest.ts | 46 ++- .../endpoint/ingest_integration.test.ts | 105 ++++--- .../server/endpoint/ingest_integration.ts | 156 +++++----- .../server/endpoint/lib/artifacts/common.ts | 16 +- .../endpoint/lib/artifacts/lists.test.ts | 94 +++++- .../server/endpoint/lib/artifacts/lists.ts | 33 +- .../endpoint/lib/artifacts/manifest.test.ts | 158 +++++----- .../server/endpoint/lib/artifacts/manifest.ts | 97 +++++- .../lib/artifacts/manifest_entry.test.ts | 28 +- .../endpoint/lib/artifacts/manifest_entry.ts | 3 +- .../server/endpoint/lib/artifacts/mocks.ts | 68 +++++ .../server/endpoint/lib/artifacts/task.ts | 84 +++-- .../server/endpoint/mocks.ts | 25 +- .../artifacts/download_exception_list.ts | 6 +- .../endpoint/schemas/artifacts/lists.mock.ts | 4 +- .../schemas/artifacts/saved_objects.mock.ts | 54 ++-- .../schemas/artifacts/saved_objects.ts | 41 ++- .../artifacts/artifact_client.test.ts | 9 +- .../services/artifacts/artifact_client.ts | 28 +- .../artifacts/manifest_client.test.ts | 5 +- .../services/artifacts/manifest_client.ts | 9 +- .../manifest_manager/manifest_manager.mock.ts | 81 ++--- .../manifest_manager/manifest_manager.test.ts | 209 +++++++++---- .../manifest_manager/manifest_manager.ts | 286 ++++++++---------- 26 files changed, 1154 insertions(+), 583 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts diff --git a/x-pack/plugins/ingest_manager/common/mocks.ts b/x-pack/plugins/ingest_manager/common/mocks.ts index e85364f2bb672..236324b11c580 100644 --- a/x-pack/plugins/ingest_manager/common/mocks.ts +++ b/x-pack/plugins/ingest_manager/common/mocks.ts @@ -44,3 +44,90 @@ export const createPackageConfigMock = (): PackageConfig => { ], }; }; + +export const createPackageConfigWithInitialManifestMock = (): PackageConfig => { + const packageConfig = createPackageConfigMock(); + packageConfig.inputs[0].config!.artifact_manifest = { + value: { + artifacts: { + 'endpoint-exceptionlist-linux-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + }, + manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + schema_version: 'v1', + }, + }; + return packageConfig; +}; + +export const createPackageConfigWithManifestMock = (): PackageConfig => { + const packageConfig = createPackageConfigMock(); + packageConfig.inputs[0].config!.artifact_manifest = { + value: { + artifacts: { + 'endpoint-exceptionlist-linux-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + encoded_sha256: '57941169bb2c5416f9bd7224776c8462cb9a2be0fe8b87e6213e77a1d29be824', + decoded_size: 292, + encoded_size: 131, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + }, + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + }, + }, + manifest_version: '520f6cf88b3f36a065c6ca81058d5f8690aadadf6fe857f8dec4cc37589e7283', + schema_version: 'v1', + }, + }; + + return packageConfig; +}; diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts index 014673ebe6398..8f2ea1f8a6452 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -12,6 +12,11 @@ export const compressionAlgorithm = t.keyof({ }); export type CompressionAlgorithm = t.TypeOf; +export const compressionAlgorithmDispatch = t.keyof({ + zlib: null, +}); +export type CompressionAlgorithmDispatch = t.TypeOf; + export const encryptionAlgorithm = t.keyof({ none: null, }); diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts index 1c8916dfdd5bb..f8bb8b70f2d5b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -7,6 +7,7 @@ import * as t from 'io-ts'; import { compressionAlgorithm, + compressionAlgorithmDispatch, encryptionAlgorithm, identifier, manifestSchemaVersion, @@ -16,25 +17,60 @@ import { size, } from './common'; -export const manifestEntrySchema = t.exact( +export const manifestEntryBaseSchema = t.exact( t.type({ relative_url: relativeUrl, decoded_sha256: sha256, decoded_size: size, encoded_sha256: sha256, encoded_size: size, - compression_algorithm: compressionAlgorithm, encryption_algorithm: encryptionAlgorithm, }) ); -export const manifestSchema = t.exact( +export const manifestEntrySchema = t.intersection([ + manifestEntryBaseSchema, + t.exact( + t.type({ + compression_algorithm: compressionAlgorithm, + }) + ), +]); +export type ManifestEntrySchema = t.TypeOf; + +export const manifestEntryDispatchSchema = t.intersection([ + manifestEntryBaseSchema, + t.exact( + t.type({ + compression_algorithm: compressionAlgorithmDispatch, + }) + ), +]); +export type ManifestEntryDispatchSchema = t.TypeOf; + +export const manifestBaseSchema = t.exact( t.type({ manifest_version: manifestVersion, schema_version: manifestSchemaVersion, - artifacts: t.record(identifier, manifestEntrySchema), }) ); -export type ManifestEntrySchema = t.TypeOf; +export const manifestSchema = t.intersection([ + manifestBaseSchema, + t.exact( + t.type({ + artifacts: t.record(identifier, manifestEntrySchema), + }) + ), +]); export type ManifestSchema = t.TypeOf; + +export const manifestDispatchSchema = t.intersection([ + manifestBaseSchema, + t.exact( + t.type({ + artifacts: t.record(identifier, manifestEntryDispatchSchema), + }) + ), +]); +export type ManifestDispatchSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts index bb035a19f33d6..be749b2ebd25a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts @@ -4,87 +4,122 @@ * you may not use this file except in compliance with the Elastic License. */ -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { loggerMock } from 'src/core/server/logging/logger.mock'; +import { loggingSystemMock } from 'src/core/server/mocks'; import { createNewPackageConfigMock } from '../../../ingest_manager/common/mocks'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; -import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { + getManifestManagerMock, + ManifestManagerMockType, +} from './services/artifacts/manifest_manager/manifest_manager.mock'; import { getPackageConfigCreateCallback } from './ingest_integration'; +import { ManifestConstants } from './lib/artifacts'; describe('ingest_integration tests ', () => { describe('ingest_integration sanity checks', () => { - test('policy is updated with manifest', async () => { - const logger = loggerMock.create(); - const manifestManager = getManifestManagerMock(); + test('policy is updated with initial manifest', async () => { + const logger = loggingSystemMock.create().get('ingest_integration.test'); + const manifestManager = getManifestManagerMock({ + mockType: ManifestManagerMockType.InitialSystemState, + }); + const callback = getPackageConfigCreateCallback(logger, manifestManager); - const policyConfig = createNewPackageConfigMock(); - const newPolicyConfig = await callback(policyConfig); + const policyConfig = createNewPackageConfigMock(); // policy config without manifest + const newPolicyConfig = await callback(policyConfig); // policy config WITH manifest + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({ artifacts: { 'endpoint-exceptionlist-linux-v1': { compression_algorithm: 'zlib', - decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - decoded_size: 287, - encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', - encoded_size: 133, + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + decoded_size: 14, + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + encoded_size: 22, + encryption_algorithm: 'none', + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + decoded_size: 14, + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + encoded_size: 22, + encryption_algorithm: 'none', + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + decoded_size: 14, + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + encoded_size: 22, encryption_algorithm: 'none', relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', }, }, - manifest_version: 'WzAsMF0=', + manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', schema_version: 'v1', }); }); - test('policy is returned even if error is encountered during artifact sync', async () => { - const logger = loggerMock.create(); + test('policy is returned even if error is encountered during artifact creation', async () => { + const logger = loggingSystemMock.create().get('ingest_integration.test'); const manifestManager = getManifestManagerMock(); - manifestManager.syncArtifacts = jest.fn().mockRejectedValue([new Error('error updating')]); - const lastDispatched = await manifestManager.getLastDispatchedManifest(); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error('error updating')]); + const lastComputed = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + const callback = getPackageConfigCreateCallback(logger, manifestManager); const policyConfig = createNewPackageConfigMock(); const newPolicyConfig = await callback(policyConfig); + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( - lastDispatched.toEndpointFormat() + lastComputed!.toEndpointFormat() ); }); - test('initial policy creation succeeds if snapshot retrieval fails', async () => { - const logger = loggerMock.create(); - const manifestManager = getManifestManagerMock(); - const lastDispatched = await manifestManager.getLastDispatchedManifest(); - manifestManager.getSnapshot = jest.fn().mockResolvedValue(null); + test('initial policy creation succeeds if manifest retrieval fails', async () => { + const logger = loggingSystemMock.create().get('ingest_integration.test'); + const manifestManager = getManifestManagerMock({ + mockType: ManifestManagerMockType.InitialSystemState, + }); + const lastComputed = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + expect(lastComputed).toEqual(null); + + manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error('abcd')); const callback = getPackageConfigCreateCallback(logger, manifestManager); const policyConfig = createNewPackageConfigMock(); const newPolicyConfig = await callback(policyConfig); + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); - expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( - lastDispatched.toEndpointFormat() - ); }); test('subsequent policy creations succeed', async () => { - const logger = loggerMock.create(); + const logger = loggingSystemMock.create().get('ingest_integration.test'); const manifestManager = getManifestManagerMock(); - const snapshot = await manifestManager.getSnapshot(); - manifestManager.getLastDispatchedManifest = jest.fn().mockResolvedValue(snapshot!.manifest); - manifestManager.getSnapshot = jest.fn().mockResolvedValue({ - manifest: snapshot!.manifest, - diffs: [], - }); + const lastComputed = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(lastComputed); // no diffs const callback = getPackageConfigCreateCallback(logger, manifestManager); const policyConfig = createNewPackageConfigMock(); const newPolicyConfig = await callback(policyConfig); + expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( - snapshot!.manifest.toEndpointFormat() + lastComputed!.toEndpointFormat() ); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index e2522ac4af778..11d4b12d0b76a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -8,9 +8,63 @@ import { Logger } from '../../../../../src/core/server'; import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; -import { ManifestManager, ManifestSnapshot } from './services/artifacts'; +import { ManifestManager } from './services/artifacts'; +import { Manifest } from './lib/artifacts'; import { reportErrors, ManifestConstants } from './lib/artifacts/common'; -import { ManifestSchemaVersion } from '../../common/endpoint/schema/common'; +import { InternalArtifactCompleteSchema } from './schemas/artifacts'; +import { manifestDispatchSchema } from '../../common/endpoint/schema/manifest'; + +const getManifest = async (logger: Logger, manifestManager: ManifestManager): Promise => { + let manifest: Manifest | null = null; + + try { + manifest = await manifestManager.getLastComputedManifest(ManifestConstants.SCHEMA_VERSION); + + // If we have not yet computed a manifest, then we have to do so now. This should only happen + // once. + if (manifest == null) { + // New computed manifest based on current state of exception list + const newManifest = await manifestManager.buildNewManifest(ManifestConstants.SCHEMA_VERSION); + const diffs = newManifest.diff(Manifest.getDefault(ManifestConstants.SCHEMA_VERSION)); + + // Compress new artifacts + const adds = diffs.filter((diff) => diff.type === 'add').map((diff) => diff.id); + for (const artifactId of adds) { + const compressError = await newManifest.compressArtifact(artifactId); + if (compressError) { + throw compressError; + } + } + + // Persist new artifacts + const artifacts = adds + .map((artifactId) => newManifest.getArtifact(artifactId)) + .filter((artifact): artifact is InternalArtifactCompleteSchema => artifact !== undefined); + if (artifacts.length !== adds.length) { + throw new Error('Invalid artifact encountered.'); + } + const persistErrors = await manifestManager.pushArtifacts(artifacts); + if (persistErrors.length) { + reportErrors(logger, persistErrors); + throw new Error('Unable to persist new artifacts.'); + } + + // Commit the manifest state + if (diffs.length) { + const error = await manifestManager.commit(newManifest); + if (error) { + throw error; + } + } + + manifest = newManifest; + } + } catch (err) { + logger.error(err); + } + + return manifest ?? Manifest.getDefault(ManifestConstants.SCHEMA_VERSION); +}; /** * Callback to handle creation of PackageConfigs in Ingest Manager @@ -31,85 +85,37 @@ export const getPackageConfigCreateCallback = ( // follow the types/schema expected let updatedPackageConfig = newPackageConfig as NewPolicyData; - // get current manifest from SO (last dispatched) - const manifest = ( - await manifestManager.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION) - )?.toEndpointFormat() ?? { - manifest_version: 'default', - schema_version: ManifestConstants.SCHEMA_VERSION as ManifestSchemaVersion, - artifacts: {}, - }; + // Get most recent manifest + const manifest = await getManifest(logger, manifestManager); + const serializedManifest = manifest.toEndpointFormat(); + if (!manifestDispatchSchema.is(serializedManifest)) { + // This should not happen. + // But if it does, we log it and return it anyway. + logger.error('Invalid manifest'); + } // Until we get the Default Policy Configuration in the Endpoint package, // we will add it here manually at creation time. - if (newPackageConfig.inputs.length === 0) { - updatedPackageConfig = { - ...newPackageConfig, - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - artifact_manifest: { - value: manifest, - }, - policy: { - value: policyConfigFactory(), - }, + updatedPackageConfig = { + ...newPackageConfig, + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + artifact_manifest: { + value: serializedManifest, + }, + policy: { + value: policyConfigFactory(), }, }, - ], - }; - } - - let snapshot: ManifestSnapshot | null = null; - let success = true; - try { - // Try to get most up-to-date manifest data. - - // get snapshot based on exception-list-agnostic SOs - // with diffs from last dispatched manifest, if it exists - snapshot = await manifestManager.getSnapshot({ initialize: true }); - - if (snapshot && snapshot.diffs.length) { - // create new artifacts - const errors = await manifestManager.syncArtifacts(snapshot, 'add'); - if (errors.length) { - reportErrors(logger, errors); - throw new Error('Error writing new artifacts.'); - } - } - - if (snapshot) { - updatedPackageConfig.inputs[0].config.artifact_manifest = { - value: snapshot.manifest.toEndpointFormat(), - }; - } - - return updatedPackageConfig; - } catch (err) { - success = false; - logger.error(err); - return updatedPackageConfig; - } finally { - if (success && snapshot !== null) { - try { - if (snapshot.diffs.length > 0) { - // TODO: let's revisit the way this callback happens... use promises? - // only commit when we know the package config was created - await manifestManager.commit(snapshot.manifest); + }, + ], + }; - // clean up old artifacts - await manifestManager.syncArtifacts(snapshot, 'delete'); - } - } catch (err) { - logger.error(err); - } - } else if (snapshot === null) { - logger.error('No manifest snapshot available.'); - } - } + return updatedPackageConfig; }; return handlePackageConfigCreate; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index 77a5e85b14199..7298a9bfa72a6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -4,6 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ import { Logger } from 'src/core/server'; +import { + InternalArtifactSchema, + InternalArtifactCompleteSchema, + internalArtifactCompleteSchema, +} from '../../schemas/artifacts'; export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', @@ -15,7 +20,16 @@ export const ArtifactConstants = { export const ManifestConstants = { SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest', SCHEMA_VERSION: 'v1', - INITIAL_VERSION: 'WzAsMF0=', +}; + +export const getArtifactId = (artifact: InternalArtifactSchema) => { + return `${artifact.identifier}-${artifact.decodedSha256}`; +}; + +export const isCompleteArtifact = ( + artifact: InternalArtifactSchema +): artifact is InternalArtifactCompleteSchema => { + return internalArtifactCompleteSchema.is(artifact); }; export const reportErrors = (logger: Logger, errors: Error[]) => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 1a19306b2fd60..d3d073efa73c1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -9,7 +9,8 @@ import { listMock } from '../../../../../lists/server/mocks'; import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { EntriesArray, EntryList } from '../../../../../lists/common/schemas/types/entries'; -import { getFullEndpointExceptionList } from './lists'; +import { buildArtifact, getFullEndpointExceptionList } from './lists'; +import { TranslatedEntry, TranslatedExceptionListItem } from '../../schemas/artifacts'; describe('buildEventTypeSignal', () => { let mockExceptionClient: ExceptionListClient; @@ -340,4 +341,95 @@ describe('buildEventTypeSignal', () => { const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp.entries.length).toEqual(0); }); + + test('it should return a stable hash regardless of order of entries', async () => { + const translatedEntries: TranslatedEntry[] = [ + { + entries: [ + { + field: 'some.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ]; + const translatedEntriesReversed = translatedEntries.reverse(); + + const translatedExceptionList = { + entries: [ + { + type: 'simple', + entries: translatedEntries, + }, + ], + }; + + const translatedExceptionListReversed = { + entries: [ + { + type: 'simple', + entries: translatedEntriesReversed, + }, + ], + }; + + const artifact1 = await buildArtifact(translatedExceptionList, 'linux', 'v1'); + const artifact2 = await buildArtifact(translatedExceptionListReversed, 'linux', 'v1'); + expect(artifact1.decodedSha256).toEqual(artifact2.decodedSha256); + }); + + test('it should return a stable hash regardless of order of items', async () => { + const translatedItems: TranslatedExceptionListItem[] = [ + { + type: 'simple', + entries: [ + { + entries: [ + { + field: 'some.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + ], + }, + { + type: 'simple', + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }, + ]; + + const translatedExceptionList = { + entries: translatedItems, + }; + + const translatedExceptionListReversed = { + entries: translatedItems.reverse(), + }; + + const artifact1 = await buildArtifact(translatedExceptionList, 'linux', 'v1'); + const artifact2 = await buildArtifact(translatedExceptionListReversed, 'linux', 'v1'); + expect(artifact1.decodedSha256).toEqual(artifact2.decodedSha256); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index e6fd4bad97c5f..68fa2a0511a48 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -25,6 +25,8 @@ import { translatedEntryMatchMatcher, translatedEntryMatchAnyMatcher, TranslatedExceptionListItem, + internalArtifactCompleteSchema, + InternalArtifactCompleteSchema, } from '../../schemas'; import { ArtifactConstants } from './common'; @@ -32,7 +34,7 @@ export async function buildArtifact( exceptions: WrappedTranslatedExceptionList, os: string, schemaVersion: string -): Promise { +): Promise { const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions)); const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex'); @@ -45,11 +47,32 @@ export async function buildArtifact( encodedSha256: sha256, decodedSize: exceptionsBuffer.byteLength, encodedSize: exceptionsBuffer.byteLength, - created: Date.now(), body: exceptionsBuffer.toString('base64'), }; } +export async function maybeCompressArtifact( + uncompressedArtifact: InternalArtifactSchema +): Promise { + const compressedArtifact = { ...uncompressedArtifact }; + if (internalArtifactCompleteSchema.is(uncompressedArtifact)) { + const compressedExceptionList = await compressExceptionList( + Buffer.from(uncompressedArtifact.body, 'base64') + ); + compressedArtifact.body = compressedExceptionList.toString('base64'); + compressedArtifact.encodedSize = compressedExceptionList.byteLength; + compressedArtifact.compressionAlgorithm = 'zlib'; + compressedArtifact.encodedSha256 = createHash('sha256') + .update(compressedExceptionList) + .digest('hex'); + } + return compressedArtifact; +} + +export function isCompressed(artifact: InternalArtifactSchema) { + return artifact.compressionAlgorithm === 'zlib'; +} + export async function getFullEndpointExceptionList( eClient: ExceptionListClient, os: string, @@ -136,7 +159,7 @@ function translateItem( const itemSet = new Set(); return { type: item.type, - entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => { + entries: item.entries.reduce((translatedEntries, entry) => { const translatedEntry = translateEntry(schemaVersion, entry); if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { const itemHash = createHash('sha256').update(JSON.stringify(translatedEntry)).digest('hex'); @@ -156,8 +179,8 @@ function translateEntry( ): TranslatedEntry | undefined { switch (entry.type) { case 'nested': { - const nestedEntries = entry.entries.reduce( - (entries: TranslatedEntryNestedEntry[], nestedEntry) => { + const nestedEntries = entry.entries.reduce( + (entries, nestedEntry) => { const translatedEntry = translateEntry(schemaVersion, nestedEntry); if (nestedEntry !== undefined && translatedEntryNestedEntry.is(translatedEntry)) { entries.push(translatedEntry); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index e1f6bac2620ea..95587c6fc105d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -5,103 +5,125 @@ */ import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; -import { InternalArtifactSchema } from '../../schemas'; -import { - getInternalArtifactMock, - getInternalArtifactMockWithDiffs, -} from '../../schemas/artifacts/saved_objects.mock'; -import { ManifestConstants } from './common'; +import { InternalArtifactCompleteSchema } from '../../schemas'; +import { ManifestConstants, getArtifactId } from './common'; import { Manifest } from './manifest'; +import { + getMockArtifacts, + getMockManifest, + getMockManifestWithDiffs, + getEmptyMockManifest, +} from './mocks'; describe('manifest', () => { describe('Manifest object sanity checks', () => { - const artifacts: InternalArtifactSchema[] = []; - const now = new Date(); + let artifacts: InternalArtifactCompleteSchema[] = []; let manifest1: Manifest; let manifest2: Manifest; + let emptyManifest: Manifest; beforeAll(async () => { - const artifactLinux = await getInternalArtifactMock('linux', 'v1'); - const artifactMacos = await getInternalArtifactMock('macos', 'v1'); - const artifactWindows = await getInternalArtifactMock('windows', 'v1'); - artifacts.push(artifactLinux); - artifacts.push(artifactMacos); - artifacts.push(artifactWindows); - - manifest1 = new Manifest(now, 'v1', ManifestConstants.INITIAL_VERSION); - manifest1.addEntry(artifactLinux); - manifest1.addEntry(artifactMacos); - manifest1.addEntry(artifactWindows); - manifest1.setVersion('abcd'); - - const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', 'v1'); - manifest2 = new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION); - manifest2.addEntry(newArtifactLinux); - manifest2.addEntry(artifactMacos); - manifest2.addEntry(artifactWindows); + artifacts = await getMockArtifacts({ compress: true }); + manifest1 = await getMockManifest({ compress: true }); + manifest2 = await getMockManifestWithDiffs({ compress: true }); + emptyManifest = await getEmptyMockManifest({ compress: true }); }); test('Can create manifest with valid schema version', () => { - const manifest = new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION); + const manifest = new Manifest('v1'); expect(manifest).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { expect(() => { - new Manifest( - new Date(), - 'abcd' as ManifestSchemaVersion, - ManifestConstants.INITIAL_VERSION - ); + new Manifest('abcd' as ManifestSchemaVersion); }).toThrow(); }); + test('Empty manifest transforms correctly to expected endpoint format', async () => { + expect(emptyManifest.toEndpointFormat()).toStrictEqual({ + artifacts: { + 'endpoint-exceptionlist-linux-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + }, + manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + schema_version: 'v1', + }); + }); + test('Manifest transforms correctly to expected endpoint format', async () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'none', + compression_algorithm: 'zlib', encryption_algorithm: 'none', - decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - decoded_size: 430, - encoded_size: 430, + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'none', + compression_algorithm: 'zlib', encryption_algorithm: 'none', - decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - decoded_size: 430, - encoded_size: 430, + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, 'endpoint-exceptionlist-windows-v1': { - compression_algorithm: 'none', + compression_algorithm: 'zlib', encryption_algorithm: 'none', - decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - decoded_size: 430, - encoded_size: 430, + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, }, - manifest_version: 'abcd', + manifest_version: 'a7f4760bfa2662e85e30fe4fb8c01b4c4a20938c76ab21d3c5a3e781e547cce7', schema_version: 'v1', }); }); test('Manifest transforms correctly to expected saved object format', async () => { expect(manifest1.toSavedObject()).toStrictEqual({ - created: now.getTime(), ids: [ - 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', ], }); }); @@ -111,12 +133,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-v1-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51', + 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); @@ -124,7 +146,7 @@ describe('manifest', () => { test('Manifest returns data for given artifact', async () => { const artifact = artifacts[0]; - const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.decodedSha256}`); + const returned = manifest1.getArtifact(getArtifactId(artifact)); expect(returned).toEqual(artifact); }); @@ -132,39 +154,35 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' ); expect(found).toEqual(true); }); test('Manifest can be created from list of artifacts', async () => { - const oldManifest = new Manifest( - new Date(), - ManifestConstants.SCHEMA_VERSION, - ManifestConstants.INITIAL_VERSION - ); + const oldManifest = new Manifest(ManifestConstants.SCHEMA_VERSION); const manifest = Manifest.fromArtifacts(artifacts, 'v1', oldManifest); expect( manifest.contains( - 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 576ecb08d6923..6ece2bf0f48e8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -4,15 +4,22 @@ * you may not use this file except in compliance with the Elastic License. */ +import { createHash } from 'crypto'; import { validate } from '../../../../common/validate'; -import { InternalArtifactSchema, InternalManifestSchema } from '../../schemas/artifacts'; +import { + InternalArtifactSchema, + InternalManifestSchema, + internalArtifactCompleteSchema, + InternalArtifactCompleteSchema, +} from '../../schemas/artifacts'; import { manifestSchemaVersion, ManifestSchemaVersion, } from '../../../../common/endpoint/schema/common'; import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; -import { ManifestConstants } from './common'; import { ManifestEntry } from './manifest_entry'; +import { maybeCompressArtifact, isCompressed } from './lists'; +import { getArtifactId } from './common'; export interface ManifestDiff { type: string; @@ -20,15 +27,13 @@ export interface ManifestDiff { } export class Manifest { - private created: Date; private entries: Record; private schemaVersion: ManifestSchemaVersion; // For concurrency control - private version: string; + private version: string | undefined; - constructor(created: Date, schemaVersion: string, version: string) { - this.created = created; + constructor(schemaVersion: string, version?: string) { this.entries = {}; this.version = version; @@ -38,20 +43,24 @@ export class Manifest { ); if (errors != null || validated === null) { - throw new Error(`Invalid manifest version: ${schemaVersion}`); + throw new Error(`Invalid manifest schema version: ${schemaVersion}`); } this.schemaVersion = validated; } + public static getDefault(schemaVersion: string) { + return new Manifest(schemaVersion); + } + public static fromArtifacts( - artifacts: InternalArtifactSchema[], + artifacts: InternalArtifactCompleteSchema[], schemaVersion: string, oldManifest: Manifest ): Manifest { - const manifest = new Manifest(new Date(), schemaVersion, oldManifest.getVersion()); + const manifest = new Manifest(schemaVersion, oldManifest.getSoVersion()); artifacts.forEach((artifact) => { - const id = `${artifact.identifier}-${artifact.decodedSha256}`; + const id = getArtifactId(artifact); const existingArtifact = oldManifest.getArtifact(id); if (existingArtifact) { manifest.addEntry(existingArtifact); @@ -62,15 +71,70 @@ export class Manifest { return manifest; } + public static fromPkgConfig(manifestPkgConfig: ManifestSchema): Manifest | null { + if (manifestSchema.is(manifestPkgConfig)) { + const manifest = new Manifest(manifestPkgConfig.schema_version); + for (const [identifier, artifactRecord] of Object.entries(manifestPkgConfig.artifacts)) { + const artifact = { + identifier, + compressionAlgorithm: artifactRecord.compression_algorithm, + encryptionAlgorithm: artifactRecord.encryption_algorithm, + decodedSha256: artifactRecord.decoded_sha256, + decodedSize: artifactRecord.decoded_size, + encodedSha256: artifactRecord.encoded_sha256, + encodedSize: artifactRecord.encoded_size, + }; + manifest.addEntry(artifact); + } + return manifest; + } else { + return null; + } + } + + public async compressArtifact(id: string): Promise { + try { + const artifact = this.getArtifact(id); + if (artifact == null) { + throw new Error(`Corrupted manifest detected. Artifact ${id} not in manifest.`); + } + + const compressedArtifact = await maybeCompressArtifact(artifact); + if (!isCompressed(compressedArtifact)) { + throw new Error(`Unable to compress artifact: ${id}`); + } else if (!internalArtifactCompleteSchema.is(compressedArtifact)) { + throw new Error(`Incomplete artifact detected: ${id}`); + } + this.addEntry(compressedArtifact); + } catch (err) { + return err; + } + return null; + } + + public equals(manifest: Manifest): boolean { + return this.getSha256() === manifest.getSha256(); + } + + public getSha256(): string { + let sha256 = createHash('sha256'); + Object.keys(this.entries) + .sort() + .forEach((docId) => { + sha256 = sha256.update(docId); + }); + return sha256.digest('hex'); + } + public getSchemaVersion(): ManifestSchemaVersion { return this.schemaVersion; } - public getVersion(): string { + public getSoVersion(): string | undefined { return this.version; } - public setVersion(version: string) { + public setSoVersion(version: string) { this.version = version; } @@ -87,8 +151,12 @@ export class Manifest { return this.entries; } + public getEntry(artifactId: string): ManifestEntry | undefined { + return this.entries[artifactId]; + } + public getArtifact(artifactId: string): InternalArtifactSchema | undefined { - return this.entries[artifactId]?.getArtifact(); + return this.getEntry(artifactId)?.getArtifact(); } public diff(manifest: Manifest): ManifestDiff[] { @@ -111,7 +179,7 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifest_version: this.version ?? ManifestConstants.INITIAL_VERSION, + manifest_version: this.getSha256(), schema_version: this.schemaVersion, artifacts: {}, }; @@ -130,7 +198,6 @@ export class Manifest { public toSavedObject(): InternalManifestSchema { return { - created: this.created.getTime(), ids: Object.keys(this.entries), }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 7ea2a07210c55..d7bd57547de0a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -14,7 +14,7 @@ describe('manifest_entry', () => { let manifestEntry: ManifestEntry; beforeAll(async () => { - artifact = await getInternalArtifactMock('windows', 'v1'); + artifact = await getInternalArtifactMock('windows', 'v1', { compress: true }); manifestEntry = new ManifestEntry(artifact); }); @@ -24,7 +24,7 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' ); }); @@ -34,21 +34,21 @@ describe('manifest_entry', () => { test('Correct sha256 is returned', () => { expect(manifestEntry.getEncodedSha256()).toEqual( - '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e' ); expect(manifestEntry.getDecodedSha256()).toEqual( - '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' ); }); test('Correct size is returned', () => { - expect(manifestEntry.getEncodedSize()).toEqual(430); - expect(manifestEntry.getDecodedSize()).toEqual(430); + expect(manifestEntry.getEncodedSize()).toEqual(147); + expect(manifestEntry.getDecodedSize()).toEqual(432); }); test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' ); }); @@ -58,17 +58,15 @@ describe('manifest_entry', () => { test('Correct record is returned', () => { expect(manifestEntry.getRecord()).toEqual({ - compression_algorithm: 'none', + compression_algorithm: 'zlib', encryption_algorithm: 'none', - decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - decoded_size: 430, - encoded_size: 430, + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }); }); - - // TODO: add test for entry with compression }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index b35e0c2b9ad6e..b6c103e24f024 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -7,6 +7,7 @@ import { InternalArtifactSchema } from '../../schemas/artifacts'; import { CompressionAlgorithm } from '../../../../common/endpoint/schema/common'; import { ManifestEntrySchema } from '../../../../common/endpoint/schema/manifest'; +import { getArtifactId } from './common'; export class ManifestEntry { private artifact: InternalArtifactSchema; @@ -16,7 +17,7 @@ export class ManifestEntry { } public getDocId(): string { - return `${this.getIdentifier()}-${this.getDecodedSha256()}`; + return getArtifactId(this.artifact); } public getIdentifier(): string { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts new file mode 100644 index 0000000000000..097151ee835ba --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; +import { + getInternalArtifactMock, + getInternalArtifactMockWithDiffs, + getEmptyInternalArtifactMock, +} from '../../schemas/artifacts/saved_objects.mock'; +import { ArtifactConstants } from './common'; +import { Manifest } from './manifest'; + +export const getMockArtifacts = async (opts?: { compress: boolean }) => { + return Promise.all( + ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map>( + async (os) => { + return getInternalArtifactMock(os, 'v1', opts); + } + ) + ); +}; + +export const getMockArtifactsWithDiff = async (opts?: { compress: boolean }) => { + return Promise.all( + ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map>( + async (os) => { + if (os === 'linux') { + return getInternalArtifactMockWithDiffs(os, 'v1'); + } + return getInternalArtifactMock(os, 'v1', opts); + } + ) + ); +}; + +export const getEmptyMockArtifacts = async (opts?: { compress: boolean }) => { + return Promise.all( + ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map>( + async (os) => { + return getEmptyInternalArtifactMock(os, 'v1', opts); + } + ) + ); +}; + +export const getMockManifest = async (opts?: { compress: boolean }) => { + const manifest = new Manifest('v1'); + const artifacts = await getMockArtifacts(opts); + artifacts.forEach((artifact) => manifest.addEntry(artifact)); + return manifest; +}; + +export const getMockManifestWithDiffs = async (opts?: { compress: boolean }) => { + const manifest = new Manifest('v1'); + const artifacts = await getMockArtifactsWithDiff(opts); + artifacts.forEach((artifact) => manifest.addEntry(artifact)); + return manifest; +}; + +export const getEmptyMockManifest = async (opts?: { compress: boolean }) => { + const manifest = new Manifest('v1'); + const artifacts = await getEmptyMockArtifacts(opts); + artifacts.forEach((artifact) => manifest.addEntry(artifact)); + return manifest; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 583f4499f591b..ba164059866ea 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -11,7 +11,8 @@ import { TaskManagerStartContract, } from '../../../../../task_manager/server'; import { EndpointAppContext } from '../../types'; -import { reportErrors } from './common'; +import { reportErrors, ManifestConstants } from './common'; +import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; export const ManifestTaskConstants = { TIMEOUT: '1m', @@ -89,37 +90,66 @@ export class ManifestTask { return; } - let errors: Error[] = []; try { - // get snapshot based on exception-list-agnostic SOs - // with diffs from last dispatched manifest - const snapshot = await manifestManager.getSnapshot(); - if (snapshot && snapshot.diffs.length > 0) { - // create new artifacts - errors = await manifestManager.syncArtifacts(snapshot, 'add'); - if (errors.length) { - reportErrors(this.logger, errors); - throw new Error('Error writing new artifacts.'); - } - // write to ingest-manager package config - errors = await manifestManager.dispatch(snapshot.manifest); - if (errors.length) { - reportErrors(this.logger, errors); - throw new Error('Error dispatching manifest.'); + // Last manifest we computed, which was saved to ES + const oldManifest = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + if (oldManifest == null) { + this.logger.debug('User manifest not available yet.'); + return; + } + + // New computed manifest based on current state of exception list + const newManifest = await manifestManager.buildNewManifest( + ManifestConstants.SCHEMA_VERSION, + oldManifest + ); + const diffs = newManifest.diff(oldManifest); + + // Compress new artifacts + const adds = diffs.filter((diff) => diff.type === 'add').map((diff) => diff.id); + for (const artifactId of adds) { + const compressError = await newManifest.compressArtifact(artifactId); + if (compressError) { + throw compressError; } - // commit latest manifest state to user-artifact-manifest SO - const error = await manifestManager.commit(snapshot.manifest); + } + + // Persist new artifacts + const artifacts = adds + .map((artifactId) => newManifest.getArtifact(artifactId)) + .filter((artifact): artifact is InternalArtifactCompleteSchema => artifact !== undefined); + if (artifacts.length !== adds.length) { + throw new Error('Invalid artifact encountered.'); + } + const persistErrors = await manifestManager.pushArtifacts(artifacts); + if (persistErrors.length) { + reportErrors(this.logger, persistErrors); + throw new Error('Unable to persist new artifacts.'); + } + + // Commit latest manifest state, if different + if (diffs.length) { + const error = await manifestManager.commit(newManifest); if (error) { - reportErrors(this.logger, [error]); - throw new Error('Error committing manifest.'); - } - // clean up old artifacts - errors = await manifestManager.syncArtifacts(snapshot, 'delete'); - if (errors.length) { - reportErrors(this.logger, errors); - throw new Error('Error cleaning up outdated artifacts.'); + throw error; } } + + // Try dispatching to ingest-manager package configs + const dispatchErrors = await manifestManager.tryDispatch(newManifest); + if (dispatchErrors.length) { + reportErrors(this.logger, dispatchErrors); + throw new Error('Error dispatching manifest.'); + } + + // Try to clean up superceded artifacts + const deletes = diffs.filter((diff) => diff.type === 'delete').map((diff) => diff.id); + const deleteErrors = await manifestManager.deleteArtifacts(deletes); + if (deleteErrors.length) { + reportErrors(this.logger, deleteErrors); + } } catch (err) { this.logger.error(err); } diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index 6a8c26e08d9dd..9ca447d53bf45 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -6,8 +6,6 @@ import { ILegacyScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { loggerMock } from 'src/core/server/logging/logger.mock'; import { xpackMocks } from '../../../../mocks'; import { AgentService, @@ -15,28 +13,24 @@ import { ExternalCallback, } from '../../../ingest_manager/server'; import { createPackageConfigServiceMock } from '../../../ingest_manager/server/mocks'; -import { ConfigType } from '../config'; import { createMockConfig } from '../lib/detection_engine/routes/__mocks__'; import { EndpointAppContextService, EndpointAppContextServiceStartContract, } from './endpoint_app_context_services'; -import { - ManifestManagerMock, - getManifestManagerMock, -} from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { ManifestManager } from './services/artifacts/manifest_manager/manifest_manager'; +import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; import { EndpointAppContext } from './types'; /** * Creates a mocked EndpointAppContext. */ export const createMockEndpointAppContext = ( - mockManifestManager?: ManifestManagerMock + mockManifestManager?: ManifestManager ): EndpointAppContext => { return { logFactory: loggingSystemMock.create(), - // @ts-ignore - config: createMockConfig() as ConfigType, + config: () => Promise.resolve(createMockConfig()), service: createMockEndpointAppContextService(mockManifestManager), }; }; @@ -45,16 +39,15 @@ export const createMockEndpointAppContext = ( * Creates a mocked EndpointAppContextService */ export const createMockEndpointAppContextService = ( - mockManifestManager?: ManifestManagerMock + mockManifestManager?: ManifestManager ): jest.Mocked => { - return { + return ({ start: jest.fn(), stop: jest.fn(), getAgentService: jest.fn(), - // @ts-ignore - getManifestManager: mockManifestManager ?? jest.fn(), + getManifestManager: jest.fn().mockReturnValue(mockManifestManager ?? jest.fn()), getScopedSavedObjectsClient: jest.fn(), - }; + } as unknown) as jest.Mocked; }; /** @@ -65,7 +58,7 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< > => { return { agentService: createMockAgentService(), - logger: loggerMock.create(), + logger: loggingSystemMock.create().get('mock_endpoint_app_context'), savedObjectsStart: savedObjectsServiceMock.createStartContract(), manifestManager: getManifestManagerMock(), registerIngestCallback: jest.fn< diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 1b364a04a4272..218f7c059da48 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -20,7 +20,7 @@ import { DownloadArtifactRequestParamsSchema, downloadArtifactRequestParamsSchema, downloadArtifactResponseSchema, - InternalArtifactSchema, + InternalArtifactCompleteSchema, } from '../../schemas/artifacts'; import { EndpointAppContext } from '../../types'; @@ -86,8 +86,8 @@ export function registerDownloadExceptionListRoute( } else { logger.debug(`Cache MISS artifact ${id}`); return scopedSOClient - .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) - .then((artifact: SavedObject) => { + .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) + .then((artifact: SavedObject) => { const body = Buffer.from(artifact.attributes.body, 'base64'); cache.set(id, body); return buildAndValidateResponse(artifact.attributes.identifier, body); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts index 343b192163479..2cef1f3be69c1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -15,13 +15,13 @@ export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList { entries: [ { - field: 'some.not.nested.field', + field: 'some.nested.field', operator: 'included', type: 'exact_cased', value: 'some value', }, ], - field: 'some.field', + field: 'some.parentField', type: 'nested', }, { diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index 183a819807ed2..d95627601a183 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -4,37 +4,53 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ArtifactConstants, buildArtifact } from '../../lib/artifacts'; +import { buildArtifact, maybeCompressArtifact, isCompressed } from '../../lib/artifacts'; import { getTranslatedExceptionListMock } from './lists.mock'; -import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; +import { + InternalManifestSchema, + internalArtifactCompleteSchema, + InternalArtifactCompleteSchema, +} from './saved_objects'; + +const compressArtifact = async (artifact: InternalArtifactCompleteSchema) => { + const compressedArtifact = await maybeCompressArtifact(artifact); + if (!isCompressed(compressedArtifact)) { + throw new Error(`Unable to compress artifact: ${artifact.identifier}`); + } else if (!internalArtifactCompleteSchema.is(compressedArtifact)) { + throw new Error(`Incomplete artifact detected: ${artifact.identifier}`); + } + return compressedArtifact; +}; export const getInternalArtifactMock = async ( os: string, - schemaVersion: string -): Promise => { - return buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); + schemaVersion: string, + opts?: { compress: boolean } +): Promise => { + const artifact = await buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); + return opts?.compress ? compressArtifact(artifact) : artifact; }; -export const getInternalArtifactMockWithDiffs = async ( +export const getEmptyInternalArtifactMock = async ( os: string, - schemaVersion: string -): Promise => { - const mock = getTranslatedExceptionListMock(); - mock.entries.pop(); - return buildArtifact(mock, os, schemaVersion); + schemaVersion: string, + opts?: { compress: boolean } +): Promise => { + const artifact = await buildArtifact({ entries: [] }, os, schemaVersion); + return opts?.compress ? compressArtifact(artifact) : artifact; }; -export const getInternalArtifactsMock = async ( +export const getInternalArtifactMockWithDiffs = async ( os: string, - schemaVersion: string -): Promise => { - // @ts-ignore - return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map(async () => { - await buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); - }); + schemaVersion: string, + opts?: { compress: boolean } +): Promise => { + const mock = getTranslatedExceptionListMock(); + mock.entries.pop(); + const artifact = await buildArtifact(mock, os, schemaVersion); + return opts?.compress ? compressArtifact(artifact) : artifact; }; export const getInternalManifestMock = (): InternalManifestSchema => ({ - created: Date.now(), ids: [], }); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index aa11f4409269a..4dea916dcb436 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -16,7 +16,7 @@ import { created } from './common'; export const body = t.string; // base64 -export const internalArtifactSchema = t.exact( +export const internalArtifactRecordSchema = t.exact( t.type({ identifier, compressionAlgorithm, @@ -25,18 +25,49 @@ export const internalArtifactSchema = t.exact( decodedSize: size, encodedSha256: sha256, encodedSize: size, - created, - body, }) ); +export type InternalArtifactRecordSchema = t.TypeOf; +export const internalArtifactAdditionalFields = { + body, +}; + +export const internalArtifactSchema = t.intersection([ + internalArtifactRecordSchema, + t.partial(internalArtifactAdditionalFields), +]); export type InternalArtifactSchema = t.TypeOf; +export const internalArtifactCompleteSchema = t.intersection([ + internalArtifactRecordSchema, + t.exact(t.type(internalArtifactAdditionalFields)), +]); +export type InternalArtifactCompleteSchema = t.TypeOf; + +export const internalArtifactCreateSchema = t.intersection([ + internalArtifactCompleteSchema, + t.exact( + t.type({ + created, + }) + ), +]); +export type InternalArtifactCreateSchema = t.TypeOf; + export const internalManifestSchema = t.exact( t.type({ - created, ids: t.array(identifier), }) ); - export type InternalManifestSchema = t.TypeOf; + +export const internalManifestCreateSchema = t.intersection([ + internalManifestSchema, + t.exact( + t.type({ + created, + }) + ), +]); +export type InternalManifestCreateSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index 3e3b12c04d65c..0787231e242cb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -5,7 +5,7 @@ */ import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { ArtifactConstants } from '../../lib/artifacts'; +import { ArtifactConstants, getArtifactId } from '../../lib/artifacts'; import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; import { getArtifactClientMock } from './artifact_client.mock'; import { ArtifactClient } from './artifact_client'; @@ -31,8 +31,11 @@ describe('artifact_client', () => { await artifactClient.createArtifact(artifact); expect(savedObjectsClient.create).toHaveBeenCalledWith( ArtifactConstants.SAVED_OBJECT_TYPE, - artifact, - { id: artifactClient.getArtifactId(artifact) } + { + ...artifact, + created: expect.any(Number), + }, + { id: getArtifactId(artifact) } ); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index ca53a891c4d6b..6138b4fb7e6dc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -5,8 +5,11 @@ */ import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; -import { ArtifactConstants } from '../../lib/artifacts'; -import { InternalArtifactSchema } from '../../schemas/artifacts'; +import { ArtifactConstants, getArtifactId } from '../../lib/artifacts'; +import { + InternalArtifactCompleteSchema, + InternalArtifactCreateSchema, +} from '../../schemas/artifacts'; export class ArtifactClient { private savedObjectsClient: SavedObjectsClientContract; @@ -15,24 +18,23 @@ export class ArtifactClient { this.savedObjectsClient = savedObjectsClient; } - public getArtifactId(artifact: InternalArtifactSchema) { - return `${artifact.identifier}-${artifact.decodedSha256}`; - } - - public async getArtifact(id: string): Promise> { - return this.savedObjectsClient.get( + public async getArtifact(id: string): Promise> { + return this.savedObjectsClient.get( ArtifactConstants.SAVED_OBJECT_TYPE, id ); } public async createArtifact( - artifact: InternalArtifactSchema - ): Promise> { - return this.savedObjectsClient.create( + artifact: InternalArtifactCompleteSchema + ): Promise> { + return this.savedObjectsClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, - artifact, - { id: this.getArtifactId(artifact) } + { + ...artifact, + created: Date.now(), + }, + { id: getArtifactId(artifact) } ); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts index fe3f193bc8ff5..6db29289e983d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -38,7 +38,10 @@ describe('manifest_client', () => { await manifestClient.createManifest(manifest); expect(savedObjectsClient.create).toHaveBeenCalledWith( ManifestConstants.SAVED_OBJECT_TYPE, - manifest, + { + ...manifest, + created: expect.any(Number), + }, { id: manifestClient.getManifestId() } ); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts index 45182841e56fc..385f115e6301a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -15,7 +15,7 @@ import { } from '../../../../common/endpoint/schema/common'; import { validate } from '../../../../common/validate'; import { ManifestConstants } from '../../lib/artifacts'; -import { InternalManifestSchema } from '../../schemas/artifacts'; +import { InternalManifestSchema, InternalManifestCreateSchema } from '../../schemas/artifacts'; interface UpdateManifestOpts { version: string; @@ -57,9 +57,12 @@ export class ManifestClient { public async createManifest( manifest: InternalManifestSchema ): Promise> { - return this.savedObjectsClient.create( + return this.savedObjectsClient.create( ManifestConstants.SAVED_OBJECT_TYPE, - manifest, + { + ...manifest, + created: Date.now(), + }, { id: this.getManifestId() } ); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 3e4fee8871b8a..08cdb9816a1c1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -6,53 +6,34 @@ import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks'; import { Logger } from 'src/core/server'; -import { createPackageConfigMock } from '../../../../../../ingest_manager/common/mocks'; +import { + createPackageConfigWithManifestMock, + createPackageConfigWithInitialManifestMock, +} from '../../../../../../ingest_manager/common/mocks'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks'; -import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { listMock } from '../../../../../../lists/server/mocks'; -import { - ExceptionsCache, - Manifest, - buildArtifact, - getFullEndpointExceptionList, -} from '../../../lib/artifacts'; -import { ManifestConstants } from '../../../lib/artifacts/common'; -import { InternalArtifactSchema } from '../../../schemas/artifacts'; +import { ExceptionsCache } from '../../../lib/artifacts'; import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; import { ManifestManager } from './manifest_manager'; +import { + getMockManifest, + getMockArtifactsWithDiff, + getEmptyMockArtifacts, +} from '../../../lib/artifacts/mocks'; -async function mockBuildExceptionListArtifacts( - os: string, - schemaVersion: string -): Promise { - const mockExceptionClient = listMock.getExceptionListClient(); - const first = getFoundExceptionListItemSchemaMock(); - mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const exceptions = await getFullEndpointExceptionList(mockExceptionClient, os, schemaVersion); - return [await buildArtifact(exceptions, os, schemaVersion)]; -} - -export class ManifestManagerMock extends ManifestManager { - protected buildExceptionListArtifacts = jest - .fn() - .mockResolvedValue(mockBuildExceptionListArtifacts('linux', 'v1')); - - public getLastDispatchedManifest = jest - .fn() - .mockResolvedValue(new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION)); - - protected getManifestClient = jest - .fn() - .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); +export enum ManifestManagerMockType { + InitialSystemState, + NormalFlow, } export const getManifestManagerMock = (opts?: { + mockType?: ManifestManagerMockType; cache?: ExceptionsCache; packageConfigService?: jest.Mocked; savedObjectsClient?: ReturnType; -}): ManifestManagerMock => { +}): ManifestManager => { let cache = new ExceptionsCache(5); if (opts?.cache !== undefined) { cache = opts.cache; @@ -64,7 +45,11 @@ export const getManifestManagerMock = (opts?: { } packageConfigService.list = jest.fn().mockResolvedValue({ total: 1, - items: [{ version: 'abcd', ...createPackageConfigMock() }], + items: [ + { version: 'policy-1-version', ...createPackageConfigWithManifestMock() }, + { version: 'policy-2-version', ...createPackageConfigWithInitialManifestMock() }, + { version: 'policy-3-version', ...createPackageConfigWithInitialManifestMock() }, + ], }); let savedObjectsClient = savedObjectsClientMock.create(); @@ -72,6 +57,32 @@ export const getManifestManagerMock = (opts?: { savedObjectsClient = opts.savedObjectsClient; } + class ManifestManagerMock extends ManifestManager { + protected buildExceptionListArtifacts = jest.fn().mockImplementation(() => { + const mockType = opts?.mockType ?? ManifestManagerMockType.NormalFlow; + switch (mockType) { + case ManifestManagerMockType.InitialSystemState: + return getEmptyMockArtifacts(); + case ManifestManagerMockType.NormalFlow: + return getMockArtifactsWithDiff(); + } + }); + + public getLastComputedManifest = jest.fn().mockImplementation(() => { + const mockType = opts?.mockType ?? ManifestManagerMockType.NormalFlow; + switch (mockType) { + case ManifestManagerMockType.InitialSystemState: + return null; + case ManifestManagerMockType.NormalFlow: + return getMockManifest({ compress: true }); + } + }); + + protected getManifestClient = jest + .fn() + .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); + } + const manifestManager = new ManifestManagerMock({ artifactClient: getArtifactClientMock(savedObjectsClient), cache, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 80d325ece765c..ff331f7d017f4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -10,40 +10,71 @@ import { createPackageConfigServiceMock } from '../../../../../../ingest_manager import { ArtifactConstants, ManifestConstants, - Manifest, ExceptionsCache, + isCompleteArtifact, } from '../../../lib/artifacts'; import { getManifestManagerMock } from './manifest_manager.mock'; describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { - test('ManifestManager can snapshot manifest', async () => { + test('ManifestManager can retrieve and diff manifests', async () => { const manifestManager = getManifestManagerMock(); - const snapshot = await manifestManager.getSnapshot(); - expect(snapshot!.diffs).toEqual([ + const oldManifest = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + const newManifest = await manifestManager.buildNewManifest( + ManifestConstants.SCHEMA_VERSION, + oldManifest! + ); + expect(newManifest.diff(oldManifest!)).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + type: 'delete', + }, + { + id: + 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); - expect(snapshot!.manifest).toBeInstanceOf(Manifest); }); test('ManifestManager populates cache properly', async () => { const cache = new ExceptionsCache(5); const manifestManager = getManifestManagerMock({ cache }); - const snapshot = await manifestManager.getSnapshot(); - expect(snapshot!.diffs).toEqual([ + const oldManifest = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + const newManifest = await manifestManager.buildNewManifest( + ManifestConstants.SCHEMA_VERSION, + oldManifest! + ); + const diffs = newManifest.diff(oldManifest!); + expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + type: 'delete', + }, + { + id: + 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); - await manifestManager.syncArtifacts(snapshot!, 'add'); - const diff = snapshot!.diffs[0]; - const entry = JSON.parse(inflateSync(cache.get(diff!.id)! as Buffer).toString()); + + const newArtifactId = diffs[1].id; + await newManifest.compressArtifact(newArtifactId); + const artifact = newManifest.getArtifact(newArtifactId)!; + + if (isCompleteArtifact(artifact)) { + await manifestManager.pushArtifacts([artifact]); // caches the artifact + } else { + throw new Error('Artifact is missing a body.'); + } + + const entry = JSON.parse(inflateSync(cache.get(newArtifactId)! as Buffer).toString()); expect(entry).toEqual({ entries: [ { @@ -52,7 +83,7 @@ describe('manifest_manager', () => { { entries: [ { - field: 'nested.field', + field: 'some.nested.field', operator: 'included', type: 'exact_cased', value: 'some value', @@ -73,28 +104,77 @@ describe('manifest_manager', () => { }); }); + test('ManifestManager cannot dispatch incomplete (uncompressed) artifact', async () => { + const packageConfigService = createPackageConfigServiceMock(); + const manifestManager = getManifestManagerMock({ packageConfigService }); + const oldManifest = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + const newManifest = await manifestManager.buildNewManifest( + ManifestConstants.SCHEMA_VERSION, + oldManifest! + ); + const dispatchErrors = await manifestManager.tryDispatch(newManifest); + expect(dispatchErrors.length).toEqual(1); + expect(dispatchErrors[0].message).toEqual('Invalid manifest'); + }); + test('ManifestManager can dispatch manifest', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const snapshot = await manifestManager.getSnapshot(); - const dispatchErrors = await manifestManager.dispatch(snapshot!.manifest); + const oldManifest = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + const newManifest = await manifestManager.buildNewManifest( + ManifestConstants.SCHEMA_VERSION, + oldManifest! + ); + const diffs = newManifest.diff(oldManifest!); + const newArtifactId = diffs[1].id; + await newManifest.compressArtifact(newArtifactId); + + const dispatchErrors = await manifestManager.tryDispatch(newManifest); + expect(dispatchErrors).toEqual([]); - const entries = snapshot!.manifest.getEntries(); - const artifact = Object.values(entries)[0].getArtifact(); + + // 2 policies updated... 1 is already up-to-date + expect(packageConfigService.update.mock.calls.length).toEqual(2); + expect( packageConfigService.update.mock.calls[0][2].inputs[0].config!.artifact_manifest.value ).toEqual({ - manifest_version: ManifestConstants.INITIAL_VERSION, + manifest_version: '520f6cf88b3f36a065c6ca81058d5f8690aadadf6fe857f8dec4cc37589e7283', schema_version: 'v1', artifacts: { - [artifact.identifier]: { - compression_algorithm: 'none', + 'endpoint-exceptionlist-linux-v1': { + compression_algorithm: 'zlib', encryption_algorithm: 'none', - decoded_sha256: artifact.decodedSha256, - encoded_sha256: artifact.encodedSha256, - decoded_size: artifact.decodedSize, - encoded_size: artifact.encodedSize, - relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.decodedSha256}`, + decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + encoded_sha256: '57941169bb2c5416f9bd7224776c8462cb9a2be0fe8b87e6213e77a1d29be824', + decoded_size: 292, + encoded_size: 131, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + }, + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, }, }); @@ -103,29 +183,20 @@ describe('manifest_manager', () => { test('ManifestManager fails to dispatch on conflict', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const snapshot = await manifestManager.getSnapshot(); - packageConfigService.update.mockRejectedValue({ status: 409 }); - const dispatchErrors = await manifestManager.dispatch(snapshot!.manifest); + const oldManifest = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + const newManifest = await manifestManager.buildNewManifest( + ManifestConstants.SCHEMA_VERSION, + oldManifest! + ); + const diffs = newManifest.diff(oldManifest!); + const newArtifactId = diffs[1].id; + await newManifest.compressArtifact(newArtifactId); + + packageConfigService.update.mockRejectedValueOnce({ status: 409 }); + const dispatchErrors = await manifestManager.tryDispatch(newManifest); expect(dispatchErrors).toEqual([{ status: 409 }]); - const entries = snapshot!.manifest.getEntries(); - const artifact = Object.values(entries)[0].getArtifact(); - expect( - packageConfigService.update.mock.calls[0][2].inputs[0].config!.artifact_manifest.value - ).toEqual({ - manifest_version: ManifestConstants.INITIAL_VERSION, - schema_version: 'v1', - artifacts: { - [artifact.identifier]: { - compression_algorithm: 'none', - encryption_algorithm: 'none', - decoded_sha256: artifact.decodedSha256, - encoded_sha256: artifact.encodedSha256, - decoded_size: artifact.decodedSize, - encoded_size: artifact.encodedSize, - relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.decodedSha256}`, - }, - }, - }); }); test('ManifestManager can commit manifest', async () => { @@ -134,37 +205,43 @@ describe('manifest_manager', () => { savedObjectsClient, }); - const snapshot = await manifestManager.getSnapshot(); - await manifestManager.syncArtifacts(snapshot!, 'add'); - - const diff = { - id: 'abcd', - type: 'delete', - }; - snapshot!.diffs.push(diff); - - const dispatched = await manifestManager.dispatch(snapshot!.manifest); - expect(dispatched).toEqual([]); + const oldManifest = await manifestManager.getLastComputedManifest( + ManifestConstants.SCHEMA_VERSION + ); + const newManifest = await manifestManager.buildNewManifest( + ManifestConstants.SCHEMA_VERSION, + oldManifest! + ); + const diffs = newManifest.diff(oldManifest!); + const oldArtifactId = diffs[0].id; + const newArtifactId = diffs[1].id; + await newManifest.compressArtifact(newArtifactId); - await manifestManager.commit(snapshot!.manifest); + const artifact = newManifest.getArtifact(newArtifactId)!; + if (isCompleteArtifact(artifact)) { + await manifestManager.pushArtifacts([artifact]); + } else { + throw new Error('Artifact is missing a body.'); + } - await manifestManager.syncArtifacts(snapshot!, 'delete'); + await manifestManager.commit(newManifest); + await manifestManager.deleteArtifacts([oldArtifactId]); // created new artifact expect(savedObjectsClient.create.mock.calls[0][0]).toEqual( ArtifactConstants.SAVED_OBJECT_TYPE ); - // deleted old artifact - expect(savedObjectsClient.delete).toHaveBeenCalledWith( - ArtifactConstants.SAVED_OBJECT_TYPE, - 'abcd' - ); - // committed new manifest expect(savedObjectsClient.create.mock.calls[1][0]).toEqual( ManifestConstants.SAVED_OBJECT_TYPE ); + + // deleted old artifact + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + oldArtifactId + ); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index c8cad32ab746e..2501f07cb26e0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -5,10 +5,11 @@ */ import { Logger, SavedObjectsClientContract } from 'src/core/server'; -import { createHash } from 'crypto'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; +import { manifestDispatchSchema } from '../../../../../common/endpoint/schema/manifest'; + import { ArtifactConstants, ManifestConstants, @@ -17,11 +18,14 @@ import { getFullEndpointExceptionList, ExceptionsCache, ManifestDiff, + getArtifactId, } from '../../../lib/artifacts'; -import { InternalArtifactSchema } from '../../../schemas/artifacts'; +import { + InternalArtifactCompleteSchema, + internalArtifactCompleteSchema, +} from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; import { ManifestClient } from '../manifest_client'; -import { compressExceptionList } from '../../../lib/artifacts/lists'; export interface ManifestManagerContext { savedObjectsClient: SavedObjectsClientContract; @@ -73,82 +77,86 @@ export class ManifestManager { * state of exception-list-agnostic SOs. * * @param schemaVersion The schema version of the artifact - * @returns {Promise} An array of uncompressed artifacts built from exception-list-agnostic SOs. + * @returns {Promise} An array of uncompressed artifacts built from exception-list-agnostic SOs. * @throws Throws/rejects if there are errors building the list. */ protected async buildExceptionListArtifacts( schemaVersion: string - ): Promise { - // TODO: should wrap in try/catch? - return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce( - async (acc: Promise, os) => { - const exceptionList = await getFullEndpointExceptionList( - this.exceptionListClient, - os, - schemaVersion - ); - const artifacts = await acc; - const artifact = await buildArtifact(exceptionList, os, schemaVersion); - artifacts.push(artifact); - return Promise.resolve(artifacts); - }, - Promise.resolve([]) - ); + ): Promise { + return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce< + Promise + >(async (acc, os) => { + const exceptionList = await getFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion + ); + const artifacts = await acc; + const artifact = await buildArtifact(exceptionList, os, schemaVersion); + return Promise.resolve([...artifacts, artifact]); + }, Promise.resolve([])); } /** - * Writes new artifact SOs based on provided snapshot. + * Writes new artifact SO. * - * @param snapshot A ManifestSnapshot to use for writing the artifacts. - * @returns {Promise} Any errors encountered. + * @param artifact An InternalArtifactCompleteSchema representing the artifact. + * @returns {Promise} An error, if encountered, or null. */ - private async writeArtifacts(snapshot: ManifestSnapshot): Promise { - const errors: Error[] = []; - for (const diff of snapshot.diffs) { - const artifact = snapshot.manifest.getArtifact(diff.id); - if (artifact === undefined) { - throw new Error( - `Corrupted manifest detected. Diff contained artifact ${diff.id} not in manifest.` - ); + protected async pushArtifact(artifact: InternalArtifactCompleteSchema): Promise { + const artifactId = getArtifactId(artifact); + try { + // Write the artifact SO + await this.artifactClient.createArtifact(artifact); + + // Cache the compressed body of the artifact + this.cache.set(artifactId, Buffer.from(artifact.body, 'base64')); + } catch (err) { + if (err.status === 409) { + this.logger.debug(`Tried to create artifact ${artifactId}, but it already exists.`); + } else { + return err; } + } - const compressedArtifact = await compressExceptionList(Buffer.from(artifact.body, 'base64')); - artifact.body = compressedArtifact.toString('base64'); - artifact.encodedSize = compressedArtifact.byteLength; - artifact.compressionAlgorithm = 'zlib'; - artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); + return null; + } - try { - // Write the artifact SO - await this.artifactClient.createArtifact(artifact); - // Cache the compressed body of the artifact - this.cache.set(diff.id, Buffer.from(artifact.body, 'base64')); - } catch (err) { - if (err.status === 409) { - this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); - } else { - // TODO: log error here? + /** + * Writes new artifact SOs. + * + * @param artifacts An InternalArtifactCompleteSchema array representing the artifacts. + * @returns {Promise} Any errors encountered. + */ + public async pushArtifacts(artifacts: InternalArtifactCompleteSchema[]): Promise { + const errors: Error[] = []; + for (const artifact of artifacts) { + if (internalArtifactCompleteSchema.is(artifact)) { + const err = await this.pushArtifact(artifact); + if (err) { errors.push(err); } + } else { + errors.push(new Error(`Incomplete artifact: ${getArtifactId(artifact)}`)); } } return errors; } /** - * Deletes old artifact SOs based on provided snapshot. + * Deletes outdated artifact SOs. + * + * The artifact may still remain in the cache. * - * @param snapshot A ManifestSnapshot to use for deleting the artifacts. + * @param artifactIds The IDs of the artifact to delete.. * @returns {Promise} Any errors encountered. */ - private async deleteArtifacts(snapshot: ManifestSnapshot): Promise { + public async deleteArtifacts(artifactIds: string[]): Promise { const errors: Error[] = []; - for (const diff of snapshot.diffs) { + for (const artifactId of artifactIds) { try { - // Delete the artifact SO - await this.artifactClient.deleteArtifact(diff.id); - // TODO: should we delete the cache entry here? - this.logger.info(`Cleaned up artifact ${diff.id}`); + await this.artifactClient.deleteArtifact(artifactId); + this.logger.info(`Cleaned up artifact ${artifactId}`); } catch (err) { errors.push(err); } @@ -157,14 +165,14 @@ export class ManifestManager { } /** - * Returns the last dispatched manifest based on the current state of the + * Returns the last computed manifest based on the state of the * user-artifact-manifest SO. * * @param schemaVersion The schema version of the manifest. - * @returns {Promise} The last dispatched manifest, or null if does not exist. + * @returns {Promise} The last computed manifest, or null if does not exist. * @throws Throws/rejects if there is an unexpected error retrieving the manifest. */ - public async getLastDispatchedManifest(schemaVersion: string): Promise { + public async getLastComputedManifest(schemaVersion: string): Promise { try { const manifestClient = this.getManifestClient(schemaVersion); const manifestSo = await manifestClient.getManifest(); @@ -173,11 +181,7 @@ export class ManifestManager { throw new Error('No version returned for manifest.'); } - const manifest = new Manifest( - new Date(manifestSo.attributes.created), - schemaVersion, - manifestSo.version - ); + const manifest = new Manifest(schemaVersion, manifestSo.version); for (const id of manifestSo.attributes.ids) { const artifactSo = await this.artifactClient.getArtifact(id); @@ -193,89 +197,42 @@ export class ManifestManager { } /** - * Snapshots a manifest based on current state of exception-list-agnostic SOs. + * Builds a new manifest based on the current user exception list. * - * @param opts Optional parameters for snapshot retrieval. - * @param opts.initialize Initialize a new Manifest when no manifest SO can be retrieved. - * @returns {Promise} A snapshot of the manifest, or null if not initialized. + * @param schemaVersion The schema version of the manifest. + * @param baselineManifest A baseline manifest to use for initializing pre-existing artifacts. + * @returns {Promise} A new Manifest object reprenting the current exception list. */ - public async getSnapshot(opts?: ManifestSnapshotOpts): Promise { - try { - let oldManifest: Manifest | null; - - // Get the last-dispatched manifest - oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); - - if (oldManifest === null && opts !== undefined && opts.initialize) { - oldManifest = new Manifest( - new Date(), - ManifestConstants.SCHEMA_VERSION, - ManifestConstants.INITIAL_VERSION - ); // create empty manifest - } else if (oldManifest == null) { - this.logger.debug('Manifest does not exist yet. Waiting...'); - return null; - } - - // Build new exception list artifacts - const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); - - // Build new manifest - const newManifest = Manifest.fromArtifacts( - artifacts, - ManifestConstants.SCHEMA_VERSION, - oldManifest - ); - - // Get diffs - const diffs = newManifest.diff(oldManifest); + public async buildNewManifest( + schemaVersion: string, + baselineManifest?: Manifest + ): Promise { + // Build new exception list artifacts + const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + + // Build new manifest + const manifest = Manifest.fromArtifacts( + artifacts, + ManifestConstants.SCHEMA_VERSION, + baselineManifest ?? Manifest.getDefault(schemaVersion) + ); - return { - manifest: newManifest, - diffs, - }; - } catch (err) { - this.logger.error(err); - return null; - } + return manifest; } /** - * Syncs artifacts based on provided snapshot. - * - * Creates artifacts that do not yet exist and cleans up old artifacts that have been - * superceded by this snapshot. + * Dispatches the manifest by writing it to the endpoint package config, if different + * from the manifest already in the config. * - * @param snapshot A ManifestSnapshot to use for sync. + * @param manifest The Manifest to dispatch. * @returns {Promise} Any errors encountered. */ - public async syncArtifacts( - snapshot: ManifestSnapshot, - diffType: 'add' | 'delete' - ): Promise { - const filteredDiffs = snapshot.diffs.filter((diff) => { - return diff.type === diffType; - }); - - const tmpSnapshot = { ...snapshot }; - tmpSnapshot.diffs = filteredDiffs; - - if (diffType === 'add') { - return this.writeArtifacts(tmpSnapshot); - } else if (diffType === 'delete') { - return this.deleteArtifacts(tmpSnapshot); + public async tryDispatch(manifest: Manifest): Promise { + const serializedManifest = manifest.toEndpointFormat(); + if (!manifestDispatchSchema.is(serializedManifest)) { + return [new Error('Invalid manifest')]; } - return [new Error(`Unsupported diff type: ${diffType}`)]; - } - - /** - * Dispatches the manifest by writing it to the endpoint package config. - * - * @param manifest The Manifest to dispatch. - * @returns {Promise} Any errors encountered. - */ - public async dispatch(manifest: Manifest): Promise { let paging = true; let page = 1; const errors: Error[] = []; @@ -293,16 +250,25 @@ export class ManifestManager { const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { value: {}, }; - artifactManifest.value = manifest.toEndpointFormat(); - newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; - - try { - await this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); - this.logger.debug( - `Updated package config ${id} with manifest version ${manifest.getVersion()}` - ); - } catch (err) { - errors.push(err); + + const oldManifest = + Manifest.fromPkgConfig(artifactManifest.value) ?? + Manifest.getDefault(ManifestConstants.SCHEMA_VERSION); + if (!manifest.equals(oldManifest)) { + newPackageConfig.inputs[0].config.artifact_manifest = { + value: serializedManifest, + }; + + try { + await this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); + this.logger.debug( + `Updated package config ${id} with manifest version ${manifest.getSha256()}` + ); + } catch (err) { + errors.push(err); + } + } else { + this.logger.debug(`No change in package config: ${id}`); } } else { errors.push(new Error(`Package config ${id} has no config.`)); @@ -317,46 +283,32 @@ export class ManifestManager { } /** - * Commits a manifest to indicate that it has been dispatched. + * Commits a manifest to indicate that a new version has been computed. * * @param manifest The Manifest to commit. - * @returns {Promise} An error if encountered, or null if successful. + * @returns {Promise} An error, if encountered, or null. */ public async commit(manifest: Manifest): Promise { try { const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); // Commit the new manifest - if (manifest.getVersion() === ManifestConstants.INITIAL_VERSION) { - await manifestClient.createManifest(manifest.toSavedObject()); + const manifestSo = manifest.toSavedObject(); + const version = manifest.getSoVersion(); + + if (version == null) { + await manifestClient.createManifest(manifestSo); } else { - const version = manifest.getVersion(); - if (version === ManifestConstants.INITIAL_VERSION) { - throw new Error('Updating existing manifest with baseline version. Bad state.'); - } - await manifestClient.updateManifest(manifest.toSavedObject(), { + await manifestClient.updateManifest(manifestSo, { version, }); } - this.logger.info(`Committed manifest ${manifest.getVersion()}`); + this.logger.info(`Committed manifest ${manifest.getSha256()}`); } catch (err) { return err; } return null; } - - /** - * Confirms that a packageConfig exists with provided name. - */ - public async confirmPackageConfigExists(name: string) { - // TODO: what if there are multiple results? uh oh. - const { total } = await this.packageConfigService.list(this.savedObjectsClient, { - page: 1, - perPage: 20, - kuery: `ingest-package-configs.name:${name}`, - }); - return total > 0; - } }