From 715d43be35cbb02748018ee9437dad5359b3459f Mon Sep 17 00:00:00 2001 From: John Schulz Date: Fri, 6 Nov 2020 15:15:22 -0500 Subject: [PATCH] [Ingest Manager] Move cache functions to from registry to archive (#82871) ## Summary Moving the memory store functions to `archive/cache.ts` to better express their role. They are archive-related functions. The registry is just one possible source of an archive/assets. Also considered moving to `assets`, but can always come back to that. `deletePackageCache` undoes side-effects from `unpackArchiveToCache` so put them in the same file --- .../epm/{registry => archive}/cache.ts | 2 +- .../server/services/epm/archive/index.ts | 22 ++++++++++++++++++- .../server/services/epm/archive/validation.ts | 2 +- .../services/epm/packages/assets.test.ts | 4 ++-- .../server/services/epm/packages/assets.ts | 2 +- .../server/services/epm/packages/remove.ts | 3 ++- .../server/services/epm/registry/index.ts | 20 +---------------- 7 files changed, 29 insertions(+), 26 deletions(-) rename x-pack/plugins/ingest_manager/server/services/epm/{registry => archive}/cache.ts (95%) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/registry/cache.ts b/x-pack/plugins/ingest_manager/server/services/epm/archive/cache.ts similarity index 95% rename from x-pack/plugins/ingest_manager/server/services/epm/registry/cache.ts rename to x-pack/plugins/ingest_manager/server/services/epm/archive/cache.ts index 695db9db73fa2..102324c18bd43 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/registry/cache.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/archive/cache.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { pkgToPkgKey } from './index'; +import { pkgToPkgKey } from '../registry/index'; const cache: Map = new Map(); export const cacheGet = (key: string) => cache.get(key); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/archive/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/archive/index.ts index ee505b205fc84..27451ed6b5e60 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/archive/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/archive/index.ts @@ -6,10 +6,18 @@ import { ArchivePackage } from '../../../../common/types'; import { PackageInvalidArchiveError, PackageUnsupportedMediaTypeError } from '../../../errors'; -import { cacheSet, setArchiveFilelist } from '../registry/cache'; +import { + cacheSet, + cacheDelete, + getArchiveFilelist, + setArchiveFilelist, + deleteArchiveFilelist, +} from './cache'; import { ArchiveEntry, getBufferExtractor } from '../registry/extract'; import { parseAndVerifyArchive } from './validation'; +export * from './cache'; + export async function loadArchivePackage({ archiveBuffer, contentType, @@ -64,3 +72,15 @@ export async function unpackArchiveToCache( } return paths; } + +export const deletePackageCache = (name: string, version: string) => { + // get cached archive filelist + const paths = getArchiveFilelist(name, version); + + // delete cached archive filelist + deleteArchiveFilelist(name, version); + + // delete cached archive files + // this has been populated in unpackArchiveToCache() + paths?.forEach((path) => cacheDelete(path)); +}; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/archive/validation.ts b/x-pack/plugins/ingest_manager/server/services/epm/archive/validation.ts index e83340124a2d0..90941aaf80cdd 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/archive/validation.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/archive/validation.ts @@ -16,7 +16,7 @@ import { } from '../../../../common/types'; import { PackageInvalidArchiveError } from '../../../errors'; import { pkgToPkgKey } from '../registry'; -import { cacheGet } from '../registry/cache'; +import { cacheGet } from './cache'; // TODO: everything below performs verification of manifest.yml files, and hence duplicates functionality already implemented in the // package registry. At some point this should probably be replaced (or enhanced) with verification based on diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.test.ts index eb43bef72db70..ab93a73a55f39 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.test.ts @@ -6,9 +6,9 @@ import { InstallablePackage } from '../../../types'; import { getAssets } from './assets'; -import { getArchiveFilelist } from '../registry/cache'; +import { getArchiveFilelist } from '../archive/cache'; -jest.mock('../registry/cache', () => { +jest.mock('../archive/cache', () => { return { getArchiveFilelist: jest.fn(), }; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.ts index 856f04c0c9b67..2e2090312c9ae 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/assets.ts @@ -6,7 +6,7 @@ import { InstallablePackage } from '../../../types'; import * as Registry from '../registry'; -import { getArchiveFilelist } from '../registry/cache'; +import { getArchiveFilelist } from '../archive/cache'; // paths from RegistryPackage are routes to the assets on EPR // e.g. `/package/nginx/1.2.0/data_stream/access/fields/fields.yml` diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts index 5db47adc983c2..9fabbaf72474e 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts @@ -21,7 +21,8 @@ import { deletePipeline } from '../elasticsearch/ingest_pipeline/'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import { deleteTransforms } from '../elasticsearch/transform/remove'; import { packagePolicyService, appContextService } from '../..'; -import { splitPkgKey, deletePackageCache } from '../registry'; +import { splitPkgKey } from '../registry'; +import { deletePackageCache } from '../archive'; export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts index b1dd9a8c3c3f1..013315eb6da23 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts @@ -19,13 +19,7 @@ import { RegistrySearchResult, } from '../../../types'; import { unpackArchiveToCache } from '../archive'; -import { - cacheGet, - cacheDelete, - getArchiveFilelist, - setArchiveFilelist, - deleteArchiveFilelist, -} from './cache'; +import { cacheGet, getArchiveFilelist, setArchiveFilelist } from '../archive'; import { ArchiveEntry } from './extract'; import { fetchUrl, getResponse, getResponseStream } from './requests'; import { streamToBuffer } from './streams'; @@ -247,15 +241,3 @@ export function groupPathsByService(paths: string[]): AssetsGroupedByServiceByTy // elasticsearch: assets.elasticsearch, }; } - -export const deletePackageCache = (name: string, version: string) => { - // get cached archive filelist - const paths = getArchiveFilelist(name, version); - - // delete cached archive filelist - deleteArchiveFilelist(name, version); - - // delete cached archive files - // this has been populated in unpackRegistryPackageToCache() - paths?.forEach((path) => cacheDelete(path)); -};