From 3ac3ad08cf7d23ed3d7461587d6a441937f5b0ff Mon Sep 17 00:00:00 2001 From: Craig Furman Date: Mon, 13 Jun 2022 14:45:21 +0100 Subject: [PATCH 1/4] fix: support HTTP(S) proxies in iac-test Configure HTTP clients to respect HTTP(S)_PROXY environment variables. Ignore TLS identity verification errors (this is dangerous) when the `--insecure` flag is passed, matching the behaviour of `snyk test`. --- .../test/iac/local-execution/local-cache.ts | 10 +++-- src/lib/request/request.ts | 39 +++++++++++++++++-- test/jest/unit/iac/local-cache.spec.ts | 17 +++++--- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/cli/commands/test/iac/local-execution/local-cache.ts b/src/cli/commands/test/iac/local-execution/local-cache.ts index 9eb136d8b7..8e2e015ab6 100644 --- a/src/cli/commands/test/iac/local-execution/local-cache.ts +++ b/src/cli/commands/test/iac/local-execution/local-cache.ts @@ -1,15 +1,14 @@ import * as path from 'path'; import * as fs from 'fs'; import { EngineType, IaCErrorCodes } from './types'; -import * as needle from 'needle'; import * as rimraf from 'rimraf'; import { createIacDir, extractBundle, isValidBundle } from './file-utils'; import * as Debug from 'debug'; import { CustomError } from '../../../../../lib/errors'; import * as analytics from '../../../../../lib/analytics'; -import ReadableStream = NodeJS.ReadableStream; import { getErrorStringCode } from './error-utils'; import config from '../../../../../lib/config'; +import { streamRequest } from '../../../../../lib/request/request'; const debug = Debug('iac-local-cache'); @@ -141,7 +140,12 @@ export async function initLocalCache({ // always overwrite whatever might be there. try { const BUNDLE_URL = 'https://static.snyk.io/cli/wasm/bundle.tar.gz'; - const response: ReadableStream = needle.get(BUNDLE_URL); + const response = await streamRequest({ + method: 'get', + url: BUNDLE_URL, + body: null, + headers: {}, + }); await extractBundle(response); } catch (e) { throw new FailedToDownloadRulesError(); diff --git a/src/lib/request/request.ts b/src/lib/request/request.ts index 9940886c82..63581c6cc2 100644 --- a/src/lib/request/request.ts +++ b/src/lib/request/request.ts @@ -17,9 +17,7 @@ const snykDebug = debugModule('snyk'); declare const global: Global; -export async function makeRequest( - payload: Payload, -): Promise<{ res: needle.NeedleResponse; body: any }> { +function setupRequest(payload: Payload) { // This ensures we support lowercase http(s)_proxy values as well // The weird IF around it ensures we don't create an envvar with a value of undefined, which throws error when trying to use it as a proxy if (process.env.HTTP_PROXY || process.env.http_proxy) { @@ -129,6 +127,14 @@ export async function makeRequest( options.rejectUnauthorized = false; } + return { method, url, data, options }; +} + +export async function makeRequest( + payload: Payload, +): Promise<{ res: needle.NeedleResponse; body: any }> { + const { method, url, data, options } = setupRequest(payload); + return new Promise((resolve, reject) => { needle.request(method, url, data, options, (err, res, respBody) => { debug(err); @@ -145,3 +151,30 @@ export async function makeRequest( }); }); } + +export async function streamRequest( + payload: Payload, +): Promise { + const { method, url, data, options } = setupRequest(payload); + + try { + const result = await needle.request(method, url, data, options); + const statusCode = await getStatusCode(result); + debug('response (%s): ', statusCode); + return result; + } catch (e) { + debug(e); + throw e; + } +} + +async function getStatusCode(stream: needle.ReadableStream): Promise { + return new Promise((resolve, reject) => { + stream.on('header', (statusCode: number) => { + resolve(statusCode); + }); + stream.on('err', (err: Error) => { + reject(err); + }); + }); +} diff --git a/test/jest/unit/iac/local-cache.spec.ts b/test/jest/unit/iac/local-cache.spec.ts index e92d7232e4..0f5e1a90a3 100644 --- a/test/jest/unit/iac/local-cache.spec.ts +++ b/test/jest/unit/iac/local-cache.spec.ts @@ -5,10 +5,10 @@ import { } from '../../../../src/cli/commands/test/iac/local-execution/local-cache'; import * as fileUtilsModule from '../../../../src/cli/commands/test/iac/local-execution/file-utils'; import { PassThrough } from 'stream'; -import * as needle from 'needle'; import * as rimraf from 'rimraf'; import * as fs from 'fs'; import * as path from 'path'; +import * as request from '../../../../src/lib/request/request'; describe('initLocalCache - downloads bundle successfully', () => { beforeEach(() => { @@ -24,12 +24,17 @@ describe('initLocalCache - downloads bundle successfully', () => { .spyOn(fileUtilsModule, 'extractBundle') .mockResolvedValue(); jest.spyOn(fileUtilsModule, 'createIacDir').mockImplementation(() => null); - jest.spyOn(needle, 'get').mockReturnValue(mockReadable); + jest + .spyOn(request, 'streamRequest') + .mockReturnValue(Promise.resolve(mockReadable)); await localCacheModule.initLocalCache(); - expect(needle.get).toHaveBeenCalledWith( - expect.stringContaining('bundle.tar.gz'), + expect(request.streamRequest).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'get', + url: expect.stringContaining('bundle.tar.gz'), + }), ); expect(spy).toHaveBeenCalledWith(mockReadable); }); @@ -43,7 +48,9 @@ describe('initLocalCache - downloads bundle successfully', () => { .mockResolvedValue(); jest.spyOn(fileUtilsModule, 'isValidBundle').mockReturnValue(true); jest.spyOn(fileUtilsModule, 'createIacDir').mockImplementation(() => null); - jest.spyOn(needle, 'get').mockReturnValue(new PassThrough()); + jest + .spyOn(request, 'streamRequest') + .mockReturnValue(Promise.resolve(new PassThrough())); jest.spyOn(fs, 'createReadStream').mockReturnValue(mockReadable); await localCacheModule.initLocalCache({ From 83b4f6ae240750ddad824c8d9427db8601b4cd4d Mon Sep 17 00:00:00 2001 From: David Agrest Date: Sun, 19 Jun 2022 12:50:22 +0300 Subject: [PATCH 2/4] feat: support for unmanaged snyk-to-html --- src/lib/ecosystems/resolve-test-facts.ts | 23 +++++++++++++++++++ src/lib/package-managers.ts | 4 +++- src/lib/polling/polling-test.ts | 8 +++++++ src/lib/snyk-test/legacy.ts | 8 ++++++- .../lib/ecosystems/resolve-test-facts.spec.ts | 8 +++++++ 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/lib/ecosystems/resolve-test-facts.ts b/src/lib/ecosystems/resolve-test-facts.ts index 0a13633725..c4b4610d9c 100644 --- a/src/lib/ecosystems/resolve-test-facts.ts +++ b/src/lib/ecosystems/resolve-test-facts.ts @@ -8,6 +8,7 @@ import { import { extractAndApplyPluginAnalytics } from './plugin-analytics'; import { findAndLoadPolicy } from '../policy'; import { filterIgnoredIssues } from './policy'; +import { IssueData, Issue } from '../snyk-test/legacy'; export async function resolveAndTestFacts( ecosystem: Ecosystem, @@ -18,6 +19,7 @@ export async function resolveAndTestFacts( ): Promise<[TestResult[], string[]]> { const results: any[] = []; const errors: string[] = []; + const packageManager = 'Unmanaged (C/C++)'; for (const [path, scanResults] of Object.entries(scans)) { await spinner(`Resolving and Testing fileSignatures in ${path}`); @@ -45,12 +47,33 @@ export async function resolveAndTestFacts( policy, ); + const issuesMap: Map = new Map(); + response.issues.forEach((i) => { + issuesMap[i.issueId] = i; + }); + + const vulnerabilities: IssueData[] = []; + for (const issuesDataKey in response.issuesData) { + const issueData = response.issuesData[issuesDataKey]; + const pkgCoordinate = `${issuesMap[issuesDataKey].pkgName}@${issuesMap[issuesDataKey].pkgVersion}`; + issueData.from = [pkgCoordinate]; + issueData.name = pkgCoordinate; + issueData.packageManager = packageManager; + vulnerabilities.push(issueData); + } + + const dependencyCount = response.issues.length; + results.push({ issues, issuesData, depGraphData: response?.depGraphData, depsFilePaths: response?.depsFilePaths, fileSignaturesDetails: response?.fileSignaturesDetails, + vulnerabilities, + path, + dependencyCount, + packageManager, }); } catch (error) { const hasStatusCodeError = error.code >= 400 && error.code <= 500; diff --git a/src/lib/package-managers.ts b/src/lib/package-managers.ts index 65e36b7575..69cd2cf0f3 100644 --- a/src/lib/package-managers.ts +++ b/src/lib/package-managers.ts @@ -14,7 +14,8 @@ export type SupportedPackageManagers = | 'composer' | 'cocoapods' | 'poetry' - | 'hex'; + | 'hex' + | 'Unmanaged (C/C++)'; export enum SUPPORTED_MANIFEST_FILES { GEMFILE = 'Gemfile', @@ -67,6 +68,7 @@ export const SUPPORTED_PACKAGE_MANAGER_NAME: { cocoapods: 'CocoaPods', poetry: 'Poetry', hex: 'Hex', + 'Unmanaged (C/C++)': 'Unmanaged (C/C++)', }; export const GRAPH_SUPPORTED_PACKAGE_MANAGERS: SupportedPackageManagers[] = [ diff --git a/src/lib/polling/polling-test.ts b/src/lib/polling/polling-test.ts index be592971a2..cd7efe4db6 100644 --- a/src/lib/polling/polling-test.ts +++ b/src/lib/polling/polling-test.ts @@ -63,6 +63,10 @@ export async function pollingTestWithTokenUntilDone( depGraphData, depsFilePaths, fileSignaturesDetails, + vulnerabilities, + path, + dependencyCount, + packageManager, } = response.result; return { issues, @@ -70,6 +74,10 @@ export async function pollingTestWithTokenUntilDone( depGraphData, depsFilePaths, fileSignaturesDetails, + vulnerabilities, + path, + dependencyCount, + packageManager, }; } diff --git a/src/lib/snyk-test/legacy.ts b/src/lib/snyk-test/legacy.ts index ce1e439b0d..41914274b6 100644 --- a/src/lib/snyk-test/legacy.ts +++ b/src/lib/snyk-test/legacy.ts @@ -92,6 +92,8 @@ export interface IssueData { legalInstructions?: string; reachability?: REACHABILITY; packageManager?: SupportedProjectTypes; + from?: string[]; + name?: string; } export type CallPath = string[]; @@ -235,7 +237,7 @@ interface TestDepGraphResult { remediation?: RemediationChanges; } -interface Issue { +export interface Issue { pkgName: string; pkgVersion?: string; issueId: string; @@ -256,6 +258,10 @@ export interface TestDependenciesResult { depsFilePaths?: DepsFilePaths; depGraphData: depGraphLib.DepGraphData; fileSignaturesDetails: FileSignaturesDetails; + vulnerabilities: IssueData[]; + path: string; + dependencyCount: number; + packageManager: SupportedProjectTypes; } export interface TestDepGraphMeta { diff --git a/test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts b/test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts index 14ff84859a..d66e2c72ac 100644 --- a/test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts +++ b/test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts @@ -108,6 +108,10 @@ describe('resolve and test facts', () => { issues: [], depGraphData, fileSignaturesDetails: {}, + vulnerabilities: [], + path: 'path', + dependencyCount: 0, + packageManager: 'Unmanaged (C/C++)', }); const extractAndApplyPluginAnalyticsSpy = jest.spyOn( @@ -138,6 +142,10 @@ describe('resolve and test facts', () => { issues: [], depGraphData, fileSignaturesDetails: {}, + vulnerabilities: [], + path: 'path', + dependencyCount: 0, + packageManager: 'Unmanaged (C/C++)', }, ]); expect(errors).toEqual([]); From 9da155ce20dd99108b4374d8c81485a561fdba42 Mon Sep 17 00:00:00 2001 From: Ofek Atar Date: Mon, 20 Jun 2022 11:36:41 +0300 Subject: [PATCH 3/4] refactor: IaC test v2 setup --- src/cli/commands/test/iac/v2/index.ts | 3 +- src/lib/iac/drift/driftctl.ts | 24 +-- src/lib/iac/file-utils.ts | 16 ++ src/lib/iac/test/v2/setup/index.ts | 7 +- .../iac/test/v2/setup/local-cache/index.ts | 19 +++ .../policy-engine/constants/index.ts | 11 ++ .../policy-engine/constants/utils.ts | 19 +++ .../setup/local-cache/policy-engine/index.ts | 13 ++ .../policy-engine/lookup-local.ts} | 34 ++-- .../test/v2/setup/{ => local-cache}/rules.ts | 18 +- src/lib/iac/test/v2/types.ts | 3 +- test/jest/unit/lib/iac/drift.spec.ts | 18 +- .../test/v2/setup/local-cache/index.spec.ts | 156 ++++++++++++++++++ .../policy-engine/constants/utils.spec.ts | 35 ++++ .../policy-engine/lookup-local.spec.ts} | 43 +++-- .../v2/setup/{ => local-cache}/rules.spec.ts | 2 +- 16 files changed, 344 insertions(+), 77 deletions(-) create mode 100644 src/lib/iac/test/v2/setup/local-cache/index.ts create mode 100644 src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts create mode 100644 src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.ts create mode 100644 src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts rename src/lib/iac/test/v2/setup/{policy-engine.ts => local-cache/policy-engine/lookup-local.ts} (66%) rename src/lib/iac/test/v2/setup/{ => local-cache}/rules.ts (81%) create mode 100644 test/jest/unit/lib/iac/test/v2/setup/local-cache/index.spec.ts create mode 100644 test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.spec.ts rename test/jest/unit/lib/iac/test/v2/setup/{policy-engine.spec.ts => local-cache/policy-engine/lookup-local.spec.ts} (63%) rename test/jest/unit/lib/iac/test/v2/setup/{ => local-cache}/rules.spec.ts (95%) diff --git a/src/cli/commands/test/iac/v2/index.ts b/src/cli/commands/test/iac/v2/index.ts index a6ada71cee..882836542a 100644 --- a/src/cli/commands/test/iac/v2/index.ts +++ b/src/cli/commands/test/iac/v2/index.ts @@ -29,9 +29,8 @@ function prepareTestConfig(paths: string[]): TestConfig { return { paths, - cachedBundlePath: pathLib.join(iacCachePath, 'bundle.tar.gz'), + iacCachePath, userBundlePath: config.IAC_BUNDLE_PATH, - cachedPolicyEnginePath: pathLib.join(iacCachePath, 'snyk-iac-test'), userPolicyEnginePath: config.IAC_POLICY_ENGINE_PATH, }; } diff --git a/src/lib/iac/drift/driftctl.ts b/src/lib/iac/drift/driftctl.ts index ee9f69a84f..23aaee8152 100644 --- a/src/lib/iac/drift/driftctl.ts +++ b/src/lib/iac/drift/driftctl.ts @@ -23,7 +23,7 @@ import * as child_process from 'child_process'; import * as fs from 'fs'; import * as os from 'os'; import * as crypto from 'crypto'; -import { isExe } from '../file-utils'; +import { createDirIfNotExists, isExe } from '../file-utils'; const debug = debugLib('driftctl'); @@ -71,12 +71,12 @@ const driftctlDefaultOptions = ['--no-version-check']; let isBinaryDownloaded = false; -export const generateArgs = ( +export const generateArgs = async ( options: DriftCTLOptions, driftIgnore?: string[], -): string[] => { +): Promise => { if (options.kind === 'describe') { - return generateScanFlags(options as DescribeOptions, driftIgnore); + return await generateScanFlags(options as DescribeOptions, driftIgnore); } if (options.kind === 'fmt') { @@ -107,10 +107,10 @@ const generateFmtFlags = (options: FmtOptions): string[] => { return args; }; -const generateScanFlags = ( +const generateScanFlags = async ( options: DescribeOptions, driftIgnore?: string[], -): string[] => { +): Promise => { const args: string[] = ['scan', ...driftctlDefaultOptions]; if (options.quiet) { @@ -177,7 +177,7 @@ const generateScanFlags = ( } let configDir = cachePath; - createIfNotExists(cachePath); + await createDirIfNotExists(cachePath); if (options['config-dir']) { configDir = options['config-dir']; } @@ -238,7 +238,7 @@ export const runDriftCTL = async ({ }): Promise => { const path = await findOrDownload(); await validateArgs(options); - const args = generateArgs(options, driftIgnore); + const args = await generateArgs(options, driftIgnore); if (!stdio) { stdio = ['pipe', 'pipe', 'inherit']; @@ -281,7 +281,7 @@ async function findOrDownload(): Promise { if (dctl === '') { binaryExist = false; try { - createIfNotExists(cachePath); + await createDirIfNotExists(cachePath); dctl = driftctlPath; const duration = new TimerMetricInstance('driftctl_download'); @@ -424,9 +424,3 @@ function driftctlUrl(): string { return `${dctlBaseUrl}/${driftctlVersion}/${driftctlFileName()}`; } - -function createIfNotExists(path: string) { - if (!fs.existsSync(path)) { - fs.mkdirSync(path, { recursive: true }); - } -} diff --git a/src/lib/iac/file-utils.ts b/src/lib/iac/file-utils.ts index e936c741d6..e7838d8ee7 100644 --- a/src/lib/iac/file-utils.ts +++ b/src/lib/iac/file-utils.ts @@ -9,3 +9,19 @@ export async function isExe(path: string): Promise { return false; } } + +export async function isExists(path: string): Promise { + try { + await fsPromises.stat(path); + return true; + } catch (err) { + return false; + } +} + +export async function createDirIfNotExists(path: string): Promise { + const isDirExists = await isExists(path); + if (!isDirExists) { + fsPromises.mkdir(path, { recursive: true }); + } +} diff --git a/src/lib/iac/test/v2/setup/index.ts b/src/lib/iac/test/v2/setup/index.ts index 58c28c86a3..a6929236d1 100644 --- a/src/lib/iac/test/v2/setup/index.ts +++ b/src/lib/iac/test/v2/setup/index.ts @@ -1,9 +1,6 @@ import { TestConfig } from '../types'; -import { initRules } from './rules'; -import { initPolicyEngine } from './policy-engine'; +import { initLocalCache } from './local-cache'; export async function setup(testConfig: TestConfig) { - const policyEnginePath = await initPolicyEngine(testConfig); - const rulesBundlePath = await initRules(testConfig); - return { policyEnginePath, rulesBundlePath }; + return await initLocalCache(testConfig); } diff --git a/src/lib/iac/test/v2/setup/local-cache/index.ts b/src/lib/iac/test/v2/setup/local-cache/index.ts new file mode 100644 index 0000000000..e595deb90f --- /dev/null +++ b/src/lib/iac/test/v2/setup/local-cache/index.ts @@ -0,0 +1,19 @@ +import { TestConfig } from '../../types'; +import { initRules } from './rules'; +import { initPolicyEngine } from './policy-engine'; +import { createDirIfNotExists } from '../../../../file-utils'; +import { CustomError } from '../../../../../errors'; +import { FailedToInitLocalCacheError } from '../../../../../../cli/commands/test/iac/local-execution/local-cache'; + +export async function initLocalCache(testConfig: TestConfig) { + try { + await createDirIfNotExists(testConfig.iacCachePath); + + const policyEnginePath = await initPolicyEngine(testConfig); + const rulesBundlePath = await initRules(testConfig); + + return { policyEnginePath, rulesBundlePath }; + } catch (err) { + throw err instanceof CustomError ? err : new FailedToInitLocalCacheError(); + } +} diff --git a/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts b/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts new file mode 100644 index 0000000000..fc7cf10891 --- /dev/null +++ b/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts @@ -0,0 +1,11 @@ +import { formatPolicyEngineFileName } from './utils'; + +/** + * The Policy Engine release version associated with this Snyk CLI version. + */ +export const releaseVersion = '0.1.0'; + +/** + * The Policy Engine executable's file name. + */ +export const policyEngineFileName = formatPolicyEngineFileName(releaseVersion); diff --git a/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.ts b/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.ts new file mode 100644 index 0000000000..b7f612939c --- /dev/null +++ b/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.ts @@ -0,0 +1,19 @@ +import * as os from 'os'; + +export function formatPolicyEngineFileName(releaseVersion: string) { + let platform = 'Linux'; + switch (os.platform()) { + case 'darwin': + platform = 'Darwin'; + break; + case 'win32': + platform = 'Windows'; + break; + } + + const arch = os.arch() === 'arm64' ? 'arm64' : 'x86_64'; + + const execExt = os.platform() === 'win32' ? '.exe' : ''; + + return `snyk-iac-test_${releaseVersion}_${platform}_${arch}${execExt}`; +} diff --git a/src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts b/src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts new file mode 100644 index 0000000000..989fac8a90 --- /dev/null +++ b/src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts @@ -0,0 +1,13 @@ +import { TestConfig } from '../../../types'; +import { InvalidUserPolicyEnginePathError, lookupLocal } from './lookup-local'; + +export async function initPolicyEngine(testConfig: TestConfig) { + const localPolicyEnginePath = await lookupLocal(testConfig); + if (localPolicyEnginePath) { + return localPolicyEnginePath; + } + + // TODO: Download Policy Engine executable + + throw new InvalidUserPolicyEnginePathError('', 'policy engine not found'); +} diff --git a/src/lib/iac/test/v2/setup/policy-engine.ts b/src/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local.ts similarity index 66% rename from src/lib/iac/test/v2/setup/policy-engine.ts rename to src/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local.ts index 4e3388845e..9dac26d9e6 100644 --- a/src/lib/iac/test/v2/setup/policy-engine.ts +++ b/src/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local.ts @@ -1,9 +1,11 @@ +import * as pathLib from 'path'; import * as createDebugLogger from 'debug'; -import { isExe } from '../../../file-utils'; -import { CustomError } from '../../../../errors'; -import { IaCErrorCodes } from '../../../../../cli/commands/test/iac/local-execution/types'; -import { getErrorStringCode } from '../../../../../cli/commands/test/iac/local-execution/error-utils'; -import { TestConfig } from '../types'; +import { isExe } from '../../../../../file-utils'; +import { CustomError } from '../../../../../../errors'; +import { IaCErrorCodes } from '../../../../../../../cli/commands/test/iac/local-execution/types'; +import { getErrorStringCode } from '../../../../../../../cli/commands/test/iac/local-execution/error-utils'; +import { TestConfig } from '../../../types'; +import { policyEngineFileName } from './constants'; const debugLogger = createDebugLogger('snyk-iac'); @@ -22,8 +24,8 @@ export class InvalidUserPolicyEnginePathError extends CustomError { } } -export async function lookupLocalPolicyEngine({ - cachedPolicyEnginePath, +export async function lookupLocal({ + iacCachePath, userPolicyEnginePath, }: TestConfig): Promise { // Lookup in custom path. @@ -41,6 +43,10 @@ export async function lookupLocalPolicyEngine({ } // Lookup in cache. else { + const cachedPolicyEnginePath = pathLib.join( + iacCachePath, + policyEngineFileName, + ); if (await isExe(cachedPolicyEnginePath)) { debugLogger( 'Found cached Policy Engine executable: %s', @@ -55,17 +61,3 @@ export async function lookupLocalPolicyEngine({ } } } - -export async function initPolicyEngine( - testConfig: TestConfig, -): Promise { - const localPolicyEnginePath = await lookupLocalPolicyEngine(testConfig); - - if (localPolicyEnginePath) { - return localPolicyEnginePath; - } - - // TODO: Download Policy Engine executable - - throw new InvalidUserPolicyEnginePathError('', 'policy engine not found'); -} diff --git a/src/lib/iac/test/v2/setup/rules.ts b/src/lib/iac/test/v2/setup/local-cache/rules.ts similarity index 81% rename from src/lib/iac/test/v2/setup/rules.ts rename to src/lib/iac/test/v2/setup/local-cache/rules.ts index 72682567b5..885d881729 100644 --- a/src/lib/iac/test/v2/setup/rules.ts +++ b/src/lib/iac/test/v2/setup/local-cache/rules.ts @@ -1,13 +1,21 @@ +import * as pathLib from 'path'; import * as fs from 'fs'; import * as tar from 'tar'; -import { CustomError } from '../../../../errors'; -import { getErrorStringCode } from '../../../../../cli/commands/test/iac/local-execution/error-utils'; -import { IaCErrorCodes } from '../../../../../cli/commands/test/iac/local-execution/types'; -import { TestConfig } from '../types'; +import { CustomError } from '../../../../../errors'; +import { getErrorStringCode } from '../../../../../../cli/commands/test/iac/local-execution/error-utils'; +import { IaCErrorCodes } from '../../../../../../cli/commands/test/iac/local-execution/types'; +import { TestConfig } from '../../types'; + +export const rulesBundleName = 'bundle.tar.gz'; export async function initRules(testConfig: TestConfig): Promise { + const cachedBundlePath = pathLib.join( + testConfig.iacCachePath, + rulesBundleName, + ); + const bundleLocator = new RulesBundleLocator( - testConfig.cachedBundlePath, + cachedBundlePath, testConfig.userBundlePath, ); diff --git a/src/lib/iac/test/v2/types.ts b/src/lib/iac/test/v2/types.ts index b095ec2879..0c5a3adacb 100644 --- a/src/lib/iac/test/v2/types.ts +++ b/src/lib/iac/test/v2/types.ts @@ -1,7 +1,6 @@ export interface TestConfig { paths: string[]; - cachedBundlePath: string; - cachedPolicyEnginePath: string; + iacCachePath: string; userBundlePath?: string; userPolicyEnginePath?: string; } diff --git a/test/jest/unit/lib/iac/drift.spec.ts b/test/jest/unit/lib/iac/drift.spec.ts index 7aaac6049b..71b542fbad 100644 --- a/test/jest/unit/lib/iac/drift.spec.ts +++ b/test/jest/unit/lib/iac/drift.spec.ts @@ -38,8 +38,8 @@ describe('driftctl integration', () => { mockFs.restore(); }); - it('describe: default arguments are correct', () => { - const args = generateArgs({ kind: 'describe' }, []); + it('describe: default arguments are correct', async () => { + const args = await generateArgs({ kind: 'describe' }, []); expect(args).toEqual([ 'scan', '--no-version-check', @@ -52,9 +52,9 @@ describe('driftctl integration', () => { ]); }); - it('describe: --all enable deep mode', () => { + it('describe: --all enable deep mode', async () => { { - const args = generateArgs( + const args = await generateArgs( { kind: 'describe', all: true } as DescribeOptions, [], ); @@ -72,7 +72,7 @@ describe('driftctl integration', () => { } { - const args = generateArgs( + const args = await generateArgs( { kind: 'describe', all: true, deep: true } as DescribeOptions, [], ); @@ -90,8 +90,8 @@ describe('driftctl integration', () => { } }); - it('describe: passing options generate correct arguments', () => { - const args = generateArgs( + it('describe: passing options generate correct arguments', async () => { + const args = await generateArgs( { kind: 'describe', 'config-dir': 'confdir', @@ -147,8 +147,8 @@ describe('driftctl integration', () => { ]); }); - it('describe: from arguments is a coma separated list', () => { - const args = generateArgs({ + it('describe: from arguments is a coma separated list', async () => { + const args = await generateArgs({ kind: 'describe', from: 'path1,path2,path3', } as DescribeOptions); diff --git a/test/jest/unit/lib/iac/test/v2/setup/local-cache/index.spec.ts b/test/jest/unit/lib/iac/test/v2/setup/local-cache/index.spec.ts new file mode 100644 index 0000000000..6a13433caa --- /dev/null +++ b/test/jest/unit/lib/iac/test/v2/setup/local-cache/index.spec.ts @@ -0,0 +1,156 @@ +import * as pathLib from 'path'; +import * as initPolicyEngineLib from '../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine'; +import * as initRulesLib from '../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/rules'; +import * as fileUtils from '../../../../../../../../../src/lib/iac/file-utils'; +import { initLocalCache } from '../../../../../../../../../src/lib/iac/test/v2/setup/local-cache'; +import { TestConfig } from '../../../../../../../../../src/lib/iac/test/v2/types'; +import { FailedToInitLocalCacheError } from '../../../../../../../../../src/cli/commands/test/iac/local-execution/local-cache'; + +describe('initLocalCache', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('creates the IaC cache directory if it does not exist', async () => { + // Arrange + const testPolicyEnginePath = 'test-policy-engine-path'; + const testRulesBundlePath = 'test-rules-bundle-path'; + const testTestConfig = { + iacCachePath: pathLib.join('iac', 'cache', 'path'), + } as TestConfig; + + jest + .spyOn(initPolicyEngineLib, 'initPolicyEngine') + .mockImplementation(async () => testPolicyEnginePath); + jest + .spyOn(initRulesLib, 'initRules') + .mockImplementation(async () => testRulesBundlePath); + const createDirIfNotExistsSpy = jest + .spyOn(fileUtils, 'createDirIfNotExists') + .mockImplementation(async () => undefined); + + // Act + await initLocalCache(testTestConfig); + + // Assert + expect(createDirIfNotExistsSpy).toHaveBeenCalledWith( + testTestConfig.iacCachePath, + ); + }); + + it('initializes the Policy Engine executable', async () => { + // Arrange + const testPolicyEnginePath = 'test-policy-engine-path'; + const testRulesBundlePath = 'test-rules-bundle-path'; + const testTestConfig = { + iacCachePath: pathLib.join('iac', 'cache', 'path'), + } as TestConfig; + + const initPolicyEngineSpy = jest + .spyOn(initPolicyEngineLib, 'initPolicyEngine') + .mockImplementation(async () => testPolicyEnginePath); + jest + .spyOn(initRulesLib, 'initRules') + .mockImplementation(async () => testRulesBundlePath); + jest + .spyOn(fileUtils, 'createDirIfNotExists') + .mockImplementation(async () => undefined); + + // Act + await initLocalCache(testTestConfig); + + // Assert + expect(initPolicyEngineSpy).toHaveBeenCalledWith(testTestConfig); + }); + + it('initializes the rules bundle', async () => { + // Arrange + const testPolicyEnginePath = 'test-policy-engine-path'; + const testRulesBundlePath = 'test-rules-bundle-path'; + const testTestConfig = { + iacCachePath: pathLib.join('iac', 'cache', 'path'), + } as TestConfig; + + jest + .spyOn(initPolicyEngineLib, 'initPolicyEngine') + .mockImplementation(async () => testPolicyEnginePath); + const initRulesSpy = jest + .spyOn(initRulesLib, 'initRules') + .mockImplementation(async () => testRulesBundlePath); + jest + .spyOn(fileUtils, 'createDirIfNotExists') + .mockImplementation(async () => undefined); + + // Act + await initLocalCache(testTestConfig); + + // Assert + expect(initRulesSpy).toHaveBeenCalledWith(testTestConfig); + }); + + it('returns the cached resrouce paths', async () => { + // Arrange + const testPolicyEnginePath = 'test-policy-engine-path'; + const testRulesBundlePath = 'test-rules-bundle-path'; + const testTestConfig = { + iacCachePath: pathLib.join('iac', 'cache', 'path'), + } as TestConfig; + + jest + .spyOn(initPolicyEngineLib, 'initPolicyEngine') + .mockImplementation(async () => testPolicyEnginePath); + jest + .spyOn(initRulesLib, 'initRules') + .mockImplementation(async () => testRulesBundlePath); + jest + .spyOn(fileUtils, 'createDirIfNotExists') + .mockImplementation(async () => undefined); + + const expected = { + policyEnginePath: testPolicyEnginePath, + rulesBundlePath: testRulesBundlePath, + }; + // Act + const res = await initLocalCache(testTestConfig); + + // Assert + expect(res).toStrictEqual(expected); + }); + + describe.each` + failingResource | module | methodName + ${'cache directory'} | ${fileUtils} | ${'createDirIfNotExists'} + ${'Policy Engine executable'} | ${initPolicyEngineLib} | ${'initPolicyEngine'} + ${'rules bundle'} | ${initRulesLib} | ${'initRules'} + `( + 'when the initialization for the $failingResource fails', + ({ module, methodName }) => { + it('throws an error', async () => { + // Arrange + const testPolicyEnginePath = 'test-policy-engine-path'; + const testRulesBundlePath = 'test-rules-bundle-path'; + const testTestConfig = { + iacCachePath: pathLib.join('iac', 'cache', 'path'), + } as TestConfig; + + jest + .spyOn(initPolicyEngineLib, 'initPolicyEngine') + .mockImplementation(async () => testPolicyEnginePath); + jest + .spyOn(initRulesLib, 'initRules') + .mockImplementation(async () => testRulesBundlePath); + jest + .spyOn(fileUtils, 'createDirIfNotExists') + .mockImplementation(async () => undefined); + jest.spyOn(module, methodName).mockImplementation(async () => { + throw new FailedToInitLocalCacheError(); + }); + + // Act + Assert + await expect(initLocalCache(testTestConfig)).rejects.toThrow( + FailedToInitLocalCacheError, + ); + }); + }, + ); +}); diff --git a/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.spec.ts b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.spec.ts new file mode 100644 index 0000000000..175b895ecf --- /dev/null +++ b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.spec.ts @@ -0,0 +1,35 @@ +import * as os from 'os'; +import { formatPolicyEngineFileName } from '../../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils'; + +describe('formatPolicyEngineFileName', () => { + const testReleaseVersion = '6.6.6'; + + afterEach(() => { + jest.restoreAllMocks(); + }); + + describe.each` + osPlatform | osArch | expectedFileName + ${'darwin'} | ${'arm64'} | ${`snyk-iac-test_${testReleaseVersion}_Darwin_arm64`} + ${'darwin'} | ${'any-other-arch'} | ${`snyk-iac-test_${testReleaseVersion}_Darwin_x86_64`} + ${'win32'} | ${'arm64'} | ${`snyk-iac-test_${testReleaseVersion}_Windows_arm64.exe`} + ${'win32'} | ${'any-other-arch'} | ${`snyk-iac-test_${testReleaseVersion}_Windows_x86_64.exe`} + ${'any-other-platform'} | ${'arm64'} | ${`snyk-iac-test_${testReleaseVersion}_Linux_arm64`} + ${'any-other-platform'} | ${'any-other-arch'} | ${`snyk-iac-test_${testReleaseVersion}_Linux_x86_64`} + `( + 'with `$osPlatform` platform and `$osArch` architecture', + ({ osPlatform, osArch, expectedFileName }) => { + it(`returns ${expectedFileName}`, () => { + // Arrange + jest.spyOn(os, 'platform').mockImplementation(() => osPlatform); + jest.spyOn(os, 'arch').mockImplementation(() => osArch); + + // Act + const res = formatPolicyEngineFileName(testReleaseVersion); + + // Assert + expect(res).toEqual(expectedFileName); + }); + }, + ); +}); diff --git a/test/jest/unit/lib/iac/test/v2/setup/policy-engine.spec.ts b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local.spec.ts similarity index 63% rename from test/jest/unit/lib/iac/test/v2/setup/policy-engine.spec.ts rename to test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local.spec.ts index 1bcdaf20f5..a1e0ef545b 100644 --- a/test/jest/unit/lib/iac/test/v2/setup/policy-engine.spec.ts +++ b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local.spec.ts @@ -1,14 +1,27 @@ +import * as pathLib from 'path'; import * as cloneDeep from 'lodash.clonedeep'; -import * as fileUtils from '../../../../../../../../src/lib/iac/file-utils'; +import * as fileUtils from '../../../../../../../../../../src/lib/iac/file-utils'; import { InvalidUserPolicyEnginePathError, - lookupLocalPolicyEngine, -} from '../../../../../../../../src/lib/iac/test/v2/setup/policy-engine'; - -describe('lookupLocalPolicyEngine', () => { + lookupLocal, +} from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local'; + +jest.mock( + '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/constants', + () => ({ + policyEngineFileName: 'policy-engine-test-file-name', + }), +); + +describe('lookupLocal', () => { + const iacCachePath = pathLib.join('iac', 'cache', 'path'); const defaultTestConfig = { - cachedPolicyEnginePath: `iac/cache/path/snyk-iac-test`, + iacCachePath, }; + const cachedPolicyEnginePath = pathLib.join( + iacCachePath, + 'policy-engine-test-file-name', + ); afterEach(() => { jest.restoreAllMocks(); @@ -22,15 +35,13 @@ describe('lookupLocalPolicyEngine', () => { jest .spyOn(fileUtils, 'isExe') - .mockImplementationOnce( - async (path) => path === testConfig.cachedPolicyEnginePath, - ); + .mockImplementation(async (path) => path === cachedPolicyEnginePath); // Act - const res = await lookupLocalPolicyEngine(testConfig); + const res = await lookupLocal(testConfig); // Assert - expect(res).toEqual(testConfig.cachedPolicyEnginePath); + expect(res).toEqual(cachedPolicyEnginePath); }); }); @@ -40,7 +51,7 @@ describe('lookupLocalPolicyEngine', () => { const testConfig = cloneDeep(defaultTestConfig); // Act - const res = await lookupLocalPolicyEngine(testConfig); + const res = await lookupLocal(testConfig); // Assert expect(res).toBeUndefined(); @@ -61,12 +72,10 @@ describe('lookupLocalPolicyEngine', () => { jest .spyOn(fileUtils, 'isExe') - .mockImplementationOnce( - async (path) => path === userPolicyEnginePath, - ); + .mockImplementation(async (path) => path === userPolicyEnginePath); // Act - const res = await lookupLocalPolicyEngine(testConfig); + const res = await lookupLocal(testConfig); // Assert expect(res).toEqual(userPolicyEnginePath); @@ -82,7 +91,7 @@ describe('lookupLocalPolicyEngine', () => { }; // Act + Assert - await expect(lookupLocalPolicyEngine(testConfig)).rejects.toThrow( + await expect(lookupLocal(testConfig)).rejects.toThrow( InvalidUserPolicyEnginePathError, ); }); diff --git a/test/jest/unit/lib/iac/test/v2/setup/rules.spec.ts b/test/jest/unit/lib/iac/test/v2/setup/local-cache/rules.spec.ts similarity index 95% rename from test/jest/unit/lib/iac/test/v2/setup/rules.spec.ts rename to test/jest/unit/lib/iac/test/v2/setup/local-cache/rules.spec.ts index 71a7dcd021..a471121869 100644 --- a/test/jest/unit/lib/iac/test/v2/setup/rules.spec.ts +++ b/test/jest/unit/lib/iac/test/v2/setup/local-cache/rules.spec.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as tar from 'tar'; import * as rimraf from 'rimraf'; -import { RulesBundleLocator } from '../../../../../../../../src/lib/iac/test/v2/setup/rules'; +import { RulesBundleLocator } from '../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/rules'; describe('PathRulesBundleLocator', () => { let root: string; From 5d72a8e2c62273bdb10eef5d8f108b4c9f1affa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:47:14 +0200 Subject: [PATCH 4/4] chore: override no_proxy from parent env before launching cliv1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit avoid accidental and purposeful redirecting of traffic between cliv1 and cliv2 Signed-off-by: Peter Schäfer <101886095+PeterSchafer@users.noreply.github.com> Signed-off-by: Peter Schäfer <101886095+PeterSchafer@users.noreply.github.com> Signed-off-by: Peter Schäfer <101886095+PeterSchafer@users.noreply.github.com> --- cliv2/internal/cliv2/cliv2.go | 16 ++++++++++++++++ cliv2/internal/cliv2/cliv2_test.go | 11 ++++++++++- cliv2/internal/utils/array.go | 12 ++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cliv2/internal/cliv2/cliv2.go b/cliv2/internal/cliv2/cliv2.go index 7138c0c10e..dd778c8e3c 100644 --- a/cliv2/internal/cliv2/cliv2.go +++ b/cliv2/internal/cliv2/cliv2.go @@ -37,6 +37,12 @@ const SNYK_INTEGRATION_NAME_ENV = "SNYK_INTEGRATION_NAME" const SNYK_INTEGRATION_VERSION_ENV = "SNYK_INTEGRATION_VERSION" const SNYK_HTTPS_PROXY_ENV = "HTTPS_PROXY" const SNYK_HTTP_PROXY_ENV = "HTTP_PROXY" +const SNYK_HTTP_NO_PROXY_ENV = "NO_PROXY" +const SNYK_NPM_PROXY_ENV = "NPM_CONFIG_PROXY" +const SNYK_NPM_HTTPS_PROXY_ENV = "NPM_CONFIG_HTTPS_PROXY" +const SNYK_NPM_HTTP_PROXY_ENV = "NPM_CONFIG_HTTP_PROXY" +const SNYK_NPM_NO_PROXY_ENV = "NPM_CONFIG_NO_PROXY" +const SNYK_NPM_ALL_PROXY = "ALL_PROXY" const SNYK_CA_CERTIFICATE_LOCATION_ENV = "NODE_EXTRA_CA_CERTS" const ( @@ -159,6 +165,16 @@ func PrepareV1EnvironmentVariables(input []string, integrationName string, integ inputAsMap[SNYK_HTTPS_PROXY_ENV] = proxyAddress inputAsMap[SNYK_HTTP_PROXY_ENV] = proxyAddress inputAsMap[SNYK_CA_CERTIFICATE_LOCATION_ENV] = caCertificateLocation + + // ensure that no existing no_proxy or other configuration causes redirecting internal communication that is meant to stay between cliv1 and cliv2 + inputAsMap[SNYK_HTTP_NO_PROXY_ENV] = "" + inputAsMap[SNYK_NPM_NO_PROXY_ENV] = "" + inputAsMap[SNYK_NPM_HTTPS_PROXY_ENV] = "" + inputAsMap[SNYK_NPM_HTTP_PROXY_ENV] = "" + inputAsMap[SNYK_NPM_PROXY_ENV] = "" + inputAsMap[SNYK_NPM_ALL_PROXY] = "" + + inputAsMap = utils.RemoveEmptyValue(inputAsMap) result = utils.ToSlice(inputAsMap, "=") } diff --git a/cliv2/internal/cliv2/cliv2_test.go b/cliv2/internal/cliv2/cliv2_test.go index 2a39600b08..7c08307b42 100644 --- a/cliv2/internal/cliv2/cliv2_test.go +++ b/cliv2/internal/cliv2/cliv2_test.go @@ -14,7 +14,16 @@ import ( func Test_PrepareV1EnvironmentVariables_Fill(t *testing.T) { - input := []string{"something=1", "in=2", "here=3=2"} + input := []string{"something=1", + "in=2", + "here=3=2", + "NO_PROXY=something", + "NPM_CONFIG_PROXY=something", + "NPM_CONFIG_HTTPS_PROXY=something", + "NPM_CONFIG_HTTP_PROXY=something", + "NPM_CONFIG_NO_PROXY=something", + "ALL_PROXY=something", + } expected := []string{"something=1", "in=2", "here=3=2", "SNYK_INTEGRATION_NAME=foo", "SNYK_INTEGRATION_VERSION=bar", "HTTP_PROXY=proxy", "HTTPS_PROXY=proxy", "NODE_EXTRA_CA_CERTS=cacertlocation"} actual, err := cliv2.PrepareV1EnvironmentVariables(input, "foo", "bar", "proxy", "cacertlocation") diff --git a/cliv2/internal/utils/array.go b/cliv2/internal/utils/array.go index 8076891a5c..4430915f21 100644 --- a/cliv2/internal/utils/array.go +++ b/cliv2/internal/utils/array.go @@ -51,3 +51,15 @@ func ToSlice(input map[string]string, combineBy string) []string { return result } + +func RemoveEmptyValue(input map[string]string) map[string]string { + result := make(map[string]string) + + for key, value := range input { + if len(value) > 0 { + result[key] = value + } + } + + return result +}