Merge branch 'master' into chore/cliv2_case_insensitive_env_vars

src/cli/commands/test/iac/local-execution/local-cache.ts

@@ -1,15 +1,14 @@
 import * as path from 'path';
 import * as fs from 'fs';
 import { EngineType, IaCErrorCodes } from './types';
-import * as needle from 'needle';
 import * as rimraf from 'rimraf';
 import { createIacDir, extractBundle, isValidBundle } from './file-utils';
 import * as Debug from 'debug';
 import { CustomError } from '../../../../../lib/errors';
 import * as analytics from '../../../../../lib/analytics';
-import ReadableStream = NodeJS.ReadableStream;
 import { getErrorStringCode } from './error-utils';
 import config from '../../../../../lib/config';
+import { streamRequest } from '../../../../../lib/request/request';
 
 const debug = Debug('iac-local-cache');
 
@@ -141,7 +140,12 @@ export async function initLocalCache({
   // always overwrite whatever might be there.
   try {
     const BUNDLE_URL = 'https://static.snyk.io/cli/wasm/bundle.tar.gz';
-    const response: ReadableStream = needle.get(BUNDLE_URL);
+    const response = await streamRequest({
+      method: 'get',
+      url: BUNDLE_URL,
+      body: null,
+      headers: {},
+    });
     await extractBundle(response);
   } catch (e) {
     throw new FailedToDownloadRulesError();

src/cli/commands/test/iac/v2/index.ts

@@ -29,9 +29,8 @@ function prepareTestConfig(paths: string[]): TestConfig {
 
   return {
     paths,
-    cachedBundlePath: pathLib.join(iacCachePath, 'bundle.tar.gz'),
+    iacCachePath,
     userBundlePath: config.IAC_BUNDLE_PATH,
-    cachedPolicyEnginePath: pathLib.join(iacCachePath, 'snyk-iac-test'),
     userPolicyEnginePath: config.IAC_POLICY_ENGINE_PATH,
   };
 }

src/lib/ecosystems/resolve-test-facts.ts

@@ -8,6 +8,7 @@ import {
 import { extractAndApplyPluginAnalytics } from './plugin-analytics';
 import { findAndLoadPolicy } from '../policy';
 import { filterIgnoredIssues } from './policy';
+import { IssueData, Issue } from '../snyk-test/legacy';
 
 export async function resolveAndTestFacts(
   ecosystem: Ecosystem,
@@ -18,6 +19,7 @@ export async function resolveAndTestFacts(
 ): Promise<[TestResult[], string[]]> {
   const results: any[] = [];
   const errors: string[] = [];
+  const packageManager = 'Unmanaged (C/C++)';
 
   for (const [path, scanResults] of Object.entries(scans)) {
     await spinner(`Resolving and Testing fileSignatures in ${path}`);
@@ -45,12 +47,33 @@ export async function resolveAndTestFacts(
           policy,
         );
 
+        const issuesMap: Map<string, Issue> = new Map();
+        response.issues.forEach((i) => {
+          issuesMap[i.issueId] = i;
+        });
+
+        const vulnerabilities: IssueData[] = [];
+        for (const issuesDataKey in response.issuesData) {
+          const issueData = response.issuesData[issuesDataKey];
+          const pkgCoordinate = `${issuesMap[issuesDataKey].pkgName}@${issuesMap[issuesDataKey].pkgVersion}`;
+          issueData.from = [pkgCoordinate];
+          issueData.name = pkgCoordinate;
+          issueData.packageManager = packageManager;
+          vulnerabilities.push(issueData);
+        }
+
+        const dependencyCount = response.issues.length;
+
         results.push({
           issues,
           issuesData,
           depGraphData: response?.depGraphData,
           depsFilePaths: response?.depsFilePaths,
           fileSignaturesDetails: response?.fileSignaturesDetails,
+          vulnerabilities,
+          path,
+          dependencyCount,
+          packageManager,
         });
       } catch (error) {
         const hasStatusCodeError = error.code >= 400 && error.code <= 500;

src/lib/iac/drift/driftctl.ts

@@ -23,7 +23,7 @@ import * as child_process from 'child_process';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as crypto from 'crypto';
-import { isExe } from '../file-utils';
+import { createDirIfNotExists, isExe } from '../file-utils';
 
 const debug = debugLib('driftctl');
 
@@ -71,12 +71,12 @@ const driftctlDefaultOptions = ['--no-version-check'];
 
 let isBinaryDownloaded = false;
 
-export const generateArgs = (
+export const generateArgs = async (
   options: DriftCTLOptions,
   driftIgnore?: string[],
-): string[] => {
+): Promise<string[]> => {
   if (options.kind === 'describe') {
-    return generateScanFlags(options as DescribeOptions, driftIgnore);
+    return await generateScanFlags(options as DescribeOptions, driftIgnore);
   }
 
   if (options.kind === 'fmt') {
@@ -107,10 +107,10 @@ const generateFmtFlags = (options: FmtOptions): string[] => {
   return args;
 };
 
-const generateScanFlags = (
+const generateScanFlags = async (
   options: DescribeOptions,
   driftIgnore?: string[],
-): string[] => {
+): Promise<string[]> => {
   const args: string[] = ['scan', ...driftctlDefaultOptions];
 
   if (options.quiet) {
@@ -177,7 +177,7 @@ const generateScanFlags = (
   }
 
   let configDir = cachePath;
-  createIfNotExists(cachePath);
+  await createDirIfNotExists(cachePath);
   if (options['config-dir']) {
     configDir = options['config-dir'];
   }
@@ -238,7 +238,7 @@ export const runDriftCTL = async ({
 }): Promise<DriftctlExecutionResult> => {
   const path = await findOrDownload();
   await validateArgs(options);
-  const args = generateArgs(options, driftIgnore);
+  const args = await generateArgs(options, driftIgnore);
 
   if (!stdio) {
     stdio = ['pipe', 'pipe', 'inherit'];
@@ -281,7 +281,7 @@ async function findOrDownload(): Promise<string> {
   if (dctl === '') {
     binaryExist = false;
     try {
-      createIfNotExists(cachePath);
+      await createDirIfNotExists(cachePath);
       dctl = driftctlPath;
 
       const duration = new TimerMetricInstance('driftctl_download');
@@ -424,9 +424,3 @@ function driftctlUrl(): string {
 
   return `${dctlBaseUrl}/${driftctlVersion}/${driftctlFileName()}`;
 }
-
-function createIfNotExists(path: string) {
-  if (!fs.existsSync(path)) {
-    fs.mkdirSync(path, { recursive: true });
-  }
-}

src/lib/iac/file-utils.ts

@@ -9,3 +9,19 @@ export async function isExe(path: string): Promise<boolean> {
     return false;
   }
 }
+
+export async function isExists(path: string): Promise<boolean> {
+  try {
+    await fsPromises.stat(path);
+    return true;
+  } catch (err) {
+    return false;
+  }
+}
+
+export async function createDirIfNotExists(path: string): Promise<void> {
+  const isDirExists = await isExists(path);
+  if (!isDirExists) {
+    fsPromises.mkdir(path, { recursive: true });
+  }
+}

src/lib/iac/test/v2/setup/index.ts

@@ -1,9 +1,6 @@
 import { TestConfig } from '../types';
-import { initRules } from './rules';
-import { initPolicyEngine } from './policy-engine';
+import { initLocalCache } from './local-cache';
 
 export async function setup(testConfig: TestConfig) {
-  const policyEnginePath = await initPolicyEngine(testConfig);
-  const rulesBundlePath = await initRules(testConfig);
-  return { policyEnginePath, rulesBundlePath };
+  return await initLocalCache(testConfig);
 }

src/lib/iac/test/v2/setup/local-cache/index.ts

@@ -0,0 +1,19 @@
+import { TestConfig } from '../../types';
+import { initRules } from './rules';
+import { initPolicyEngine } from './policy-engine';
+import { createDirIfNotExists } from '../../../../file-utils';
+import { CustomError } from '../../../../../errors';
+import { FailedToInitLocalCacheError } from '../../../../../../cli/commands/test/iac/local-execution/local-cache';
+
+export async function initLocalCache(testConfig: TestConfig) {
+  try {
+    await createDirIfNotExists(testConfig.iacCachePath);
+
+    const policyEnginePath = await initPolicyEngine(testConfig);
+    const rulesBundlePath = await initRules(testConfig);
+
+    return { policyEnginePath, rulesBundlePath };
+  } catch (err) {
+    throw err instanceof CustomError ? err : new FailedToInitLocalCacheError();
+  }
+}

src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts

@@ -0,0 +1,11 @@
+import { formatPolicyEngineFileName } from './utils';
+
+/**
+ * The Policy Engine release version associated with this Snyk CLI version.
+ */
+export const releaseVersion = '0.1.0';
+
+/**
+ * The Policy Engine executable's file name.
+ */
+export const policyEngineFileName = formatPolicyEngineFileName(releaseVersion);

src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/utils.ts

@@ -0,0 +1,19 @@
+import * as os from 'os';
+
+export function formatPolicyEngineFileName(releaseVersion: string) {
+  let platform = 'Linux';
+  switch (os.platform()) {
+    case 'darwin':
+      platform = 'Darwin';
+      break;
+    case 'win32':
+      platform = 'Windows';
+      break;
+  }
+
+  const arch = os.arch() === 'arm64' ? 'arm64' : 'x86_64';
+
+  const execExt = os.platform() === 'win32' ? '.exe' : '';
+
+  return `snyk-iac-test_${releaseVersion}_${platform}_${arch}${execExt}`;
+}

src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts

@@ -0,0 +1,13 @@
+import { TestConfig } from '../../../types';
+import { InvalidUserPolicyEnginePathError, lookupLocal } from './lookup-local';
+
+export async function initPolicyEngine(testConfig: TestConfig) {
+  const localPolicyEnginePath = await lookupLocal(testConfig);
+  if (localPolicyEnginePath) {
+    return localPolicyEnginePath;
+  }
+
+  // TODO: Download Policy Engine executable
+
+  throw new InvalidUserPolicyEnginePathError('', 'policy engine not found');
+}

src/lib/iac/test/v2/setup/policy-engine.ts → src/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local.ts

@@ -1,9 +1,11 @@
+import * as pathLib from 'path';
 import * as createDebugLogger from 'debug';
-import { isExe } from '../../../file-utils';
-import { CustomError } from '../../../../errors';
-import { IaCErrorCodes } from '../../../../../cli/commands/test/iac/local-execution/types';
-import { getErrorStringCode } from '../../../../../cli/commands/test/iac/local-execution/error-utils';
-import { TestConfig } from '../types';
+import { isExe } from '../../../../../file-utils';
+import { CustomError } from '../../../../../../errors';
+import { IaCErrorCodes } from '../../../../../../../cli/commands/test/iac/local-execution/types';
+import { getErrorStringCode } from '../../../../../../../cli/commands/test/iac/local-execution/error-utils';
+import { TestConfig } from '../../../types';
+import { policyEngineFileName } from './constants';
 
 const debugLogger = createDebugLogger('snyk-iac');
 
@@ -22,8 +24,8 @@ export class InvalidUserPolicyEnginePathError extends CustomError {
   }
 }
 
-export async function lookupLocalPolicyEngine({
-  cachedPolicyEnginePath,
+export async function lookupLocal({
+  iacCachePath,
   userPolicyEnginePath,
 }: TestConfig): Promise<string | undefined> {
   // Lookup in custom path.
@@ -41,6 +43,10 @@ export async function lookupLocalPolicyEngine({
   }
   // Lookup in cache.
   else {
+    const cachedPolicyEnginePath = pathLib.join(
+      iacCachePath,
+      policyEngineFileName,
+    );
     if (await isExe(cachedPolicyEnginePath)) {
       debugLogger(
         'Found cached Policy Engine executable: %s',
@@ -55,17 +61,3 @@ export async function lookupLocalPolicyEngine({
     }
   }
 }
-
-export async function initPolicyEngine(
-  testConfig: TestConfig,
-): Promise<string> {
-  const localPolicyEnginePath = await lookupLocalPolicyEngine(testConfig);
-
-  if (localPolicyEnginePath) {
-    return localPolicyEnginePath;
-  }
-
-  // TODO: Download Policy Engine executable
-
-  throw new InvalidUserPolicyEnginePathError('', 'policy engine not found');
-}

src/lib/iac/test/v2/setup/rules.ts → src/lib/iac/test/v2/setup/local-cache/rules.ts

@@ -1,13 +1,21 @@
+import * as pathLib from 'path';
 import * as fs from 'fs';
 import * as tar from 'tar';
-import { CustomError } from '../../../../errors';
-import { getErrorStringCode } from '../../../../../cli/commands/test/iac/local-execution/error-utils';
-import { IaCErrorCodes } from '../../../../../cli/commands/test/iac/local-execution/types';
-import { TestConfig } from '../types';
+import { CustomError } from '../../../../../errors';
+import { getErrorStringCode } from '../../../../../../cli/commands/test/iac/local-execution/error-utils';
+import { IaCErrorCodes } from '../../../../../../cli/commands/test/iac/local-execution/types';
+import { TestConfig } from '../../types';
+
+export const rulesBundleName = 'bundle.tar.gz';
 
 export async function initRules(testConfig: TestConfig): Promise<string> {
+  const cachedBundlePath = pathLib.join(
+    testConfig.iacCachePath,
+    rulesBundleName,
+  );
+
   const bundleLocator = new RulesBundleLocator(
-    testConfig.cachedBundlePath,
+    cachedBundlePath,
     testConfig.userBundlePath,
   );
 

src/lib/iac/test/v2/types.ts

@@ -1,7 +1,6 @@
 export interface TestConfig {
   paths: string[];
-  cachedBundlePath: string;
-  cachedPolicyEnginePath: string;
+  iacCachePath: string;
   userBundlePath?: string;
   userPolicyEnginePath?: string;
 }

src/lib/package-managers.ts

@@ -14,7 +14,8 @@ export type SupportedPackageManagers =
   | 'composer'
   | 'cocoapods'
   | 'poetry'
-  | 'hex';
+  | 'hex'
+  | 'Unmanaged (C/C++)';
 
 export enum SUPPORTED_MANIFEST_FILES {
   GEMFILE = 'Gemfile',
@@ -67,6 +68,7 @@ export const SUPPORTED_PACKAGE_MANAGER_NAME: {
   cocoapods: 'CocoaPods',
   poetry: 'Poetry',
   hex: 'Hex',
+  'Unmanaged (C/C++)': 'Unmanaged (C/C++)',
 };
 
 export const GRAPH_SUPPORTED_PACKAGE_MANAGERS: SupportedPackageManagers[] = [

src/lib/polling/polling-test.ts

@@ -63,13 +63,21 @@ export async function pollingTestWithTokenUntilDone(
       depGraphData,
       depsFilePaths,
       fileSignaturesDetails,
+      vulnerabilities,
+      path,
+      dependencyCount,
+      packageManager,
     } = response.result;
     return {
       issues,
       issuesData,
       depGraphData,
       depsFilePaths,
       fileSignaturesDetails,
+      vulnerabilities,
+      path,
+      dependencyCount,
+      packageManager,
     };
   }