Merge pull request snyk#2341 from snyk/feat/yarn-workspaces-all-projects

feat: support yarn workspaces projects in --all-projects

src/lib/plugins/get-multi-plugin-result.ts

@@ -1,5 +1,7 @@
 const cloneDeep = require('lodash.clonedeep');
-import * as path from 'path';
+import * as pathLib from 'path';
+import sortBy = require('lodash.sortby');
+import groupBy = require('lodash.groupby');
 import * as cliInterface from '@snyk/cli-interface';
 import chalk from 'chalk';
 import { icon } from '../theme';
@@ -14,6 +16,7 @@ import { convertMultiResultToMultiCustom } from './convert-multi-plugin-res-to-m
 import { PluginMetadata } from '@snyk/cli-interface/legacy/plugin';
 import { CallGraph } from '@snyk/cli-interface/legacy/common';
 import { errorMessageWithRetry, FailedToRunTestError } from '../errors';
+import { processYarnWorkspaces } from './nodejs-plugin/yarn-workspaces-parser';
 
 const debug = debugModule('snyk-test');
 export interface ScannedProjectCustom
@@ -42,11 +45,19 @@ export async function getMultiPluginResult(
   const allResults: ScannedProjectCustom[] = [];
   const failedResults: FailedProjectScanError[] = [];
 
-  for (const targetFile of targetFiles) {
+  // process any yarn workspaces first
+  // the files need to be proceeded together as they provide context to each other
+  const {
+    scannedProjects,
+    unprocessedFiles,
+  } = await processYarnWorkspacesProjects(root, options, targetFiles);
+  allResults.push(...scannedProjects);
+  // process the rest 1 by 1 sent to relevant plugins
+  for (const targetFile of unprocessedFiles) {
     const optionsClone = cloneDeep(options);
-    optionsClone.file = path.relative(root, targetFile);
+    optionsClone.file = pathLib.relative(root, targetFile);
     optionsClone.packageManager = detectPackageManagerFromFile(
-      path.basename(targetFile),
+      pathLib.basename(targetFile),
     );
     try {
       const inspectRes = await getSinglePluginResult(
@@ -78,16 +89,18 @@ export async function getMultiPluginResult(
       );
 
       allResults.push(...pluginResultWithCustomScannedProjects.scannedProjects);
-    } catch (err) {
+    } catch (error) {
+      const errMessage =
+        error.message ?? 'Something went wrong getting dependencies';
       // TODO: propagate this all the way back and include in --json output
       failedResults.push({
         targetFile,
-        error: err,
-        errMessage: err.message || 'Something went wrong getting dependencies',
+        error,
+        errMessage: errMessage,
       });
       debug(
         chalk.bold.red(
-          `\n${icon.ISSUE} Failed to get dependencies for ${targetFile}\nERROR: ${err.message}\n`,
+          `\n${icon.ISSUE} Failed to get dependencies for ${targetFile}\nERROR: ${errMessage}\n`,
         ),
       );
     }
@@ -109,3 +122,72 @@ export async function getMultiPluginResult(
     failedResults,
   };
 }
+
+async function processYarnWorkspacesProjects(
+  root: string,
+  options: Options & (TestOptions | MonitorOptions),
+  targetFiles: string[],
+): Promise<{
+  scannedProjects: ScannedProjectCustom[];
+  unprocessedFiles: string[];
+}> {
+  try {
+    const { scannedProjects } = await processYarnWorkspaces(
+      root,
+      {
+        strictOutOfSync: options.strictOutOfSync,
+        dev: options.dev,
+      },
+      targetFiles,
+    );
+
+    const unprocessedFiles = filterOutProcessedWorkspaces(
+      scannedProjects,
+      targetFiles,
+    );
+    return { scannedProjects, unprocessedFiles };
+  } catch (e) {
+    debug('Error during detecting or processing Yarn Workspaces: ', e);
+    return { scannedProjects: [], unprocessedFiles: targetFiles };
+  }
+}
+
+function filterOutProcessedWorkspaces(
+  scannedProjects: ScannedProjectCustom[],
+  allTargetFiles: string[],
+): string[] {
+  const mapped = allTargetFiles.map((p) => ({ path: p, ...pathLib.parse(p) }));
+  const sorted = sortBy(mapped, 'dir');
+  const targetFilesByDirectory: {
+    [dir: string]: Array<{
+      path: string;
+      base: string;
+      dir: string;
+    }>;
+  } = groupBy(sorted, 'dir');
+
+  const scanned = scannedProjects.map((p) => p.targetFile!);
+  const targetFiles: string[] = [];
+
+  for (const directory of Object.keys(targetFilesByDirectory)) {
+    for (const targetFile of targetFilesByDirectory[directory]) {
+      const { base, path } = targetFile;
+
+      // any non yarn workspace files should be scanned
+      if (!['package.json', 'yarn.lock'].includes(base)) {
+        targetFiles.push(path);
+        continue;
+      }
+
+      // check if Node manifest has already been processed a part or a workspace
+      const packageJsonFileName = pathLib.join(directory, 'package.json');
+      const alreadyScanned = scanned.some((f) =>
+        packageJsonFileName.endsWith(f),
+      );
+      if (!alreadyScanned) {
+        targetFiles.push(path);
+      }
+    }
+  }
+  return targetFiles;
+}

src/lib/plugins/nodejs-plugin/yarn-workspaces-parser.ts

@@ -1,32 +1,32 @@
 import * as baseDebug from 'debug';
 import * as pathUtil from 'path';
-// import * as _ from 'lodash';
 const sortBy = require('lodash.sortby');
 const groupBy = require('lodash.groupby');
 import * as micromatch from 'micromatch';
 
 const debug = baseDebug('snyk-yarn-workspaces');
 import * as lockFileParser from 'snyk-nodejs-lockfile-parser';
-import { NoSupportedManifestsFoundError } from '../../errors';
 import {
   MultiProjectResultCustom,
   ScannedProjectCustom,
 } from '../get-multi-plugin-result';
 import { getFileContents } from '../../get-file-contents';
+import { NoSupportedManifestsFoundError } from '../../errors';
 
 export async function processYarnWorkspaces(
   root: string,
   settings: {
     strictOutOfSync?: boolean;
     dev?: boolean;
+    yarnWorkspaces?: boolean;
   },
   targetFiles: string[],
 ): Promise<MultiProjectResultCustom> {
   // the order of yarnTargetFiles folders is important
   // must have the root level most folders at the top
   const mappedAndFiltered = targetFiles
     .map((p) => ({ path: p, ...pathUtil.parse(p) }))
-    .filter((res) => ['package.json'].includes(res.base));
+    .filter((res) => ['package.json', 'yarn.lock'].includes(res.base));
   const sorted = sortBy(mappedAndFiltered, 'dir');
   const grouped = groupBy(sorted, 'dir');
 
@@ -39,7 +39,7 @@ export async function processYarnWorkspaces(
   } = grouped;
 
   debug(`Processing potential Yarn workspaces (${targetFiles.length})`);
-  if (Object.keys(yarnTargetFiles).length === 0) {
+  if (settings.yarnWorkspaces && Object.keys(yarnTargetFiles).length === 0) {
     throw NoSupportedManifestsFoundError([root]);
   }
   let yarnWorkspacesMap = {};
@@ -54,6 +54,7 @@ export async function processYarnWorkspaces(
   let rootWorkspaceManifestContent = {};
   // the folders must be ordered highest first
   for (const directory of Object.keys(yarnTargetFiles)) {
+    debug(`Processing ${directory} as a potential Yarn workspace`);
     let isYarnWorkspacePackage = false;
     let isRootPackageJson = false;
     const packageJsonFileName = pathUtil.join(directory, 'package.json');
@@ -81,7 +82,13 @@ export async function processYarnWorkspaces(
       }
     }
 
-    if (isYarnWorkspacePackage || isRootPackageJson) {
+    if (!(isYarnWorkspacePackage || isRootPackageJson)) {
+      debug(
+        `${packageJsonFileName} is not part of any detected workspace, skipping`,
+      );
+      continue;
+    }
+    try {
       const rootDir = isYarnWorkspacePackage
         ? pathUtil.dirname(yarnWorkspacesFilesMap[packageJsonFileName].root)
         : pathUtil.dirname(packageJsonFileName);
@@ -117,10 +124,11 @@ export async function processYarnWorkspaces(
         },
       };
       result.scannedProjects.push(project);
-    } else {
-      debug(
-        `${packageJsonFileName} is not part of any detected workspace, skipping`,
-      );
+    } catch (e) {
+      if (settings.yarnWorkspaces) {
+        throw e;
+      }
+      debug(`Error process workspace: ${packageJsonFileName}. ERROR: ${e}`);
     }
   }
   if (!result.scannedProjects.length) {

src/lib/snyk-test/run-test.ts

@@ -593,7 +593,7 @@ async function assembleLocalPayloads(
       );
       if (options['fail-fast']) {
         throw new FailedToRunTestError(
-          'Your test request could not be completed. Please email support@snyk.io',
+          errorMessageWithRetry('Your test request could not be completed.'),
         );
       }
     }

test/acceptance/cli-monitor/cli-monitor.all-projects.spec.ts

@@ -664,8 +664,7 @@ export const AllProjectsTests: AcceptanceTests = {
         'same body for --all-projects and --file=mono-repo-go/hello-vendor/vendor/vendor.json',
       );
     },
-
-    '`monitor mono-repo-go with --all-projects and --detectin-depth=3`': (
+    '`monitor mono-repo-go with --all-projects and --detection-depth=3`': (
       params,
       utils,
     ) => async (t) => {
@@ -955,5 +954,105 @@ export const AllProjectsTests: AcceptanceTests = {
         );
       });
     },
+    'monitor yarn-workspaces --all-projects --detection-depth=5 finds Yarn workspaces & Npm projects': (
+      params,
+      utils,
+    ) => async (t) => {
+      t.teardown(() => {
+        loadPlugin.restore();
+      });
+      utils.chdirWorkspaces();
+      const loadPlugin = sinon.spy(params.plugins, 'loadPlugin');
+
+      const result = await params.cli.monitor('yarn-workspaces', {
+        allProjects: true,
+        detectionDepth: 5,
+      });
+      // the parser is used directly
+      t.ok(
+        loadPlugin.withArgs('yarn').notCalled,
+        'skips load plugin for yarn as a parser is used directly',
+      );
+      t.equal(loadPlugin.withArgs('npm').callCount, 1, 'calls npm plugin once');
+
+      t.match(
+        result,
+        'Monitoring yarn-workspaces (package.json)',
+        'yarn workspace root was monitored',
+      );
+      t.match(
+        result,
+        'Monitoring yarn-workspaces (apple-lib)',
+        'yarn workspace was monitored',
+      );
+      t.match(
+        result,
+        'Monitoring yarn-workspaces (apples)',
+        'yarn workspace was monitored',
+      );
+      t.match(
+        result,
+        'Monitoring yarn-workspaces (tomatoes)',
+        'yarn workspace was monitored',
+      );
+      t.match(
+        result,
+        'Monitoring yarn-workspaces (not-in-a-workspace)',
+        'npm project was monitored',
+      );
+
+      const requests = params.server
+        .getRequests()
+        .filter((req) => req.url.includes('/monitor/'));
+      t.equal(requests.length, 5, 'correct amount of monitor requests');
+      let policyCount = 0;
+      const applesWorkspace =
+        process.platform === 'win32'
+          ? '\\apples\\package.json'
+          : 'apples/package.json';
+      const tomatoesWorkspace =
+        process.platform === 'win32'
+          ? '\\tomatoes\\package.json'
+          : 'tomatoes/package.json';
+      const rootWorkspace =
+        process.platform === 'win32'
+          ? '\\yarn-workspaces\\package.json'
+          : 'yarn-workspaces/package.json';
+      requests.forEach((req) => {
+        t.match(
+          req.url,
+          /\/api\/v1\/monitor\/(yarn\/graph|npm\/graph)/,
+          'puts at correct url',
+        );
+        t.equal(req.method, 'PUT', 'makes PUT request');
+        t.equal(
+          req.headers['x-snyk-cli-version'],
+          params.versionNumber,
+          'sends version number',
+        );
+        if (req.body.targetFileRelativePath.endsWith(applesWorkspace)) {
+          t.match(
+            req.body.policy,
+            'npm:node-uuid:20160328',
+            'policy is as expected',
+          );
+          t.ok(req.body.policy, 'body contains policy');
+          policyCount += 1;
+        } else if (
+          req.body.targetFileRelativePath.endsWith(tomatoesWorkspace)
+        ) {
+          t.notOk(req.body.policy, 'body does not contain policy');
+        } else if (req.body.targetFileRelativePath.endsWith(rootWorkspace)) {
+          t.match(
+            req.body.policy,
+            'npm:node-uuid:20111130',
+            'policy is as expected',
+          );
+          t.ok(req.body.policy, 'body contains policy');
+          policyCount += 1;
+        }
+      });
+      t.equal(policyCount, 2, '2 policies found in a workspace');
+    },
   },
 };