feat: support yarn workspaces projects in --all-projects

By default include Yarn workspace projects for duscovery during
--all-projects as well. Filter out any files that were scanned
as part of a workspace before forewarding on the rest of the detected
manifests to be scanned by the individual plugins.

src/lib/plugins/get-multi-plugin-result.ts

@@ -1,5 +1,7 @@
 const cloneDeep = require('lodash.clonedeep');
-import * as path from 'path';
+import * as pathLib from 'path';
+import sortBy = require('lodash.sortby');
+import groupBy = require('lodash.groupby');
 import * as cliInterface from '@snyk/cli-interface';
 import chalk from 'chalk';
 import { icon } from '../theme';
@@ -14,6 +16,7 @@ import { convertMultiResultToMultiCustom } from './convert-multi-plugin-res-to-m
 import { PluginMetadata } from '@snyk/cli-interface/legacy/plugin';
 import { CallGraph } from '@snyk/cli-interface/legacy/common';
 import { FailedToRunTestError } from '../errors';
+import { processYarnWorkspaces } from './nodejs-plugin/yarn-workspaces-parser';
 
 const debug = debugModule('snyk-test');
 export interface ScannedProjectCustom
@@ -42,11 +45,19 @@ export async function getMultiPluginResult(
   const allResults: ScannedProjectCustom[] = [];
   const failedResults: FailedProjectScanError[] = [];
 
-  for (const targetFile of targetFiles) {
+  // process any yarn workspaces first
+  // the files need to be proceeded together as they provide context to each other
+  const {
+    scannedProjects,
+    unprocessedFiles,
+  } = await processYarnWorkspacesProjects(root, options, targetFiles);
+  allResults.push(...scannedProjects);
+  // process the rest 1 by 1 sent to relevant plugins
+  for (const targetFile of unprocessedFiles) {
     const optionsClone = cloneDeep(options);
-    optionsClone.file = path.relative(root, targetFile);
+    optionsClone.file = pathLib.relative(root, targetFile);
     optionsClone.packageManager = detectPackageManagerFromFile(
-      path.basename(targetFile),
+      pathLib.basename(targetFile),
     );
     try {
       const inspectRes = await getSinglePluginResult(
@@ -78,16 +89,18 @@ export async function getMultiPluginResult(
       );
 
       allResults.push(...pluginResultWithCustomScannedProjects.scannedProjects);
-    } catch (err) {
+    } catch (error) {
+      const errMessage =
+        error.message ?? 'Something went wrong getting dependencies';
       // TODO: propagate this all the way back and include in --json output
       failedResults.push({
         targetFile,
-        error: err,
-        errMessage: err.message || 'Something went wrong getting dependencies',
+        error,
+        errMessage: errMessage,
       });
       debug(
         chalk.bold.red(
-          `\n${icon.ISSUE} Failed to get dependencies for ${targetFile}\nERROR: ${err.message}\n`,
+          `\n${icon.ISSUE} Failed to get dependencies for ${targetFile}\nERROR: ${errMessage}\n`,
         ),
       );
     }
@@ -107,3 +120,71 @@ export async function getMultiPluginResult(
     failedResults,
   };
 }
+
+async function processYarnWorkspacesProjects(
+  root: string,
+  options: Options & (TestOptions | MonitorOptions),
+  targetFiles: string[],
+): Promise<{
+  scannedProjects: ScannedProjectCustom[];
+  unprocessedFiles: string[];
+}> {
+  try {
+    const { scannedProjects } = await processYarnWorkspaces(
+      root,
+      {
+        strictOutOfSync: options.strictOutOfSync,
+        dev: options.dev,
+      },
+      targetFiles,
+    );
+
+    const unprocessedFiles = filterOutProcessedWorkspaces(
+      scannedProjects,
+      targetFiles,
+    );
+    return { scannedProjects, unprocessedFiles };
+  } catch (e) {
+    return { scannedProjects: [], unprocessedFiles: targetFiles };
+  }
+}
+
+function filterOutProcessedWorkspaces(
+  scannedProjects: ScannedProjectCustom[],
+  allTargetFiles: string[],
+): string[] {
+  const mapped = allTargetFiles.map((p) => ({ path: p, ...pathLib.parse(p) }));
+  const sorted = sortBy(mapped, 'dir');
+  const targetFilesByDirectory: {
+    [dir: string]: Array<{
+      path: string;
+      base: string;
+      dir: string;
+    }>;
+  } = groupBy(sorted, 'dir');
+
+  const scanned = scannedProjects.map((p) => p.targetFile!);
+  const targetFiles: string[] = [];
+
+  for (const directory of Object.keys(targetFilesByDirectory)) {
+    for (const targetFile of targetFilesByDirectory[directory]) {
+      const { base, path } = targetFile;
+
+      // any non yarn workspace files should be scanned
+      if (!['package.json', 'yarn.lock'].includes(base)) {
+        targetFiles.push(path);
+        continue;
+      }
+
+      // check if Node manifest has already been processed a part or a workspace
+      const packageJsonFileName = pathLib.join(directory, 'package.json');
+      const alreadyScanned = scanned.some((f) =>
+        packageJsonFileName.endsWith(f),
+      );
+      if (!alreadyScanned) {
+        targetFiles.push(path);
+      }
+    }
+  }
+  return targetFiles;
+}

src/lib/plugins/nodejs-plugin/yarn-workspaces-parser.ts

@@ -1,13 +1,11 @@
 import * as baseDebug from 'debug';
 import * as pathUtil from 'path';
-// import * as _ from 'lodash';
 const sortBy = require('lodash.sortby');
 const groupBy = require('lodash.groupby');
 import * as micromatch from 'micromatch';
 
 const debug = baseDebug('snyk-yarn-workspaces');
 import * as lockFileParser from 'snyk-nodejs-lockfile-parser';
-import { NoSupportedManifestsFoundError } from '../../errors';
 import {
   MultiProjectResultCustom,
   ScannedProjectCustom,
@@ -26,7 +24,7 @@ export async function processYarnWorkspaces(
   // must have the root level most folders at the top
   const mappedAndFiltered = targetFiles
     .map((p) => ({ path: p, ...pathUtil.parse(p) }))
-    .filter((res) => ['package.json'].includes(res.base));
+    .filter((res) => ['package.json', 'yarn.lock'].includes(res.base));
   const sorted = sortBy(mappedAndFiltered, 'dir');
   const grouped = groupBy(sorted, 'dir');
 
@@ -39,9 +37,6 @@ export async function processYarnWorkspaces(
   } = grouped;
 
   debug(`Processing potential Yarn workspaces (${targetFiles.length})`);
-  if (Object.keys(yarnTargetFiles).length === 0) {
-    throw NoSupportedManifestsFoundError([root]);
-  }
   let yarnWorkspacesMap = {};
   const yarnWorkspacesFilesMap = {};
   const result: MultiProjectResultCustom = {