Poetry root list from pyproject.toml (#1386)

* Poetry root list from pyproject.toml Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Fixes 1385 Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Fixes 1385 Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Fix test Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Handle duplicate components in lock files Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
CycloneDX · Sep 22, 2024 · 5385422 · 5385422
1 parent 9e36e8e
commit 5385422
Show file tree

Hide file tree

Showing 53 changed files with 99 additions and 39 deletions.
diff --git a/lib/cli/index.js b/lib/cli/index.js
@@ -2876,16 +2876,18 @@ export async function createPythonBom(path, options) {
           );
         }
       }
-      const parentDependsOn = [];
-      // Complete the dependency tree by making parent component depend on the first level
-      for (const p of retMap.rootList) {
-        parentDependsOn.push(`pkg:pypi/${p.name.toLowerCase()}@${p.version}`);
+      if (retMap.rootList) {
+        const parentDependsOn = [];
+        // Complete the dependency tree by making parent component depend on the first level
+        for (const p of retMap.rootList) {
+          parentDependsOn.push(`pkg:pypi/${p.name.toLowerCase()}@${p.version}`);
+        }
+        const pdependencies = {
+          ref: parentComponent["bom-ref"],
+          dependsOn: parentDependsOn,
+        };
+        dependencies.splice(0, 0, pdependencies);
       }
-      const pdependencies = {
-        ref: parentComponent["bom-ref"],
-        dependsOn: parentDependsOn,
-      };
-      dependencies.splice(0, 0, pdependencies);
     }
     options.parentComponent = parentComponent;
   } // poetryMode

diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js
@@ -3944,17 +3944,57 @@ export function parsePyProjectToml(tomlFile) {
  *
  * @param {Object} lockData JSON data from poetry.lock
  * @param {string} lockFile Lock file name for evidence
+ * @param {string} pyProjectFile pyproject.toml file
  */
-export async function parsePoetrylockData(lockData, lockFile) {
+export async function parsePoetrylockData(lockData, lockFile, pyProjectFile) {
   let pkgList = [];
+  const rootList = [];
   const dependenciesList = [];
   const depsMap = {};
   const existingPkgMap = {};
+  const directDepsKeys = {};
+  const groupDepsKeys = {};
   let pkg = null;
   let lastPkgRef = undefined;
   let depsMode = false;
   if (!lockData) {
-    return pkgList;
+    return { pkgList, dependenciesList };
+  }
+  if (!pyProjectFile && lockFile) {
+    pyProjectFile = lockFile.replace("poetry.lock", "pyproject.toml");
+    if (existsSync(pyProjectFile)) {
+      const pyprojTomlFile = readFileSync(pyProjectFile, { encoding: "utf-8" });
+      const tomlData = toml.parse(pyprojTomlFile);
+      if (tomlData?.tool?.poetry) {
+        for (const adep of Object.keys(tomlData?.tool?.poetry?.dependencies)) {
+          if (
+            ![
+              "python",
+              "py",
+              "pytest",
+              "pylint",
+              "ruff",
+              "setuptools",
+              "bandit",
+            ].includes(adep)
+          ) {
+            directDepsKeys[adep] = true;
+          }
+        } // for
+        if (tomlData?.tool?.poetry?.group) {
+          for (const agroup of Object.keys(tomlData.tool.poetry.group)) {
+            for (const adep of Object.keys(
+              tomlData.tool.poetry.group[agroup]?.dependencies,
+            )) {
+              if (!groupDepsKeys[adep]) {
+                groupDepsKeys[adep] = [];
+              }
+              groupDepsKeys[adep].push(agroup);
+            }
+          } // for
+        }
+      }
+    }
   }
   lockData.split("\n").forEach((l) => {
     l = l.replace("\r", "");
@@ -3995,11 +4035,18 @@ export async function parsePoetrylockData(lockData, lockFile) {
             ],
           },
         };
-        // This would help look
-        if (!existingPkgMap[pkg.name.toLowerCase()]) {
-          existingPkgMap[pkg.name.toLowerCase()] = pkg["bom-ref"];
-          pkgList.push(pkg);
+        if (groupDepsKeys[pkg.name]) {
+          pkg.scope = "optional";
+          pkg.properties = groupDepsKeys[pkg.name].map((g) => {
+            return { name: "cdx:poetry:group", value: g };
+          });
+        }
+        if (directDepsKeys[pkg.name]) {
+          rootList.push(pkg);
         }
+        // This would help the lookup
+        existingPkgMap[pkg.name.toLowerCase()] = pkg["bom-ref"];
+        pkgList.push(pkg);
         lastPkgRef = pkg["bom-ref"];
         if (!depsMap[lastPkgRef]) {
           depsMap[lastPkgRef] = new Set();
@@ -4057,7 +4104,7 @@ export async function parsePoetrylockData(lockData, lockFile) {
   }
   return {
     pkgList,
-    rootList: pkgList,
+    rootList,
     dependenciesList,
   };
 }
@@ -12469,6 +12516,9 @@ export function isValidIriReference(iri) {
  * @returns {Boolean} True if the dependency tree lacks any non-root parents without children. False otherwise.
  */
 export function isPartialTree(dependencies, componentsCount = 1) {
+  if (componentsCount <= 1) {
+    return false;
+  }
   if (dependencies?.length <= 1) {
     return true;
   }
@@ -12478,7 +12528,10 @@ export function isPartialTree(dependencies, componentsCount = 1) {
       parentsWithChildsCount++;
     }
   }
-  return parentsWithChildsCount <= Math.max(Math.round(componentsCount / 3), 1);
+  return (
+    parentsWithChildsCount <
+    Math.min(Math.round(componentsCount / 3), componentsCount)
+  );
 }
 
 /**

diff --git a/lib/helpers/utils.test.js b/lib/helpers/utils.test.js
@@ -3500,7 +3500,7 @@ test("parseYarnLock", async () => {
   expect(parsedList.dependenciesList.length).toEqual(1);
   expect(
     isPartialTree(parsedList.dependenciesList, parsedList.pkgList.length),
-  ).toBeTruthy();
+  ).toBeFalsy();
   parsedList = await parseYarnLock("./test/data/yarn_locks/yarn-at.lock");
   expect(parsedList.pkgList.length).toEqual(4);
   expect(parsedList.dependenciesList.length).toEqual(4);
@@ -3993,7 +3993,7 @@ test("parse poetry.lock", async () => {
     readFileSync("./test/data/pdm.lock", { encoding: "utf-8" }),
     "./test/data/pdm.lock",
   );
-  expect(retMap.pkgList.length).toEqual(37);
+  expect(retMap.pkgList.length).toEqual(39);
   expect(retMap.dependenciesList.length).toEqual(37);
 }, 120000);
 

diff --git a/lib/helpers/validator.js b/lib/helpers/validator.js
@@ -223,7 +223,7 @@ export const validateRefs = (bomJson) => {
   if (bomJson?.dependencies) {
     if (isPartialTree(bomJson.dependencies, bomJson?.components?.length)) {
       warningsList.push(
-        "Dependency tree is partial with multiple empty dependsOn attribute.",
+        "Dependency tree is partial with multiple empty dependsOn attributes.",
       );
     }
     for (const dep of bomJson.dependencies) {

diff --git a/tsconfig.json b/tsconfig.json
@@ -1,6 +1,6 @@
 {
-  "include": ["*.js"],
-  "exclude": ["*.test.js", "./types/**/*"],
+  "include": ["*.js", "lib/**/*.js"],
+  "exclude": ["*.test.js", "lib/**/*.test.js", "./types/**/*"],
   "compilerOptions": {
     "allowJs": true,
     // Generate d.ts files

diff --git a/types/analyzer.d.ts.map b/types/analyzer.d.ts.map
diff --git a/types/binary.d.ts.map b/types/binary.d.ts.map
diff --git a/types/cbomutils.d.ts.map b/types/cbomutils.d.ts.map
diff --git a/types/db.d.ts.map b/types/db.d.ts.map
diff --git a/types/display.d.ts.map b/types/display.d.ts.map
diff --git a/types/docker.d.ts.map b/types/docker.d.ts.map
diff --git a/types/envcontext.d.ts.map b/types/envcontext.d.ts.map
diff --git a/types/evinser.d.ts.map b/types/evinser.d.ts.map
diff --git a/types/index.d.ts.map b/types/index.d.ts.map
diff --git a/types/index.d.ts → types/lib/cli/index.d.ts b/types/index.d.ts → types/lib/cli/index.d.ts
diff --git a/types/lib/cli/index.d.ts.map b/types/lib/cli/index.d.ts.map
diff --git a/types/evinser.d.ts → types/lib/evinser/evinser.d.ts b/types/evinser.d.ts → types/lib/evinser/evinser.d.ts
diff --git a/types/lib/evinser/evinser.d.ts.map b/types/lib/evinser/evinser.d.ts.map
diff --git a/types/analyzer.d.ts → types/lib/helpers/analyzer.d.ts b/types/analyzer.d.ts → types/lib/helpers/analyzer.d.ts
diff --git a/types/lib/helpers/analyzer.d.ts.map b/types/lib/helpers/analyzer.d.ts.map
diff --git a/types/cbomutils.d.ts → types/lib/helpers/cbomutils.d.ts b/types/cbomutils.d.ts → types/lib/helpers/cbomutils.d.ts
diff --git a/types/lib/helpers/cbomutils.d.ts.map b/types/lib/helpers/cbomutils.d.ts.map
diff --git a/types/db.d.ts → types/lib/helpers/db.d.ts b/types/db.d.ts → types/lib/helpers/db.d.ts
diff --git a/types/lib/helpers/db.d.ts.map b/types/lib/helpers/db.d.ts.map
diff --git a/types/display.d.ts → types/lib/helpers/display.d.ts b/types/display.d.ts → types/lib/helpers/display.d.ts
diff --git a/types/lib/helpers/display.d.ts.map b/types/lib/helpers/display.d.ts.map
diff --git a/types/envcontext.d.ts → types/lib/helpers/envcontext.d.ts b/types/envcontext.d.ts → types/lib/helpers/envcontext.d.ts
diff --git a/types/lib/helpers/envcontext.d.ts.map b/types/lib/helpers/envcontext.d.ts.map
diff --git a/types/protobom.d.ts → types/lib/helpers/protobom.d.ts b/types/protobom.d.ts → types/lib/helpers/protobom.d.ts
diff --git a/types/lib/helpers/protobom.d.ts.map b/types/lib/helpers/protobom.d.ts.map
diff --git a/types/utils.d.ts → types/lib/helpers/utils.d.ts b/types/utils.d.ts → types/lib/helpers/utils.d.ts
@@ -408,8 +408,13 @@ export function parsePyProjectToml(tomlFile: string): {};
  *
  * @param {Object} lockData JSON data from poetry.lock
  * @param {string} lockFile Lock file name for evidence
+ * @param {string} pyProjectFile pyproject.toml file
  */
-export function parsePoetrylockData(lockData: any, lockFile: string): Promise<any[] | {
+export function parsePoetrylockData(lockData: any, lockFile: string, pyProjectFile: string): Promise<{
+    pkgList: any[];
+    dependenciesList: any[];
+    rootList?: undefined;
+} | {
     pkgList: any[];
     rootList: any[];
     dependenciesList: {