Native jar parsing

.github/workflows/repotests.yml

@@ -31,6 +31,9 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - name: Trim CI agent
         run: |
           chmod +x contrib/free_disk_space.sh
@@ -54,7 +57,7 @@ jobs:
         env:
           CI: true
       - name: Setup Android SDK
-        uses: android-actions/setup-android@v2
+        uses: android-actions/setup-android@v3
       - uses: swift-actions/setup-swift@v1
         if: matrix.os == 'ubuntu-latest'
       - uses: actions/checkout@v4
@@ -356,13 +359,13 @@ jobs:
           ls -ltr bomresults
         shell: bash
       - name: denotests
-        if: github.ref == 'refs/heads/master' && matrix.os == 'ubuntu-latest'
         run: |
-          docker build -t ghcr.io/cyclonedx/cdxgen-deno -f ci/Dockerfile-deno .
-          docker run --rm -t -e "CDXGEN_DEBUG_MODE=debug" -v $(pwd):/app ghcr.io/cyclonedx/cdxgen-deno -p -r -t java /app/repotests/shiftleft-java-example -o /app/denoresults/bom-java.json
-          docker run --rm -t -e "CDXGEN_DEBUG_MODE=debug" -v $(pwd):/app ghcr.io/cyclonedx/cdxgen-deno -p -r -t python /app/repotests/DjanGoat -o /app/denoresults/bom-python.json
-          ls -ltr denoresults
+          deno info bin/cdxgen.js
+          deno info bin/evinse.js
+          deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-deno.json
+          deno run --allow-read --allow-env --allow-run --allow-sys=uid,systemMemoryInfo,gid --allow-write --allow-net bin/cdxgen.js -p -t python repotests/django-DefectDojo -o bomresults/django-DefectDojo-deno.json
       - uses: actions/upload-artifact@v4
+        if: github.ref == 'refs/heads/master' && matrix.os == 'ubuntu-latest'
         with:
           name: bomresults
           path: bomresults

README.md

@@ -383,7 +383,7 @@ cdxgen can retain the dependency tree under the `dependencies` attribute for a s
 | SBOM_SIGN_ALGORITHM          | Signature algorithm. Some valid values are RS256, RS384, RS512, PS256, PS384, PS512, ES256 etc                                       |
 | SBOM_SIGN_PRIVATE_KEY        | Private key to use for signing                                                                                                       |
 | SBOM_SIGN_PUBLIC_KEY         | Optional. Public key to include in the SBOM signature                                                                                |
-| CDX_MAVEN_PLUGIN             | CycloneDX Maven plugin to use. Default "org.cyclonedx:cyclonedx-maven-plugin:2.7.10"                                                 |
+| CDX_MAVEN_PLUGIN             | CycloneDX Maven plugin to use. Default "org.cyclonedx:cyclonedx-maven-plugin:2.7.11"                                                 |
 | CDX_MAVEN_GOAL               | CycloneDX Maven plugin goal to use. Default makeAggregateBom. Other options: makeBom, makePackageBom                                 |
 | CDX_MAVEN_INCLUDE_TEST_SCOPE | Whether test scoped dependencies should be included from Maven projects, Default: true                                               |
 | ASTGEN_IGNORE_DIRS           | Comma separated list of directories to ignore while analyzing using babel. The environment variable is also used by atom and astgen. |

deno.json

@@ -22,7 +22,7 @@
       "include": ["bin/", "*.js"],
       "exclude": ["test/", "data/", "contrib/"]
     },
-    "lock": true,
+    "lock": false,
     "nodeModulesDir": true,
     "unstable": [],
     "test": {

evinser.js

@@ -4,7 +4,8 @@ import {
   getGradleCommand,
   getMavenCommand,
   collectGradleDependencies,
-  collectMvnDependencies
+  collectMvnDependencies,
+  DEBUG_MODE
 } from "./utils.js";
 import { tmpdir } from "node:os";
 import path from "node:path";
@@ -111,7 +112,7 @@ export const catalogMavenDeps = async (
     console.log("About to collect jar dependencies for the path", dirPath);
     const mavenCmd = getMavenCommand(dirPath, dirPath);
     // collect all jars including from the cache if data-flow mode is enabled
-    jarNSMapping = collectMvnDependencies(
+    jarNSMapping = await collectMvnDependencies(
       mavenCmd,
       dirPath,
       false,
@@ -145,7 +146,7 @@ export const catalogGradleDeps = async (dirPath, purlsJars, Namespaces) => {
   );
   const gradleCmd = getGradleCommand(dirPath, dirPath);
   // collect all jars including from the cache if data-flow mode is enabled
-  const jarNSMapping = collectGradleDependencies(
+  const jarNSMapping = await collectGradleDependencies(
     gradleCmd,
     dirPath,
     false,
@@ -1163,7 +1164,7 @@ export const collectDataFlowFrames = async (
               referredPurls.add(ns.purl);
             }
             typePurlsCache[typeFullName] = nsHits;
-          } else {
+          } else if (DEBUG_MODE) {
             console.log("Unable to identify purl for", typeFullName);
           }
         }

index.js

@@ -1055,14 +1055,14 @@ export const createJarBom = async (path, options) => {
   let nsMapping = {};
   const parentComponent = createDefaultParentComponent(path, "maven", options);
   if (options.useGradleCache) {
-    nsMapping = collectGradleDependencies(
+    nsMapping = await collectGradleDependencies(
       getGradleCommand(path, null),
       path,
       false,
       true
     );
   } else if (options.useMavenCache) {
-    nsMapping = collectMvnDependencies(
+    nsMapping = await collectMvnDependencies(
       getMavenCommand(path, null),
       null,
       false,
@@ -1129,7 +1129,7 @@ export const createJavaBom = async (path, options) => {
         console.log(`Retrieving packages from ${path}`);
       }
       const tempDir = mkdtempSync(join(tmpdir(), "war-deps-"));
-      jarNSMapping = collectJarNS(tempDir);
+      jarNSMapping = await collectJarNS(tempDir);
       pkgList = await extractJarArchive(path, tempDir, jarNSMapping);
       if (pkgList.length) {
         pkgList = await getMvnMetadata(pkgList);
@@ -1164,7 +1164,7 @@ export const createJavaBom = async (path, options) => {
     ) {
       const cdxMavenPlugin =
         process.env.CDX_MAVEN_PLUGIN ||
-        "org.cyclonedx:cyclonedx-maven-plugin:2.7.10";
+        "org.cyclonedx:cyclonedx-maven-plugin:2.7.11";
       const cdxMavenGoal = process.env.CDX_MAVEN_GOAL || "makeAggregateBom";
       let mvnArgs = [`${cdxMavenPlugin}:${cdxMavenGoal}`, "-DoutputName=bom"];
       if (includeMavenTestScope) {
@@ -1190,7 +1190,7 @@ export const createJavaBom = async (path, options) => {
         const mavenCmd = getMavenCommand(basePath, path);
         // Should we attempt to resolve class names
         if (options.resolveClass || options.deep) {
-          const tmpjarNSMapping = collectMvnDependencies(
+          const tmpjarNSMapping = await collectMvnDependencies(
             mavenCmd,
             basePath,
             true,
@@ -1557,7 +1557,7 @@ export const createJavaBom = async (path, options) => {
       }
       // Should we attempt to resolve class names
       if (options.resolveClass || options.deep) {
-        jarNSMapping = collectJarNS(GRADLE_CACHE_DIR);
+        jarNSMapping = await collectJarNS(GRADLE_CACHE_DIR);
       }
       pkgList = await getMvnMetadata(pkgList, jarNSMapping);
       return buildBomNSData(options, pkgList, "maven", {
@@ -1854,7 +1854,7 @@ export const createJavaBom = async (path, options) => {
       }
       // Should we attempt to resolve class names
       if (options.resolveClass || options.deep) {
-        jarNSMapping = collectJarNS(SBT_CACHE_DIR);
+        jarNSMapping = await collectJarNS(SBT_CACHE_DIR);
       }
       pkgList = await getMvnMetadata(pkgList, jarNSMapping);
       return buildBomNSData(options, pkgList, "maven", {

utils.js

@@ -6527,12 +6527,12 @@ export const parseSwiftResolved = (resolvedFile) => {
  * @param {boolean} cleanup Remove temporary directories
  * @param {boolean} includeCacheDir Include maven and gradle cache directories
  */
-export const collectMvnDependencies = function (
+export const collectMvnDependencies = async (
   mavenCmd,
   basePath,
   cleanup = true,
   includeCacheDir = false
-) {
+) => {
   let jarNSMapping = {};
   const MAVEN_CACHE_DIR =
     process.env.MAVEN_CACHE_DIR || join(homedir(), ".m2", "repository");
@@ -6573,12 +6573,12 @@ export const collectMvnDependencies = function (
         "3. Ensure the temporary directory is available and has sufficient disk space to copy all the artifacts."
       );
     } else {
-      jarNSMapping = collectJarNS(tempDir);
+      jarNSMapping = await collectJarNS(tempDir);
     }
   }
   if (includeCacheDir || basePath === MAVEN_CACHE_DIR) {
     // slow operation
-    jarNSMapping = collectJarNS(MAVEN_CACHE_DIR);
+    jarNSMapping = await collectJarNS(MAVEN_CACHE_DIR);
   }
 
   // Clean up
@@ -6588,7 +6588,7 @@ export const collectMvnDependencies = function (
   return jarNSMapping;
 };
 
-export const collectGradleDependencies = (
+export const collectGradleDependencies = async (
   gradleCmd,
   basePath,
   cleanup = true, // eslint-disable-line no-unused-vars
@@ -6618,7 +6618,7 @@ export const collectGradleDependencies = (
   for (const apom of pomFiles) {
     pomPathMap[basename(apom)] = apom;
   }
-  const jarNSMapping = collectJarNS(GRADLE_CACHE_DIR, pomPathMap);
+  const jarNSMapping = await collectJarNS(GRADLE_CACHE_DIR, pomPathMap);
   return jarNSMapping;
 };
 
@@ -6630,7 +6630,7 @@ export const collectGradleDependencies = (
  *
  * @return object containing jar name and class list
  */
-export const collectJarNS = function (jarPath, pomPathMap = {}) {
+export const collectJarNS = async (jarPath, pomPathMap = {}) => {
   const jarNSMapping = {};
   console.log(
     `About to identify class names for all jars in the path ${jarPath}`
@@ -6645,14 +6645,10 @@ export const collectJarNS = function (jarPath, pomPathMap = {}) {
       "bin"
     )}`;
   }
-  let jarCommandAvailable = true;
-  // Execute jar tvf to get class names
+  // Parse jar files to get class names
   const jarFiles = getAllFiles(jarPath, "**/*.jar");
   if (jarFiles && jarFiles.length) {
     for (const jf of jarFiles) {
-      if (!jarCommandAvailable) {
-        break;
-      }
       const jarname = jf;
       let pomname =
         pomPathMap[basename(jf).replace(".jar", ".pom")] ||
@@ -6756,44 +6752,9 @@ export const collectJarNS = function (jarPath, pomPathMap = {}) {
         jarNSMapping[purl] = jarNSMapping_cache[purl];
       } else {
         if (DEBUG_MODE) {
-          console.log(`Executing 'jar tf ${jf}'`);
-        }
-        const jarResult = spawnSync("jar", ["-tf", jf], {
-          encoding: "utf-8",
-          shell: isWin,
-          maxBuffer: 50 * 1024 * 1024,
-          env
-        });
-        if (
-          jarResult &&
-          jarResult.stderr &&
-          jarResult.stderr.includes(
-            "is not recognized as an internal or external command"
-          )
-        ) {
-          jarCommandAvailable = false;
-          console.log(
-            "jar command is not available in PATH. Ensure JDK >= 21 is installed and set the environment variables JAVA_HOME and PATH to the bin directory inside JAVA_HOME."
-          );
+          console.log(`Parsing ${jf}`);
         }
-        const consolelines = (jarResult.stdout || "").split("\n");
-        const nsList = consolelines
-          .filter((l) => {
-            return (
-              (l.includes(".class") ||
-                l.includes(".java") ||
-                l.includes(".kt")) &&
-              !l.includes("-INF") &&
-              !l.includes("module-info")
-            );
-          })
-          .map((e) => {
-            return e
-              .replace("\r", "")
-              .replace(/.(class|java|kt)/, "")
-              .replace(/\/$/, "")
-              .replace(/\//g, ".");
-          });
+        const nsList = await getJarClasses(jf);
         jarNSMapping[purl || jf] = {
           jarFile: jf,
           pom: pomData,
@@ -7390,6 +7351,53 @@ export const readZipEntry = async function (
   return retData;
 };
 
+/**
+ * Method to get the classes and relevant sources in a jar file
+ *
+ * @param {string} jarFile Jar file to read
+ *
+ * @returns List of classes and sources matching certain known patterns
+ */
+export const getJarClasses = async function (jarFile) {
+  const retList = [];
+  try {
+    const zip = new StreamZip.async({ file: jarFile });
+    const entriesCount = await zip.entriesCount;
+    if (!entriesCount) {
+      return [];
+    }
+    const entries = await zip.entries();
+    for (const entry of Object.values(entries)) {
+      if (entry.isDirectory) {
+        continue;
+      }
+      if (
+        (entry.name.includes(".class") ||
+          entry.name.includes(".java") ||
+          entry.name.includes(".scala") ||
+          entry.name.includes(".groovy") ||
+          entry.name.includes(".kt")) &&
+        !entry.name.includes("-INF") &&
+        !entry.name.includes("module-info")
+      ) {
+        retList.push(
+          entry.name
+            .replace("\r", "")
+            .replace(/.(class|java|kt|scala|groovy)/g, "")
+            .replace(/\/$/, "")
+            .replace(/\//g, ".")
+        );
+      }
+    }
+    zip.close();
+  } catch (e) {
+    if (DEBUG_MODE) {
+      console.log(`Unable to parse ${jarFile}. Skipping.`);
+    }
+  }
+  return retList;
+};
+
 /**
  * Method to return the gradle command to use.
  *

utils.test.js

@@ -1215,7 +1215,7 @@ test("parse github actions workflow data", () => {
   dep_list = parseGitHubWorkflowData(
     readFileSync("./.github/workflows/repotests.yml", { encoding: "utf-8" })
   );
-  expect(dep_list.length).toEqual(7);
+  expect(dep_list.length).toEqual(8);
   expect(dep_list[0]).toEqual({
     group: "actions",
     name: "checkout",