Skip to content

Commit

Permalink
[Gradle] Scanning of all modules fixed (#1383)
Browse files Browse the repository at this point in the history 
* Missed a part for deep-scanning

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>

* Make sure the rootProject isn't added in the list of components either

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>

* Added elasticsearch to repotests

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>

* Made multithreaded gradle the default and removed singlethreaded code

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>

* Removed printing output

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>

* Minimized the adding/removing of the ':'-prefix

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>

* Added the module-name to the metadata for all modules, not just those resolved from NPM

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>

---------

Signed-off-by: Roland Asmann <roland.asmann@gmail.com>
  • Loading branch information
@malice00
malice00 authored Sep 22, 2024
1 parent 5385422 commit f1b4be6
Show file tree
Hide file tree
Showing 5 changed files with 155 additions and 249 deletions.
14 changes: 10 additions & 4 deletions .github/workflows/repotests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,10 @@ jobs:
repository: 'malice00/cdxgen-expo-test'
ref: 'android'
path: 'repotests/expo-test'
- uses: actions/checkout@v4
with:
repository: 'elastic/elasticsearch'
path: 'repotests/elasticsearch'
- uses: dtolnay/rust-toolchain@stable
- name: setup sdkman
run: |
Expand Down Expand Up @@ -451,10 +455,12 @@ jobs:
- name: repotests expo
run: |
cd repotests/expo-test && npm ci && cd ../..
GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo.json
GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_MULTI_THREADED=true GRADLE_SKIP_MODULES=root bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-multi.json
custom-json-diff -i bomresults/bom-expo.json bomresults/bom-expo-multi.json -o bomresults/diff-expo
GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_MULTI_THREADED=true GRADLE_SKIP_MODULES=root GRADLE_RESOLVE_FROM_NODE=true bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-npm.json
GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo.json
GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root GRADLE_RESOLVE_FROM_NODE=true bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-npm.json
shell: bash
- name: repotests elasticsearch
run: |
bin/cdxgen.js -t gradle repotests/elasticsearch -o bomresults/bom-elasticsearch.json
shell: bash
- name: jenkins plugins
run: |
Expand Down
2 changes: 1 addition & 1 deletion docs/ENV.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ The following environment variables are available to configure the bom generatio
| GRADLE_CMD | Set to override gradle command |
| GRADLE_DEPENDENCY_TASK | By default cdxgen use the task "dependencies" to collect packages. Set to override the task name. |
| GRADLE_RESOLVE_FROM_NODE | If some of your gradle modules are included from node (eg when using expo or react-native), set this to true to use the npm-packages as your dependencies. The big advantage of this, is that the generated purls will be of actually known components (eg in OSS Index) instead of generic names for the packages. |
| GRADLE_SKIP_MODULES | Comma-separated list of modules to skip during the "dependencies" task. This can be useful if you have modules that would fail the gradle build, eg when they do not have dependencies in the given configuration. Use "root" if the top most module should be skipped, use the name (without leading ":") for all others. |
| GRADLE_SKIP_MODULES | Comma-separated list of modules to skip during the "dependencies" task. This can be useful if you have modules that would fail the gradle build, eg when they do not have dependencies in the given configuration. Use "root" if the top most module should be skipped, use their gradle-name (so WITH leading ":") for all others. |
| SBT_CACHE_DIR | Specify sbt cache directory. Useful for class name resolving |
| FETCH_LICENSE | Set this variable to `true` or `1` to fetch license information from the registry. npm and golang |
| SEARCH_MAVEN_ORG | If maven metadata is missing in jar file, a search is performed on search.maven.org. Set to `false` or `0` to disable search. (defaults to `true`) |
Expand Down
267 changes: 90 additions & 177 deletions lib/cli/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1635,7 +1635,7 @@ export async function createJavaBom(path, options) {
gradleFiles?.length &&
isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options)
) {
let retMap = executeGradleProperties(gradleRootPath, null);
const retMap = executeGradleProperties(gradleRootPath, null);
const allProjectsStr = retMap.projects || [];
const rootProject = retMap.rootProject;
if (rootProject) {
Expand All @@ -1647,50 +1647,31 @@ export async function createJavaBom(path, options) {
}
// Get the sub-project properties and set the root dependencies
if (allProjectsStr?.length) {
if (process.env.GRADLE_MULTI_THREADED) {
const parallelPropTaskOut = executeParallelGradleProperties(
gradleRootPath,
allProjectsStr,
);
const splitPropTaskOut = splitOutputByGradleProjects(
parallelPropTaskOut,
["properties"],
);
const parallelPropTaskOut = executeParallelGradleProperties(
gradleRootPath,
allProjectsStr,
);
const splitPropTaskOut = splitOutputByGradleProjects(
parallelPropTaskOut,
["properties"],
);

for (const [key, propTaskOut] of splitPropTaskOut.entries()) {
const retMap = parseGradleProperties(propTaskOut);
const rootSubProject = retMap.rootProject;
if (rootSubProject) {
const rootSubProjectObj = await buildObjectForGradleModule(
rootSubProject.replace(/^:/, ""),
retMap.metadata,
);
if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) {
allProjects.push(rootSubProjectObj);
rootDependsOn.push(rootSubProjectObj["bom-ref"]);
allProjectsAddedPurls.push(rootSubProjectObj["purl"]);
}
gradleModules.set(key.replace(/^:/, ""), rootSubProjectObj);
}
}
} else {
for (const spstr of allProjectsStr) {
retMap = executeGradleProperties(gradleRootPath, spstr);
const rootSubProject = retMap.rootProject;
if (rootSubProject) {
const rootSubProjectObj = await buildObjectForGradleModule(
rootSubProject.replace(/^:/, ""),
retMap.metadata,
);
if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) {
allProjects.push(rootSubProjectObj);
rootDependsOn.push(rootSubProjectObj["bom-ref"]);
allProjectsAddedPurls.push(rootSubProjectObj["purl"]);
}
gradleModules.set(spstr.replace(/^:/, ""), rootSubProjectObj);
for (const [key, propTaskOut] of splitPropTaskOut.entries()) {
const retMap = parseGradleProperties(propTaskOut, key);
const rootSubProject = retMap.rootProject;
if (rootSubProject) {
const rootSubProjectObj = await buildObjectForGradleModule(
rootSubProject,
retMap.metadata,
);
if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) {
allProjects.push(rootSubProjectObj);
rootDependsOn.push(rootSubProjectObj["bom-ref"]);
allProjectsAddedPurls.push(rootSubProjectObj["purl"]);
}
gradleModules.set(key, rootSubProjectObj);
}
} //end else
}
// Bug #317 fix
parentComponent.components = allProjects.flatMap((s) => {
delete s.qualifiers;
Expand All @@ -1714,155 +1695,87 @@ export async function createJavaBom(path, options) {
? process.env.GRADLE_DEPENDENCY_TASK
: "dependencies";

if (process.env.GRADLE_MULTI_THREADED) {
const gradleSubCommands = [];
const modulesToSkip = process.env.GRADLE_SKIP_MODULES
? process.env.GRADLE_SKIP_MODULES.split(",")
: [];
if (!modulesToSkip.includes("root")) {
gradleSubCommands.push(gradleDepTask);
}
for (const [key, sp] of gradleModules) {
//create single command for dependencies tasks on all subprojects
if (sp.purl !== parentComponent.purl && !modulesToSkip.includes(key)) {
gradleSubCommands.push(`:${key}:${gradleDepTask}`);
}
}
const gradleArguments = buildGradleCommandArguments(
process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
gradleSubCommands,
process.env.GRADLE_ARGS_DEPENDENCIES
? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
: [],
);
console.log(
"Executing",
gradleCmd,
gradleArguments.join(" "),
"in",
gradleRootPath,
);
const sresult = spawnSync(gradleCmd, gradleArguments, {
cwd: gradleRootPath,
encoding: "utf-8",
timeout: TIMEOUT_MS,
maxBuffer: MAX_BUFFER,
});
const gradleSubCommands = [];
const modulesToSkip = process.env.GRADLE_SKIP_MODULES
? process.env.GRADLE_SKIP_MODULES.split(",")
: [];
if (!modulesToSkip.includes("root")) {
gradleSubCommands.push(gradleDepTask);
}
for (const [key, sp] of gradleModules) {
//create single command for dependencies tasks on all subprojects
if (sp.purl !== parentComponent.purl && !modulesToSkip.includes(key)) {
gradleSubCommands.push(`${key}:${gradleDepTask}`);
}
}
const gradleArguments = buildGradleCommandArguments(
process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
gradleSubCommands,
process.env.GRADLE_ARGS_DEPENDENCIES
? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
: [],
);
console.log(
"Executing",
gradleCmd,
gradleArguments.join(" "),
"in",
gradleRootPath,
);
const sresult = spawnSync(gradleCmd, gradleArguments, {
cwd: gradleRootPath,
encoding: "utf-8",
timeout: TIMEOUT_MS,
maxBuffer: MAX_BUFFER,
});

if (sresult.status !== 0 || sresult.error) {
if (options.failOnError || DEBUG_MODE) {
console.error(sresult.stdout, sresult.stderr);
}
options.failOnError && process.exit(1);
}
const sstdout = sresult.stdout;
if (sstdout) {
const cmdOutput = Buffer.from(sstdout).toString();
const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [
gradleDepTask,
]);
for (const [key, sp] of gradleModules) {
const parsedList = await parseGradleDep(
perProjectOutput.has(key) ? perProjectOutput.get(key) : "",
key,
gradleModules,
gradleRootPath,
);
const dlist = parsedList.pkgList;
if (parsedList.dependenciesList && parsedList.dependenciesList) {
dependencies = mergeDependencies(
dependencies,
parsedList.dependenciesList,
parentComponent,
);
}
if (dlist?.length) {
if (DEBUG_MODE) {
console.log(
"Found",
dlist.length,
"packages in gradle project",
key,
);
}
pkgList = pkgList.concat(dlist);
}
}
}
} else {
if (DEBUG_MODE) {
console.log(
"Try the new multi-threaded mode for gradle. Set the environment variable GRADLE_MULTI_THREADED to true to enable this.",
);
if (sresult.status !== 0 || sresult.error) {
if (options.failOnError || DEBUG_MODE) {
console.error(sresult.stdout, sresult.stderr);
}
options.failOnError && process.exit(1);
}
const sstdout = sresult.stdout;
if (sstdout) {
const cmdOutput = Buffer.from(sstdout).toString();
const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [
gradleDepTask,
]);
for (const [key, sp] of gradleModules) {
const gradleArguments = buildGradleCommandArguments(
process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
[
sp.purl === parentComponent.purl
? gradleDepTask
: `:${key}:${gradleDepTask}`,
],
process.env.GRADLE_ARGS_DEPENDENCIES
? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
: [],
);

console.log(
"Executing",
gradleCmd,
gradleArguments.join(" "),
"in",
const parsedList = await parseGradleDep(
perProjectOutput.has(key) ? perProjectOutput.get(key) : "",
key,
gradleModules,
gradleRootPath,
);
const sresult = spawnSync(gradleCmd, gradleArguments, {
cwd: gradleRootPath,
encoding: "utf-8",
timeout: TIMEOUT_MS,
maxBuffer: MAX_BUFFER,
});
if (sresult.status !== 0 || sresult.error) {
if (options.failOnError || DEBUG_MODE) {
console.error(sresult.stdout, sresult.stderr);
}
options.failOnError && process.exit(1);
}
const sstdout = sresult.stdout;
if (sstdout) {
const cmdOutput = Buffer.from(sstdout).toString();
const parsedList = await parseGradleDep(
cmdOutput,
key,
gradleModules,
gradleRootPath,
const dlist = parsedList.pkgList;
if (parsedList.dependenciesList && parsedList.dependenciesList) {
dependencies = mergeDependencies(
dependencies,
parsedList.dependenciesList,
parentComponent,
);
const dlist = parsedList.pkgList;
if (parsedList.dependenciesList && parsedList.dependenciesList) {
dependencies = mergeDependencies(
dependencies,
parsedList.dependenciesList,
parentComponent,
}
if (dlist?.length) {
if (DEBUG_MODE) {
console.log(
"Found",
dlist.length,
"packages in gradle project",
key,
);
}
if (dlist?.length) {
if (DEBUG_MODE) {
console.log(
"Found",
dlist.length,
"packages in gradle project",
key,
);
}
pkgList = pkgList.concat(dlist);
}
pkgList = pkgList.concat(dlist);
}
} // for
}
}
if (pkgList.length) {
if (parentComponent.components?.length) {
for (const subProj of parentComponent.components) {
pkgList = pkgList.filter(
(pkg) => pkg["bom-ref"] !== subProj["bom-ref"],
(pkg) =>
pkg["bom-ref"] !== subProj["bom-ref"] &&
pkg["bom-ref"] !== parentComponent["bom-ref"],
);
}
}
Expand Down
Loading

0 comments on commit f1b4be6

Please sign in to comment.