Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multi-project audit caching improved #7791

Merged
merged 2 commits into from
Oct 14, 2024
Merged

Multi-project audit caching improved #7791

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3f331a2
Support for !!strict modifier for versions
sdedic Sep 25, 2024
aece753
Cache each project in the knowledgebase separately
sdedic Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions enterprise/cloud.oracle/src/org/netbeans/modules/cloud/oracle/adm/AuditCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.netbeans.api.project.Project;
import org.netbeans.api.project.ProjectUtils;
import org.openide.filesystems.FileUtil;

/**
* Caches audit results in NB cache folder. Uses Jackson to serialize the audit summary +
Expand Down Expand Up @@ -154,10 +157,18 @@ public void serializeAsField(Object pojo, JsonGenerator jgen, SerializerProvider
}
}
}
public VulnerabilityReport cacheAuditResults(VulnerabilityReport report) throws IOException {
private static String key(Project project, String knowledgeBaseId) {
Project p2 = ProjectUtils.rootOf(project);
String k = "knowledge.segment." + knowledgeBaseId;
if (p2 != project) {
k = k + "." + FileUtil.getRelativePath(p2.getProjectDirectory(), project.getProjectDirectory()).replace('/', '.');
}
return k;
}

public VulnerabilityReport cacheAuditResults(Project project, VulnerabilityReport report) throws IOException {
Properties segments = loadSegments();
String k = "knowledge.segment." + report.summary.getKnowledgeBaseId();

String k = key(project, report.summary.getKnowledgeBaseId());
File cacheDir = Places.getCacheSubdirectory(CACHE_SUBDIR);
Path segPath = cacheDir.toPath().resolve(SEGMENTS_FILE);
boolean writeSegment = !Files.exists(segPath);
Expand Down Expand Up @@ -214,9 +225,9 @@ public VulnerabilityReport cacheAuditResults(VulnerabilityReport report) throws
* @param knowledgeBaseId
* @return
*/
public VulnerabilityReport loadAudit(String knowledgeBaseId) throws IOException {
public VulnerabilityReport loadAudit(Project project, String knowledgeBaseId) throws IOException {
Properties segments = loadSegments();
String k = "knowledge.segment." + knowledgeBaseId;
String k = key(project, knowledgeBaseId);
String segName = segments.getProperty(k);

if (segName == null) {
Expand Down
46 changes: 40 additions & 6 deletions enterprise/cloud.oracle/src/org/netbeans/modules/cloud/oracle/adm/VulnerabilityWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import org.openide.DialogDisplayer;
import org.openide.NotifyDescriptor;
import org.openide.filesystems.FileObject;
import org.openide.filesystems.FileUtil;
import org.openide.util.Exceptions;
import org.openide.util.Lookup;
import org.openide.util.NbBundle;
Expand Down Expand Up @@ -127,6 +128,8 @@ public class VulnerabilityWorker implements ErrorProvider{
// PENDING: should be customizable from project configuration somehow.
private static final String GOV_DETAIL_URL = "https://nvd.nist.gov/vuln/detail/";

private static final String TAG_SUBPROJECT_PATH = "netbeans_cloud_oracle_SubProjectPath"; // NOI18N

// @GuardedBy(self)
private static final HashMap<Project, CacheItem> cache = new HashMap<>();

Expand Down Expand Up @@ -558,6 +561,12 @@ public List<Diagnostic> getDiagnosticsForFile(FileObject file) {

List<Diagnostic> result = new ArrayList<>();
SourceLocation containerLocation = null;
if (!sourceMapping.files.contains(file)) {
return null;
}
if (dependencyResult.getProject().getProjectDirectory() != file.getParent()) {
return null;
}

for (VulnerabilityItem item : this.itemIndex.values()) {
boolean unreported = true;
Expand Down Expand Up @@ -703,6 +712,15 @@ public AuditResult vulnerabilityAudit(Project project, AuditOptions auditOptions
kbItem.refresh();
}
}

private static String findSubprojectPath(Project project) {
Project p2 = ProjectUtils.rootOf(project);
if (p2 == project) {
return "";
} else {
return FileUtil.getRelativePath(p2.getProjectDirectory(), project.getProjectDirectory()).replace('/', '.');
}
}

private AuditResult doFindVulnerability(Project project, String compartmentId, String knowledgeBaseId, String projectDisplayName, AuditOptions auditOptions,
ProgressHandle progressHandle, AtomicBoolean remoteCall) throws AuditException {
Expand All @@ -714,16 +732,15 @@ private AuditResult doFindVulnerability(Project project, String compartmentId, S
boolean auditDone = false;

DependencyResult dr = null;

String subprojectPath = findSubprojectPath(project);
if (!auditOptions.isForceAuditExecution()) {
if (!auditOptions.isDisableCache()) {
try {
savedAudit = AuditCache.getInstance().loadAudit(knowledgeBaseId);
savedAudit = AuditCache.getInstance().loadAudit(project, knowledgeBaseId);
} catch (IOException ex) {
LOG.log(Level.WARNING, "Could not load cached audit data", ex);
}
}

if (savedAudit == null) {
// attempt to find an active most recent audit:
remoteCall.set(true);
Expand All @@ -737,14 +754,30 @@ private AuditResult doFindVulnerability(Project project, String compartmentId, S
.knowledgeBaseId(knowledgeBaseId)
.lifecycleState(LifecycleState.Active)
.sortBy(ListVulnerabilityAuditsRequest.SortBy.TimeCreated)
.limit(200)
.sortOrder(SortOrder.Desc).build();
ListVulnerabilityAuditsResponse response = admClient.listVulnerabilityAudits(request);
if (!response.getVulnerabilityAuditCollection().getItems().isEmpty()) {
VulnerabilityAuditSummary summary = response.getVulnerabilityAuditCollection().getItems().get(0);
VulnerabilityAuditSummary found = null;
D: while (true) {
for (VulnerabilityAuditSummary summary : response.getVulnerabilityAuditCollection().getItems()) {
String p = summary.getFreeformTags().get(TAG_SUBPROJECT_PATH);
if (subprojectPath.equals(p)) {
found = summary;
break D;
}
}
if (response.getOpcNextPage() == null) {
break;
}
request = ListVulnerabilityAuditsRequest
.builder().copy(request).page(response.getOpcNextPage()).build();
response = admClient.listVulnerabilityAudits(request);
}
progressHandle.progress(Bundle.MSG_AuditCollectDependencies());
dr = ProjectDependencies.findDependencies(project, ProjectDependencies.newQuery(Scopes.RUNTIME));
convert(dr.getRoot(), new HashMap<>(), result);
cacheItem = fetchVulnerabilityItems(project, admClient, dr, summary, projectDisplayName);
cacheItem = fetchVulnerabilityItems(project, admClient, dr, found, projectDisplayName);
savedAudit = new VulnerabilityReport(cacheItem.getAudit(), cacheItem.getVulnerabilities());
}
} catch (BmcException ex) {
Expand Down Expand Up @@ -780,6 +813,7 @@ private AuditResult doFindVulnerability(Project project, String compartmentId, S
.buildType(VulnerabilityAudit.BuildType.Maven)
.configuration(auditConfiguration)
.applicationDependencies(result)
.freeformTags(Map.of(TAG_SUBPROJECT_PATH, subprojectPath))
.build();
progressHandle.progress(Bundle.MSG_ExecuteAudit());
CreateVulnerabilityAuditResponse response = admClient.createVulnerabilityAudit(CreateVulnerabilityAuditRequest
Expand Down Expand Up @@ -977,7 +1011,7 @@ private CacheItem fetchVulnerabilityItems(Project project, ApplicationDependency
report.setMappedVulnerabilities(mapped);

try {
AuditCache.getInstance().cacheAuditResults(report);
AuditCache.getInstance().cacheAuditResults(project, report);
} catch (IOException ex) {
LOG.log(Level.WARNING, "Could not cache audit results for knowledgebase {0}, compartment {1}, project {2}", new Object[] {
auditSummary.getKnowledgeBaseId(), auditSummary.getCompartmentId(),
Expand Down
10 changes: 9 additions & 1 deletion java/gradle.java/src/org/netbeans/modules/gradle/java/queries/DependencyText.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,22 @@ public String toString() {
return sb.toString();
}

static String stripStrict(String s) {
if (!s.endsWith("!!")) {
return s;
} else {
return s.substring(0, s.length() - 2);
}
}

public String getContentsOrGav() {
if (contents != null) {
return contents;
} else {
StringBuilder sb = new StringBuilder();
sb.append(group).append(':').append(name);
if (version != null && !version.isEmpty()) {
sb.append(':').append(version);
sb.append(":").append(stripStrict(version));
}
return sb.toString();
}
Expand Down
4 changes: 2 additions & 2 deletions java/gradle.java/src/org/netbeans/modules/gradle/java/queries/TextDependencyScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -685,13 +685,13 @@ private DependencyText findDependency(Dependency d) {
if (DependencyText.KEYWORD_PROJECT.equals(t.keyword) &&
t.contents.equals(projectName)) {
return t;
} else if (t.keyword == null && t.getContentsOrGav().equals(gav) && scopeMatches(d, t)) {
} else if (t.keyword == null && DependencyText.stripStrict(t.getContentsOrGav()).equals(gav) && scopeMatches(d, t)) {
return t;
}
}

for (DependencyText t : dependencies) {
if (t.keyword == null && t.contents != null && t.contents.equals(groupAndName) && scopeMatches(d, t)) {
if (t.keyword == null && t.contents != null && DependencyText.stripStrict(t.contents).equals(groupAndName) && scopeMatches(d, t)) {
return t;
}
}
Expand Down
Loading