Skip to content

Commit

Permalink
Merge pull request #7791 from sdedic/gradle/adm-dependencies2
Browse files Browse the repository at this point in the history 
Multi-project audit caching improved
  • Loading branch information
@sdedic
sdedic authored Oct 14, 2024
2 parents 01a871f + aece753 commit bde1b33
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 14 deletions.
21 changes: 16 additions & 5 deletions enterprise/cloud.oracle/src/org/netbeans/modules/cloud/oracle/adm/AuditCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.netbeans.api.project.Project;
import org.netbeans.api.project.ProjectUtils;
import org.openide.filesystems.FileUtil;

/**
* Caches audit results in NB cache folder. Uses Jackson to serialize the audit summary +
Expand Down Expand Up @@ -154,10 +157,18 @@ public void serializeAsField(Object pojo, JsonGenerator jgen, SerializerProvider
}
}
}
public VulnerabilityReport cacheAuditResults(VulnerabilityReport report) throws IOException {
private static String key(Project project, String knowledgeBaseId) {
Project p2 = ProjectUtils.rootOf(project);
String k = "knowledge.segment." + knowledgeBaseId;
if (p2 != project) {
k = k + "." + FileUtil.getRelativePath(p2.getProjectDirectory(), project.getProjectDirectory()).replace('/', '.');
}
return k;
}

public VulnerabilityReport cacheAuditResults(Project project, VulnerabilityReport report) throws IOException {
Properties segments = loadSegments();
String k = "knowledge.segment." + report.summary.getKnowledgeBaseId();

String k = key(project, report.summary.getKnowledgeBaseId());
File cacheDir = Places.getCacheSubdirectory(CACHE_SUBDIR);
Path segPath = cacheDir.toPath().resolve(SEGMENTS_FILE);
boolean writeSegment = !Files.exists(segPath);
Expand Down Expand Up @@ -214,9 +225,9 @@ public VulnerabilityReport cacheAuditResults(VulnerabilityReport report) throws
* @param knowledgeBaseId
* @return
*/
public VulnerabilityReport loadAudit(String knowledgeBaseId) throws IOException {
public VulnerabilityReport loadAudit(Project project, String knowledgeBaseId) throws IOException {
Properties segments = loadSegments();
String k = "knowledge.segment." + knowledgeBaseId;
String k = key(project, knowledgeBaseId);
String segName = segments.getProperty(k);

if (segName == null) {
Expand Down
46 changes: 40 additions & 6 deletions enterprise/cloud.oracle/src/org/netbeans/modules/cloud/oracle/adm/VulnerabilityWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import org.openide.DialogDisplayer;
import org.openide.NotifyDescriptor;
import org.openide.filesystems.FileObject;
import org.openide.filesystems.FileUtil;
import org.openide.util.Exceptions;
import org.openide.util.Lookup;
import org.openide.util.NbBundle;
Expand Down Expand Up @@ -127,6 +128,8 @@ public class VulnerabilityWorker implements ErrorProvider{
// PENDING: should be customizable from project configuration somehow.
private static final String GOV_DETAIL_URL = "https://nvd.nist.gov/vuln/detail/";

private static final String TAG_SUBPROJECT_PATH = "netbeans_cloud_oracle_SubProjectPath"; // NOI18N

// @GuardedBy(self)
private static final HashMap<Project, CacheItem> cache = new HashMap<>();

Expand Down Expand Up @@ -558,6 +561,12 @@ public List<Diagnostic> getDiagnosticsForFile(FileObject file) {

List<Diagnostic> result = new ArrayList<>();
SourceLocation containerLocation = null;
if (!sourceMapping.files.contains(file)) {
return null;
}
if (dependencyResult.getProject().getProjectDirectory() != file.getParent()) {
return null;
}

for (VulnerabilityItem item : this.itemIndex.values()) {
boolean unreported = true;
Expand Down Expand Up @@ -703,6 +712,15 @@ public AuditResult vulnerabilityAudit(Project project, AuditOptions auditOptions
kbItem.refresh();
}
}

private static String findSubprojectPath(Project project) {
Project p2 = ProjectUtils.rootOf(project);
if (p2 == project) {
return "";
} else {
return FileUtil.getRelativePath(p2.getProjectDirectory(), project.getProjectDirectory()).replace('/', '.');
}
}

private AuditResult doFindVulnerability(Project project, String compartmentId, String knowledgeBaseId, String projectDisplayName, AuditOptions auditOptions,
ProgressHandle progressHandle, AtomicBoolean remoteCall) throws AuditException {
Expand All @@ -714,16 +732,15 @@ private AuditResult doFindVulnerability(Project project, String compartmentId, S
boolean auditDone = false;

DependencyResult dr = null;

String subprojectPath = findSubprojectPath(project);
if (!auditOptions.isForceAuditExecution()) {
if (!auditOptions.isDisableCache()) {
try {
savedAudit = AuditCache.getInstance().loadAudit(knowledgeBaseId);
savedAudit = AuditCache.getInstance().loadAudit(project, knowledgeBaseId);
} catch (IOException ex) {
LOG.log(Level.WARNING, "Could not load cached audit data", ex);
}
}

if (savedAudit == null) {
// attempt to find an active most recent audit:
remoteCall.set(true);
Expand All @@ -737,14 +754,30 @@ private AuditResult doFindVulnerability(Project project, String compartmentId, S
.knowledgeBaseId(knowledgeBaseId)
.lifecycleState(LifecycleState.Active)
.sortBy(ListVulnerabilityAuditsRequest.SortBy.TimeCreated)
.limit(200)
.sortOrder(SortOrder.Desc).build();
ListVulnerabilityAuditsResponse response = admClient.listVulnerabilityAudits(request);
if (!response.getVulnerabilityAuditCollection().getItems().isEmpty()) {
VulnerabilityAuditSummary summary = response.getVulnerabilityAuditCollection().getItems().get(0);
VulnerabilityAuditSummary found = null;
D: while (true) {
for (VulnerabilityAuditSummary summary : response.getVulnerabilityAuditCollection().getItems()) {
String p = summary.getFreeformTags().get(TAG_SUBPROJECT_PATH);
if (subprojectPath.equals(p)) {
found = summary;
break D;
}
}
if (response.getOpcNextPage() == null) {
break;
}
request = ListVulnerabilityAuditsRequest
.builder().copy(request).page(response.getOpcNextPage()).build();
response = admClient.listVulnerabilityAudits(request);
}
progressHandle.progress(Bundle.MSG_AuditCollectDependencies());
dr = ProjectDependencies.findDependencies(project, ProjectDependencies.newQuery(Scopes.RUNTIME));
convert(dr.getRoot(), new HashMap<>(), result);
cacheItem = fetchVulnerabilityItems(project, admClient, dr, summary, projectDisplayName);
cacheItem = fetchVulnerabilityItems(project, admClient, dr, found, projectDisplayName);
savedAudit = new VulnerabilityReport(cacheItem.getAudit(), cacheItem.getVulnerabilities());
}
} catch (BmcException ex) {
Expand Down Expand Up @@ -780,6 +813,7 @@ private AuditResult doFindVulnerability(Project project, String compartmentId, S
.buildType(VulnerabilityAudit.BuildType.Maven)
.configuration(auditConfiguration)
.applicationDependencies(result)
.freeformTags(Map.of(TAG_SUBPROJECT_PATH, subprojectPath))
.build();
progressHandle.progress(Bundle.MSG_ExecuteAudit());
CreateVulnerabilityAuditResponse response = admClient.createVulnerabilityAudit(CreateVulnerabilityAuditRequest
Expand Down Expand Up @@ -977,7 +1011,7 @@ private CacheItem fetchVulnerabilityItems(Project project, ApplicationDependency
report.setMappedVulnerabilities(mapped);

try {
AuditCache.getInstance().cacheAuditResults(report);
AuditCache.getInstance().cacheAuditResults(project, report);
} catch (IOException ex) {
LOG.log(Level.WARNING, "Could not cache audit results for knowledgebase {0}, compartment {1}, project {2}", new Object[] {
auditSummary.getKnowledgeBaseId(), auditSummary.getCompartmentId(),
Expand Down
10 changes: 9 additions & 1 deletion java/gradle.java/src/org/netbeans/modules/gradle/java/queries/DependencyText.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,22 @@ public String toString() {
return sb.toString();
}

static String stripStrict(String s) {
if (!s.endsWith("!!")) {
return s;
} else {
return s.substring(0, s.length() - 2);
}
}

public String getContentsOrGav() {
if (contents != null) {
return contents;
} else {
StringBuilder sb = new StringBuilder();
sb.append(group).append(':').append(name);
if (version != null && !version.isEmpty()) {
sb.append(':').append(version);
sb.append(":").append(stripStrict(version));
}
return sb.toString();
}
Expand Down
4 changes: 2 additions & 2 deletions java/gradle.java/src/org/netbeans/modules/gradle/java/queries/TextDependencyScanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -685,13 +685,13 @@ private DependencyText findDependency(Dependency d) {
if (DependencyText.KEYWORD_PROJECT.equals(t.keyword) &&
t.contents.equals(projectName)) {
return t;
} else if (t.keyword == null && t.getContentsOrGav().equals(gav) && scopeMatches(d, t)) {
} else if (t.keyword == null && DependencyText.stripStrict(t.getContentsOrGav()).equals(gav) && scopeMatches(d, t)) {
return t;
}
}

for (DependencyText t : dependencies) {
if (t.keyword == null && t.contents != null && t.contents.equals(groupAndName) && scopeMatches(d, t)) {
if (t.keyword == null && t.contents != null && DependencyText.stripStrict(t.contents).equals(groupAndName) && scopeMatches(d, t)) {
return t;
}
}
Expand Down

0 comments on commit bde1b33

Please sign in to comment.