Skip to content

Commit

Permalink
[QD-7638] Support multiple (& possibly different) fingerprint versions (
Browse files Browse the repository at this point in the history
  • Loading branch information
jckoenen authored Jan 12, 2024
1 parent d93e4be commit ed27354
Show file tree
Hide file tree
Showing 11 changed files with 281 additions and 136 deletions.
103 changes: 103 additions & 0 deletions sarif/src/main/java/com/jetbrains/qodana/sarif/baseline/Baseline.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package com.jetbrains.qodana.sarif.baseline

import com.jetbrains.qodana.sarif.baseline.BaselineCalculation.Options
import com.jetbrains.qodana.sarif.model.Result
import com.jetbrains.qodana.sarif.model.Result.BaselineState
import com.jetbrains.qodana.sarif.model.Run

private typealias FingerprintIndex = MultiMap<String, Result>

private fun <T : Any> Iterable<T?>?.noNulls(): Sequence<T> =
this?.asSequence().orEmpty().filterNotNull()

private val Result.equalIndicators: Sequence<String>
get() = partialFingerprints?.getValues(BaselineCalculation.EQUAL_INDICATOR)?.entries
.noNulls()
.map { (k, v) -> "$k:$v" }
.sortedDescending() // higher versions should have higher priority

internal class DiffState(private val options: Options) {
var new = 0
private set
var unchanged = 0
private set
var absent = 0
private set

val results = mutableListOf<Result>()

fun put(result: Result, state: BaselineState): Boolean {
if (state == BaselineState.UNCHANGED && !options.includeUnchanged) return false
if (state == BaselineState.ABSENT && !options.includeAbsent) return false

results.add(result.withBaselineState(if (options.fillBaselineState) state else null))
when (state) {
BaselineState.NEW -> new++
BaselineState.UNCHANGED -> unchanged++
BaselineState.ABSENT -> absent++
BaselineState.UPDATED -> Unit
}
return true
}
}

/** CAUTION: This mutates results in report and baseline **/
internal fun applyBaseline(report: Run, baseline: Run, options: Options): DiffState {
val state = DiffState(options)

val reportDescriptors = DescriptorLookup(report)
val baselineDescriptors = DescriptorLookup(baseline)
val reportIndex = FingerprintIndex()
val baselineCounter = Counter<ResultKey>()

val undecidedFromReport = report.results.noNulls()
.filterNot { it.baselineState == BaselineState.ABSENT }
.onEach { result ->
result.equalIndicators
.forEach { print -> reportIndex.add(print, result) }
}
.toCollection(IdentitySet(report.results?.size ?: 0))

val undecidedFromBaseline = buildList {
baseline.results.noNulls()
.filterNot { it.baselineState == BaselineState.ABSENT }
.onEach { result -> baselineCounter.increment(ResultKey(result)) }
.forEach { result ->
val removedFromReport = result.equalIndicators
.flatMap(reportIndex::getOrEmpty)
.any(undecidedFromReport::remove)

if (removedFromReport || !options.wasChecked.apply(result)) {
state.put(result, BaselineState.UNCHANGED)
} else {
add(result)
}
}
}


undecidedFromReport.forEach { result ->
val key = ResultKey(result)
val inBaseline = baselineCounter[key]
if (inBaseline <= 0) {
state.put(result, BaselineState.NEW)
} else {
baselineCounter.decrement(key)
state.put(result, BaselineState.UNCHANGED)
}
}

undecidedFromBaseline
.asSequence()
.filter { baselineCounter.decrement(ResultKey(it)) >= 0 }
.forEach { result ->
val added = state.put(result, BaselineState.ABSENT)
if (added && reportDescriptors.findById(result.ruleId) == null) {
baselineDescriptors.findById(result.ruleId)?.addTo(report)
}
}

report.withResults(state.results)

return state
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public void fillBaselineState(SarifReport report, SarifReport baseline) {

if (first.isPresent()) {
Run baselineRun = first.get();
new RunResultGroup(run, baselineRun).build();
applyBaseline(run, baselineRun);
baselineRuns.remove(baselineRun);
} else {
unmatched.add(run);
Expand All @@ -71,13 +71,20 @@ public void fillBaselineState(SarifReport report, SarifReport baseline) {
markRunAsNew(run);
continue;
}
new RunResultGroup(run, baselineRun).build();
applyBaseline(run, baselineRun);
}
}

private void applyBaseline(Run run, Run baseline) {
DiffState state = BaselineKt.applyBaseline(run, baseline, options);
unchangedResults += state.getUnchanged();
newResults += state.getNew();
absentResults += state.getAbsent();
}

private void markRunAsNew(Run run) {
for (Result result : run.getResults()) {
setBaselineState(result, NEW);
result.setBaselineState(NEW);
newResults++;
}
}
Expand All @@ -91,11 +98,9 @@ private String getToolName(Run run) {
}

public static class Options {
public static final Options DEFAULT = new Options();

private final boolean includeAbsent;
private final boolean includeUnchanged;
private final boolean fillBaselineState;
final boolean includeAbsent;
final boolean includeUnchanged;
final boolean fillBaselineState;

/**
* Provides information about incremental build.
Expand All @@ -109,7 +114,9 @@ public static class Options {
* Typically, wasChecked is true if result is in the scope of current check.
*/
private static final Function<Result, Boolean> ALL_CHECKED = (result) -> true;
private final Function<Result, Boolean> wasChecked;
final Function<Result, Boolean> wasChecked;

public static final Options DEFAULT = new Options();

public Options() {
includeAbsent = false;
Expand Down Expand Up @@ -151,123 +158,4 @@ public boolean isFillBaselineState() {
return fillBaselineState;
}
}

private class RunResultGroup {
private final Map<String, List<Result>> baselineHashes = new HashMap<>();
private final Map<String, List<Result>> reportHashes = new HashMap<>();
private final Map<ResultKey, List<Result>> diffBaseline = new HashMap<>();
private final Map<ResultKey, List<Result>> diffReport = new HashMap<>();
private final Run report;
private final DescriptorLookup reportLookup;
private final DescriptorLookup baselineLookup;

public RunResultGroup(Run report, Run baseline) {
this.report = report;
this.reportLookup = new DescriptorLookup(report);
this.baselineLookup = new DescriptorLookup(baseline);
buildMap(baseline, baselineHashes, diffBaseline);
removeProblemsWithState(report, ABSENT);
buildMap(report, reportHashes, diffReport);
}

private void removeProblemsWithState(Run report, Result.BaselineState state) {
report.getResults().removeIf(result -> result.getBaselineState() == state);
}

private void buildMap(Run run, Map<String, List<Result>> map, Map<ResultKey, List<Result>> diffSet) {
for (Result result : run.getResults()) {
if (result.getBaselineState() == ABSENT) continue;
VersionedMap<String> fingerprints = result.getPartialFingerprints();
String equalIndicator = fingerprints != null ? fingerprints.getLastValue(EQUAL_INDICATOR) : null;
if (equalIndicator != null) {
List<Result> resultBucket = map.compute(
equalIndicator,
(key, value) -> value != null ? value : new ArrayList<>());
resultBucket.add(result);
} else {
addToDiff(result, diffSet);
}
}
}

public void addToDiff(Result result, Map<ResultKey, List<Result>> diffSet) {
List<Result> resultBucket = diffSet.compute(
new ResultKey(result),
(key, value) -> value != null ? value : new ArrayList<>()
);
resultBucket.add(result);
}

public void build() {
reportHashes.forEach((hash, results) -> {
if (baselineHashes.containsKey(hash)) {
results.forEach((it) -> setBaselineState(it, UNCHANGED));
unchangedResults += results.size();
} else {
results.forEach((it) -> addToDiff(it, diffReport));
}
});

baselineHashes.forEach((hash, results) -> {
if (!reportHashes.containsKey(hash)) {
for (Result result : results) {
if (options.wasChecked.apply(result)) {
addToDiff(result, diffBaseline);
} else {
result.setBaselineState(UNCHANGED);
report.getResults().add(result);
unchangedResults += 1;
}
}
}
});

diffReport.forEach((key, reportDiffBucket) -> {
List<Result> baselineDiffBucket = diffBaseline.getOrDefault(key, Collections.emptyList());
for (Result result : reportDiffBucket) {
if (baselineDiffBucket.isEmpty()) {
setBaselineState(result, NEW);
newResults++;
} else {
setBaselineState(result, UNCHANGED);
baselineDiffBucket.remove(baselineDiffBucket.size() - 1);
unchangedResults++;
}
}
});

diffBaseline.entrySet().stream().flatMap((it) -> it.getValue().stream()).forEach(result -> {
if (options.wasChecked.apply(result)) {
if (options.includeAbsent) {
setBaselineState(result, ABSENT);
absentResults++;
report.getResults().add(result);
if (reportLookup.findById(result.getRuleId()) == null) {
DescriptorWithLocation descriptor = baselineLookup.findById(result.getRuleId());
if (descriptor != null) descriptor.addTo(report);
}
}
} else {
result.setBaselineState(UNCHANGED);
report.getResults().add(result);
unchangedResults += 1;
}
});

if (!options.includeUnchanged) {
removeProblemsWithState(report, UNCHANGED);
unchangedResults = 0;
}

if (!options.fillBaselineState) {
for (Result result : report.getResults()) {
result.setBaselineState(null);
}
}
}
}

private void setBaselineState(Result result, Result.BaselineState state) {
result.setBaselineState(state);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package com.jetbrains.qodana.sarif.baseline

import java.util.*

internal class MultiMap<K, V> private constructor(
private val underlying: MutableMap<K, MutableList<V>>
) : Iterable<Map.Entry<K, List<V>>> {
constructor() : this(mutableMapOf())

fun add(key: K, value: V) {
underlying.compute(key) { _, old -> old?.apply { add(value) } ?: mutableListOf(value) }
}

fun getOrEmpty(key: K): MutableList<V> = underlying[key] ?: mutableListOf()

override fun iterator(): Iterator<Map.Entry<K, List<V>>> = underlying.iterator()
}

internal class IdentitySet<T> private constructor(
private val map: IdentityHashMap<T, Int>
) : MutableSet<T> by map.keys {
constructor(expectedSize: Int) : this(IdentityHashMap(expectedSize))

private var insertCounter = 0

override fun add(element: T): Boolean = map.putIfAbsent(element, insertCounter++) == null
override fun addAll(elements: Collection<T>): Boolean = elements.fold(false) { r, e -> add(e) || r }

override fun iterator(): MutableIterator<T> = object : MutableIterator<T> {
private val underlying = map.entries.sortedBy(MutableMap.MutableEntry<T, Int>::value).iterator()

override fun hasNext(): Boolean = underlying.hasNext()
override fun next(): T = underlying.next().key
override fun remove() = throw UnsupportedOperationException("Cannot remove elements from this iterator")
}
}

internal class Counter<T> {
private val underlying: MutableMap<T, Int> = mutableMapOf()

operator fun get(key: T) = underlying.getOrDefault(key, 0)
fun increment(key: T) = underlying.compute(key) { _, o -> (o ?: 0).inc() }!!
fun decrement(key: T) = underlying.compute(key) { _, o -> (o ?: 0).dec() }!!
}
Loading

0 comments on commit ed27354

Please sign in to comment.