Skip to content

Commit

Permalink
Use median instead of average to avoid outliers
Browse files Browse the repository at this point in the history
  • Loading branch information
planger committed Oct 4, 2023
1 parent 50ced9e commit b26a5e7
Showing 1 changed file with 11 additions and 6 deletions.
17 changes: 11 additions & 6 deletions scripts/performance-report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ export async function generatePerformanceReport(path: string) {
data: {
datasets: [
{
label: '${valueLabel} (average of 10 runs)',
label: '${valueLabel} (median of 10 runs)',
data: ${JSON.stringify(data)}
},
{
Expand Down Expand Up @@ -289,12 +289,12 @@ export async function readEntries(path: string, values: string[]): Promise<{ val
* This function walks through all value labels and each history element for that value label and applies the following:
* * If the entry represents the measurement of a single run (i.e. ends with `_X`) and has the same entry label as the previous entry, track it to be combined into one entry.
* * If the entry does not represent a single run (i.e. does not end with `_X`) or has a different entry label, conclude the previous collection and start a new one.
* Concluding the previous collection means computing the average and the best of ten value and putting it into a single `ValueHistoryEntry` that combines all runs of the current
* Concluding the previous collection means computing the median and the best of ten value and putting it into a single `ValueHistoryEntry` that combines all runs of the current
* collection. The first element of the collection determines the label of the entire collection.
*
* @param values raw values as read from performance metrics files (key is the label of the value, value is the history of values),
* whereas the ValueHistory.history` of each entry must be sorted ascending by date and run.
* @returns post processed values with the averages and best of ten values of multiple runs.
* @returns post processed values with the median values and best of ten values of multiple runs.
*/
export function processValues(values: Map<string, ValueHistory>): Map<string, ValueHistory> {
const processedValues = new Map<string, ValueHistory>();
Expand Down Expand Up @@ -345,7 +345,7 @@ function toCombinedValueHistoryEntry(entries: ValueHistoryEntry[]): ValueHistory
const combinesRuns = entries.map(entry => entry.entryLabel);
const matchArray = combinesRuns[0].match(matchUntilUnderscoreOrDot);
const entryLabel = matchArray ? matchArray[0] + (entries.length > 1 ? '[]' : '') : combinesRuns[0];
const value = averageValue(values);
const value = medianValue(values);
const best = bestValue(values);
return {
entryLabel,
Expand All @@ -355,8 +355,13 @@ function toCombinedValueHistoryEntry(entries: ValueHistoryEntry[]): ValueHistory
}
}

function averageValue(values: number[]): number {
return values.reduce((a, b) => a + b, 0) / values.length;
function medianValue(values: number[]): number {
const mid = Math.floor(values.length / 2);
const sortedValues = values.sort();
if (values.length % 2 === 0) {
return (sortedValues[mid - 1] + sortedValues[mid]) / 2;
}
return sortedValues[mid];
}

function bestValue(values: number[]): number {
Expand Down

0 comments on commit b26a5e7

Please sign in to comment.