diff --git a/JetStreamDriver.js b/JetStreamDriver.js index 7e0eb620..e556c758 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -126,21 +126,6 @@ function assert(b, m = "") { throw new Error(`Bad assertion: ${m}`); } -function firstID(benchmark) { - return `results-cell-${benchmark.name}-first`; -} - -function worst4ID(benchmark) { - return `results-cell-${benchmark.name}-worst4`; -} - -function avgID(benchmark) { - return `results-cell-${benchmark.name}-avg`; -} - -function scoreID(benchmark) { - return `results-cell-${benchmark.name}-score`; -} function mean(values) { assert(values instanceof Array); @@ -679,9 +664,20 @@ class Benchmark { } get score() { + const subScores = Object.values(this.subScores()); + return geomean(subScores); + } + + subScores() { throw new Error("Subclasses need to implement this"); } + allScores() { + const allScores = this.subScores(); + allScores["Score"] = this.score; + return allScores; + } + get prerunCode() { return null; } get preIterationCode() { @@ -993,15 +989,23 @@ class Benchmark { this.preloads = Object.entries(this.plan.preload ?? {}); } - scoreIdentifiers() { throw new Error("Must be implemented by subclasses"); } + scoreIdentifiers() { + const ids = Object.keys(this.allScores()).map(name => this.scoreIdentifier(name)); + return ids; + } + + scoreIdentifier(scoreName) { + return `results-cell-${this.name}-${scoreName}`; + } updateUIBeforeRun() { - if (!isInBrowser) { - if (!dumpJSONResults) - console.log(`Running ${this.name}:`); - return; - } + if (!dumpJSONResults) + console.log(`Running ${this.name}:`); + if (isInBrowser) + this.updateUIBeforeRunInBrowser(); + } + updateUIBeforeRunInBrowser() { const containerUI = document.getElementById("results"); const resultsBenchmarkUI = document.getElementById(`benchmark-${this.name}`); containerUI.insertBefore(resultsBenchmarkUI, containerUI.firstChild); @@ -1012,13 +1016,43 @@ class Benchmark { } updateUIAfterRun() { - if (!isInBrowser) + const scoreEntries = Object.entries(this.allScores()); + if (isInBrowser) + this.updateUIAfterRunInBrowser(scoreEntries); + if (dumpJSONResults) return; + this.updateConsoleAfterRun(scoreEntries); + } + updateUIAfterRunInBrowser(scoreEntries) { const benchmarkResultsUI = document.getElementById(`benchmark-${this.name}`); benchmarkResultsUI.classList.remove("benchmark-running"); benchmarkResultsUI.classList.add("benchmark-done"); + for (const [name, value] of scoreEntries) + document.getElementById(this.scoreIdentifier(name)).innerHTML = uiFriendlyScore(value); + } + + updateConsoleAfterRun(scoreEntries) { + // FIXME: consider removing this mapping. + // Rename for backwards compatibility. + const legacyScoreNameMap = { + __proto__: null, + "First": "Startup", + "Worst": "Worst Case", + "MainRun": "Tests", + "Runtime": "Run time", + }; + for (let [name, value] of scoreEntries) { + if (name in legacyScoreNameMap) + name = legacyScoreNameMap[name]; + console.log(` ${name}:`, uiFriendlyScore(value)); + } + if (RAMification) { + console.log(" Current Footprint:", uiFriendlyNumber(this.currentFootprint)); + console.log(" Peak Footprint:", uiFriendlyNumber(this.peakFootprint)); + } + console.log(" Wall time:", uiFriendlyDuration(this.endTime - this.startTime)); } }; @@ -1063,10 +1097,6 @@ class DefaultBenchmark extends Benchmark { this.averageScore = toScore(this.averageTime); } - get score() { - return geomean([this.firstIterationScore, this.worst4Score, this.averageScore]); - } - subScores() { return { "First": this.firstIterationScore, @@ -1074,35 +1104,6 @@ class DefaultBenchmark extends Benchmark { "Average": this.averageScore, }; } - - scoreIdentifiers() { - return [firstID(this), worst4ID(this), avgID(this), scoreID(this)]; - } - - updateUIAfterRun() { - super.updateUIAfterRun(); - - if (isInBrowser) { - document.getElementById(firstID(this)).innerHTML = uiFriendlyScore(this.firstIterationScore); - document.getElementById(worst4ID(this)).innerHTML = uiFriendlyScore(this.worst4Score); - document.getElementById(avgID(this)).innerHTML = uiFriendlyScore(this.averageScore); - document.getElementById(scoreID(this)).innerHTML = uiFriendlyScore(this.score); - return; - } - - if (dumpJSONResults) - return; - - console.log(" Startup:", uiFriendlyScore(this.firstIterationScore)); - console.log(" Worst Case:", uiFriendlyScore(this.worst4Score)); - console.log(" Average:", uiFriendlyScore(this.averageScore)); - console.log(" Score:", uiFriendlyScore(this.score)); - if (RAMification) { - console.log(" Current Footprint:", uiFriendlyNumber(this.currentFootprint)); - console.log(" Peak Footprint:", uiFriendlyNumber(this.peakFootprint)); - } - console.log(" Wall time:", uiFriendlyDuration(this.endTime - this.startTime)); - } } class AsyncBenchmark extends DefaultBenchmark { @@ -1250,10 +1251,6 @@ class WSLBenchmark extends Benchmark { this.mainRunScore = toScore(results[1]); } - get score() { - return geomean([this.stdlibScore, this.mainRunScore]); - } - get runnerCode() { return ` let benchmark = new Benchmark(); @@ -1292,33 +1289,6 @@ class WSLBenchmark extends Benchmark { "MainRun": this.mainRunScore, }; } - - scoreIdentifiers() { - return ["wsl-stdlib-score", "wsl-tests-score", "wsl-score-score"]; - } - - updateUIAfterRun() { - super.updateUIAfterRun(); - - if (isInBrowser) { - document.getElementById("wsl-stdlib-score").innerHTML = uiFriendlyScore(this.stdlibScore); - document.getElementById("wsl-tests-score").innerHTML = uiFriendlyScore(this.mainRunScore); - document.getElementById("wsl-score-score").innerHTML = uiFriendlyScore(this.score); - return; - } - - if (dumpJSONResults) - return; - - console.log(" Stdlib:", uiFriendlyScore(this.stdlibScore)); - console.log(" Tests:", uiFriendlyScore(this.mainRunScore)); - console.log(" Score:", uiFriendlyScore(this.score)); - if (RAMification) { - console.log(" Current Footprint:", uiFriendlyNumber(this.currentFootprint)); - console.log(" Peak Footprint:", uiFriendlyNumber(this.peakFootprint)); - } - console.log(" Wall time:", uiFriendlyDuration(this.endTime - this.startTime)); - } }; class WasmLegacyBenchmark extends Benchmark { @@ -1338,10 +1308,6 @@ class WasmLegacyBenchmark extends Benchmark { this.runScore = toScore(results[1]); } - get score() { - return geomean([this.startupScore, this.runScore]); - } - get prerunCode() { const str = ` let verbose = false; @@ -1464,43 +1430,6 @@ class WasmLegacyBenchmark extends Benchmark { "Runtime": this.runScore, }; } - - get startupID() { - return `wasm-startup-id${this.name}`; - } - get runID() { - return `wasm-run-id${this.name}`; - } - get scoreID() { - return `wasm-score-id${this.name}`; - } - - scoreIdentifiers() { - return [this.startupID, this.runID, this.scoreID]; - } - - updateUIAfterRun() { - super.updateUIAfterRun(); - - if (isInBrowser) { - document.getElementById(this.startupID).innerHTML = uiFriendlyScore(this.startupScore); - document.getElementById(this.runID).innerHTML = uiFriendlyScore(this.runScore); - document.getElementById(this.scoreID).innerHTML = uiFriendlyScore(this.score); - return; - } - - if (dumpJSONResults) - return; - - console.log(" Startup:", uiFriendlyScore(this.startupScore)); - console.log(" Run time:", uiFriendlyScore(this.runScore)); - console.log(" Score:", uiFriendlyScore(this.score)); - if (RAMification) { - console.log(" Current Footprint:", uiFriendlyNumber(this.currentFootprint)); - console.log(" Peak Footprint:", uiFriendlyNumber(this.peakFootprint)); - } - console.log(" Wall time:", uiFriendlyDuration(this.endTime - this.startTime)); - } }; function dotnetPreloads(type) diff --git a/tests/unit-tests.js b/tests/unit-tests.js index 7260a3f6..f8efd23a 100644 --- a/tests/unit-tests.js +++ b/tests/unit-tests.js @@ -64,3 +64,30 @@ function assertEquals(actual, expected, message) { assertTrue(enabledBenchmarkNames.has(benchmark.name)); } })(); + + +(function testBenchmarkSubScores() { + for (const benchmark of BENCHMARKS) { + const subScores = benchmark.subScores(); + assertTrue(subScores instanceof Object); + assertTrue(Object.keys(subScores).length > 0); + for (const [name, value] of Object.entries(subScores)) { + assertTrue(typeof(name) == "string"); + // "Score" can only be part of allScores(). + assertFalse(name == "Score"); + // Without running all values should be null. + assertEquals(value, null); + } + } +})(); + +(function testBenchmarkAllScores() { + for (const benchmark of BENCHMARKS) { + const subScores = benchmark.subScores(); + const allScores = benchmark.allScores(); + assertTrue("Score" in allScores); + // All subScore items are part of allScores. + for (const name of Object.keys(subScores)) + assertTrue(name in allScores); + } +})();