From 3424bb816b1a08e37b5d4ba4ff06f0c74d9b78b1 Mon Sep 17 00:00:00 2001 From: Keith Miller Date: Wed, 13 Aug 2025 18:09:55 -0400 Subject: [PATCH 1/4] Add a new GroupedBenchmark that runs a series of benchmarks. Use this for all the SunSpider tests. WIP Only note is that `subScores` re-computes everything every time because it's called before the benchmark runs. --- JetStreamDriver.js | 72 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index 387a4664..f74bb8b5 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -1121,6 +1121,69 @@ class Benchmark { } }; +class GroupedBenchmark extends Benchmark { + constructor(plan, benchmarks) { + super(plan); + assert(benchmarks.length); + for (const benchmark of benchmarks) { + // FIXME: Tags don't work for grouped tests anyway but if they did then this would be weird and probably wrong. + assert(benchmark.plan.tags.indexOf("Default") === -1, `Grouped benchmark sub-benchmarks shouldn't have the "Default" tag`, benchmark.tags); + } + benchmarks.sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? 1 : -1); + this.benchmarks = benchmarks; + } + + async prefetchResourcesForBrowser() { + for (const benchmark of this.benchmarks) + await benchmark.prefetchResourcesForBrowser(); + } + + async retryPrefetchResourcesForBrowser() { + for (const benchmark of this.benchmarks) + await benchmark.retryPrefetchResourcesForBrowser(); + } + + prefetchResourcesForShell() { + for (const benchmark of this.benchmarks) + benchmark.prefetchResourcesForShell(); + } + + async run() { + performance.mark(this.name); + this.startTime = performance.now(); + + for (const benchmark of this.benchmarks) + await benchmark.run(); + + this.endTime = performance.now(); + performance.measure(this.name, this.name); + + this.processResults(); + } + + processResults() { + this.results = []; + for (const benchmark of this.benchmarks) + this.results = this.results.concat(benchmark.results); + } + + subScores() { + const results = {}; + + for (const benchmark of this.benchmarks) { + let scores = benchmark.subScores(); + for (let subScore in scores) { + results[subScore] ??= []; + results[subScore].push(scores[subScore]); + } + } + + for (let subScore in results) + results[subScore] = geomeanScore(results[subScore]); + return results; + } +}; + class DefaultBenchmark extends Benchmark { constructor(...args) { super(...args); @@ -2426,15 +2489,20 @@ const SUNSPIDER_TESTS = [ "string-unpack-code", "tagcloud", ]; +let SUNSPIDER_BENCHMARKS = []; for (const test of SUNSPIDER_TESTS) { - BENCHMARKS.push(new DefaultBenchmark({ + SUNSPIDER_BENCHMARKS.push(new DefaultBenchmark({ name: `${test}-SP`, files: [ `./SunSpider/${test}.js` ], - tags: ["Default", "SunSpider"], + tags: [], })); } +BENCHMARKS.push(new GroupedBenchmark({ + name: "Sunspider", + tags: ["Default", "SunSpider"], +}, SUNSPIDER_BENCHMARKS)) // WTB (Web Tooling Benchmark) tests const WTB_TESTS = [ From a6df0248dbe54ea3c7862c03683238dd0b56f804 Mon Sep 17 00:00:00 2001 From: Keith Miller Date: Thu, 14 Aug 2025 09:05:21 -0400 Subject: [PATCH 2/4] Fix unit tests --- tests/unit-tests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit-tests.js b/tests/unit-tests.js index c99e53e6..9c476841 100644 --- a/tests/unit-tests.js +++ b/tests/unit-tests.js @@ -86,8 +86,8 @@ function assertEquals(actual, expected, message) { assertTrue(typeof(name) == "string"); // "Score" can only be part of allScores(). assertFalse(name == "Score"); - // Without running all values should be null. - assertEquals(value, null); + // Without running values should be either null (or 0 for GroupedBenchmark) + assertFalse(value) } } })(); From 7b548914a72e9219e1727ddc18b30f0037a63e76 Mon Sep 17 00:00:00 2001 From: Keith Miller Date: Thu, 14 Aug 2025 10:23:09 -0400 Subject: [PATCH 3/4] rebase and fix assert to console.assert as well as hasAnyTag --- JetStreamDriver.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index f74bb8b5..c3a58a3c 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -1124,10 +1124,10 @@ class Benchmark { class GroupedBenchmark extends Benchmark { constructor(plan, benchmarks) { super(plan); - assert(benchmarks.length); + console.assert(benchmarks.length); for (const benchmark of benchmarks) { // FIXME: Tags don't work for grouped tests anyway but if they did then this would be weird and probably wrong. - assert(benchmark.plan.tags.indexOf("Default") === -1, `Grouped benchmark sub-benchmarks shouldn't have the "Default" tag`, benchmark.tags); + console.assert(!benchmark.hasAnyTag("Default"), `Grouped benchmark sub-benchmarks shouldn't have the "Default" tag`, benchmark.tags); } benchmarks.sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? 1 : -1); this.benchmarks = benchmarks; From dbbea1aee2b22a2e74b3f65fec474c2053264f5b Mon Sep 17 00:00:00 2001 From: Keith Miller Date: Thu, 14 Aug 2025 10:45:38 -0400 Subject: [PATCH 4/4] Fix prefetchResources ref counting for GroupedBenchmark --- JetStreamDriver.js | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index c3a58a3c..b25353c3 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -262,7 +262,7 @@ class Driver { if (isInBrowser && globalThis.prefetchResources) { const cache = JetStream.blobDataCache; - for (const file of benchmark.plan.files) { + for (const file of benchmark.files) { const blobData = cache[file]; blobData.refCount--; if (!blobData.refCount) @@ -709,6 +709,7 @@ class Benchmark { } get name() { return this.plan.name; } + get files() { return this.plan.files; } get isDone() { return this._state == BenchmarkState.DONE || this._state == BenchmarkState.ERROR; @@ -1148,17 +1149,37 @@ class GroupedBenchmark extends Benchmark { benchmark.prefetchResourcesForShell(); } + get files() { + let files = []; + for (const benchmark of this.benchmarks) + files = files.concat(benchmark.files); + return files; + } + async run() { + this._state = BenchmarkState.PREPARE; performance.mark(this.name); this.startTime = performance.now(); - for (const benchmark of this.benchmarks) - await benchmark.run(); + let benchmark; + try { + this._state = BenchmarkState.RUNNING; + for (benchmark of this.benchmarks) + await benchmark.run(); + } catch (e) { + this._state = BenchmarkState.ERROR; + console.log(`Error in runCode of grouped benchmark ${benchmark.name}: `, e); + console.log(e.stack); + throw e; + } finally { + this._state = BenchmarkState.FINALIZE; + } this.endTime = performance.now(); performance.measure(this.name, this.name); this.processResults(); + this._state = BenchmarkState.DONE; } processResults() {