From 8ab3973917dc331c9a36705dd6d88606721c45b0 Mon Sep 17 00:00:00 2001 From: AviralAv03 Date: Sat, 11 Apr 2026 21:11:19 +0530 Subject: [PATCH] fix: use multiple seeds per task in regression baselines --- chuck/regression.py | 24 ++-- data/compute_core/regression.json | 6 +- data/data_encoding/regression.json | 8 +- data/graph_analytics/regression.json | 8 +- data/io_pipeline/regression.json | 10 +- data/memory_index/regression.json | 2 +- data/memory_tier/regression.json | 24 ++-- data/ordering_core/regression.json | 10 +- data/prime_analytics/regression.json | 8 +- data/regression.json | 180 +++++++++++++++++++++++-- data/relational_fusion/regression.json | 6 +- data/retrieval_core/regression.json | 14 +- 12 files changed, 233 insertions(+), 67 deletions(-) diff --git a/chuck/regression.py b/chuck/regression.py index a05a58a..9e909dd 100644 --- a/chuck/regression.py +++ b/chuck/regression.py @@ -19,16 +19,22 @@ def _baseline_path(task_name: str) -> Path: def _expected_cases() -> list[dict[str, Any]]: cases = [] + SEEDS_PER_TASK = 5 + for index, task in enumerate(TASKS, start=1): - payload = task.generator(task.regression_size, index) - cases.append( - { - "task": task.name, - "seed": index, - "size": task.regression_size, - "expected": solve_with_backend(task=task, payload=payload, backend="python"), - } - ) + for offset in range(SEEDS_PER_TASK): + + seed = (index * 10) + offset + + payload = task.generator(task.regression_size, seed) + cases.append( + { + "task": task.name, + "seed": seed, + "size": task.regression_size, + "expected": solve_with_backend(task=task, payload=payload, backend="python"), + } + ) return cases diff --git a/data/compute_core/regression.json b/data/compute_core/regression.json index 7a20ba9..ebe4ddd 100644 --- a/data/compute_core/regression.json +++ b/data/compute_core/regression.json @@ -2,11 +2,11 @@ { "expected": { "backend": "python", - "checksum": 76643, + "checksum": 85094, "size": 16, - "trace": 4787 + "trace": 5409 }, - "seed": 9, + "seed": 94, "size": 16, "task": "compute_core" } diff --git a/data/data_encoding/regression.json b/data/data_encoding/regression.json index 4973863..1f8b8e3 100644 --- a/data/data_encoding/regression.json +++ b/data/data_encoding/regression.json @@ -2,13 +2,13 @@ { "expected": { "backend": "python", - "compressed_bytes": 5388, + "compressed_bytes": 5439, "input_bytes": 8192, - "ratio": 0.6577, + "ratio": 0.6639, "roundtrip": true, - "sha256": "9937957b8c18ae7965f7ecf6fd355064db5a8d6ea251719f602f49d2817f0631" + "sha256": "fd1c38e6389403240f49d439a8cb65c9c90dfee38e5a50f027263984e8b00cdd" }, - "seed": 4, + "seed": 44, "size": 8192, "task": "data_encoding" } diff --git a/data/graph_analytics/regression.json b/data/graph_analytics/regression.json index ffe3bdf..3a82779 100644 --- a/data/graph_analytics/regression.json +++ b/data/graph_analytics/regression.json @@ -2,12 +2,12 @@ { "expected": { "backend": "python", - "checksum": 23.048914, + "checksum": 24.231153, "node_count": 48, - "top_node": "n0010", - "top_score": 0.051786 + "top_node": "n0028", + "top_score": 0.063059 }, - "seed": 5, + "seed": 54, "size": 48, "task": "graph_analytics" } diff --git a/data/io_pipeline/regression.json b/data/io_pipeline/regression.json index 4efc0ff..9a28c98 100644 --- a/data/io_pipeline/regression.json +++ b/data/io_pipeline/regression.json @@ -3,12 +3,12 @@ "expected": { "backend": "python", "records": 64, - "top_pair": "acct_003|bucket_03", - "top_value": 3220, - "total_value": 33802, - "unique_pairs": 33 + "top_pair": "acct_005|bucket_02", + "top_value": 2227, + "total_value": 33077, + "unique_pairs": 30 }, - "seed": 1, + "seed": 14, "size": 64, "task": "io_pipeline" } diff --git a/data/memory_index/regression.json b/data/memory_index/regression.json index 8d0929f..e05c77d 100644 --- a/data/memory_index/regression.json +++ b/data/memory_index/regression.json @@ -11,7 +11,7 @@ "probes": 192, "true_positives": 64 }, - "seed": 8, + "seed": 84, "size": 128, "task": "memory_index" } diff --git a/data/memory_tier/regression.json b/data/memory_tier/regression.json index 0a7a92f..fcf61dd 100644 --- a/data/memory_tier/regression.json +++ b/data/memory_tier/regression.json @@ -3,25 +3,25 @@ "expected": { "backend": "python", "final_keys": [ + 5, + 25, + 6, + 2, 12, - 7, - 31, 3, - 10, - 13, - 18, - 1, + 26, + 9, + 31, 8, - 15, 11, - 25 + 28 ], - "hit_rate": 0.3672, - "hits": 47, - "misses": 81, + "hit_rate": 0.4453, + "hits": 57, + "misses": 71, "requests": 128 }, - "seed": 7, + "seed": 74, "size": 128, "task": "memory_tier" } diff --git a/data/ordering_core/regression.json b/data/ordering_core/regression.json index 81d329b..09ecdb4 100644 --- a/data/ordering_core/regression.json +++ b/data/ordering_core/regression.json @@ -2,13 +2,13 @@ { "expected": { "backend": "python", - "checksum": 795494423, + "checksum": 585457486, "count": 128, - "max": 991026, - "median": 102583, - "min": -949966 + "max": 998651, + "median": 30351, + "min": -973342 }, - "seed": 2, + "seed": 24, "size": 128, "task": "ordering_core" } diff --git a/data/prime_analytics/regression.json b/data/prime_analytics/regression.json index d5d0a1b..dc703eb 100644 --- a/data/prime_analytics/regression.json +++ b/data/prime_analytics/regression.json @@ -3,13 +3,13 @@ "expected": { "backend": "python", "candidates": 128, - "checksum": 28615758, + "checksum": 30628377, "confidence": 0.9961, - "prime_density_estimate": 0.125, + "prime_density_estimate": 0.1016, "probabilistic": true, - "probable_primes": 16 + "probable_primes": 13 }, - "seed": 6, + "seed": 64, "size": 128, "task": "prime_analytics" } diff --git a/data/regression.json b/data/regression.json index 1624f23..0a74575 100644 --- a/data/regression.json +++ b/data/regression.json @@ -1,42 +1,202 @@ [ { - "path": "data/io_pipeline/regression.json", + "path": "data\\io_pipeline\\regression.json", "task": "io_pipeline" }, { - "path": "data/ordering_core/regression.json", + "path": "data\\io_pipeline\\regression.json", + "task": "io_pipeline" + }, + { + "path": "data\\io_pipeline\\regression.json", + "task": "io_pipeline" + }, + { + "path": "data\\io_pipeline\\regression.json", + "task": "io_pipeline" + }, + { + "path": "data\\io_pipeline\\regression.json", + "task": "io_pipeline" + }, + { + "path": "data\\ordering_core\\regression.json", + "task": "ordering_core" + }, + { + "path": "data\\ordering_core\\regression.json", "task": "ordering_core" }, { - "path": "data/retrieval_core/regression.json", + "path": "data\\ordering_core\\regression.json", + "task": "ordering_core" + }, + { + "path": "data\\ordering_core\\regression.json", + "task": "ordering_core" + }, + { + "path": "data\\ordering_core\\regression.json", + "task": "ordering_core" + }, + { + "path": "data\\retrieval_core\\regression.json", + "task": "retrieval_core" + }, + { + "path": "data\\retrieval_core\\regression.json", + "task": "retrieval_core" + }, + { + "path": "data\\retrieval_core\\regression.json", "task": "retrieval_core" }, { - "path": "data/data_encoding/regression.json", + "path": "data\\retrieval_core\\regression.json", + "task": "retrieval_core" + }, + { + "path": "data\\retrieval_core\\regression.json", + "task": "retrieval_core" + }, + { + "path": "data\\data_encoding\\regression.json", + "task": "data_encoding" + }, + { + "path": "data\\data_encoding\\regression.json", + "task": "data_encoding" + }, + { + "path": "data\\data_encoding\\regression.json", + "task": "data_encoding" + }, + { + "path": "data\\data_encoding\\regression.json", "task": "data_encoding" }, { - "path": "data/graph_analytics/regression.json", + "path": "data\\data_encoding\\regression.json", + "task": "data_encoding" + }, + { + "path": "data\\graph_analytics\\regression.json", + "task": "graph_analytics" + }, + { + "path": "data\\graph_analytics\\regression.json", + "task": "graph_analytics" + }, + { + "path": "data\\graph_analytics\\regression.json", + "task": "graph_analytics" + }, + { + "path": "data\\graph_analytics\\regression.json", + "task": "graph_analytics" + }, + { + "path": "data\\graph_analytics\\regression.json", "task": "graph_analytics" }, { - "path": "data/prime_analytics/regression.json", + "path": "data\\prime_analytics\\regression.json", "task": "prime_analytics" }, { - "path": "data/memory_tier/regression.json", + "path": "data\\prime_analytics\\regression.json", + "task": "prime_analytics" + }, + { + "path": "data\\prime_analytics\\regression.json", + "task": "prime_analytics" + }, + { + "path": "data\\prime_analytics\\regression.json", + "task": "prime_analytics" + }, + { + "path": "data\\prime_analytics\\regression.json", + "task": "prime_analytics" + }, + { + "path": "data\\memory_tier\\regression.json", + "task": "memory_tier" + }, + { + "path": "data\\memory_tier\\regression.json", "task": "memory_tier" }, { - "path": "data/memory_index/regression.json", + "path": "data\\memory_tier\\regression.json", + "task": "memory_tier" + }, + { + "path": "data\\memory_tier\\regression.json", + "task": "memory_tier" + }, + { + "path": "data\\memory_tier\\regression.json", + "task": "memory_tier" + }, + { + "path": "data\\memory_index\\regression.json", + "task": "memory_index" + }, + { + "path": "data\\memory_index\\regression.json", + "task": "memory_index" + }, + { + "path": "data\\memory_index\\regression.json", "task": "memory_index" }, { - "path": "data/compute_core/regression.json", + "path": "data\\memory_index\\regression.json", + "task": "memory_index" + }, + { + "path": "data\\memory_index\\regression.json", + "task": "memory_index" + }, + { + "path": "data\\compute_core\\regression.json", + "task": "compute_core" + }, + { + "path": "data\\compute_core\\regression.json", + "task": "compute_core" + }, + { + "path": "data\\compute_core\\regression.json", + "task": "compute_core" + }, + { + "path": "data\\compute_core\\regression.json", "task": "compute_core" }, { - "path": "data/relational_fusion/regression.json", + "path": "data\\compute_core\\regression.json", + "task": "compute_core" + }, + { + "path": "data\\relational_fusion\\regression.json", + "task": "relational_fusion" + }, + { + "path": "data\\relational_fusion\\regression.json", + "task": "relational_fusion" + }, + { + "path": "data\\relational_fusion\\regression.json", + "task": "relational_fusion" + }, + { + "path": "data\\relational_fusion\\regression.json", + "task": "relational_fusion" + }, + { + "path": "data\\relational_fusion\\regression.json", "task": "relational_fusion" } ] diff --git a/data/relational_fusion/regression.json b/data/relational_fusion/regression.json index 96624ef..83726a7 100644 --- a/data/relational_fusion/regression.json +++ b/data/relational_fusion/regression.json @@ -1,13 +1,13 @@ [ { "expected": { - "aggregate": 276559, + "aggregate": 261106, "backend": "python", - "join_rows": 269, + "join_rows": 253, "left_rows": 128, "right_rows": 64 }, - "seed": 10, + "seed": 104, "size": 128, "task": "relational_fusion" } diff --git a/data/retrieval_core/regression.json b/data/retrieval_core/regression.json index 352b8f6..39fcc00 100644 --- a/data/retrieval_core/regression.json +++ b/data/retrieval_core/regression.json @@ -6,17 +6,17 @@ "doc_count": 48, "probabilistic": true, "query_hits": { - "alpha": 26, - "delta": 34, - "eta": 34, - "kappa": 40 + "beta": 28, + "epsilon": 28, + "lambda": 40, + "theta": 46 }, "sampled_docs": 24, - "top_term": "kappa", - "top_term_docs": 40, + "top_term": "theta", + "top_term_docs": 46, "vocab_size": 12 }, - "seed": 3, + "seed": 34, "size": 48, "task": "retrieval_core" }