Add repeats aggregation to stabilize latency metrics

example/pgcompare.yaml

@@ -11,6 +11,7 @@ benchmark:
   warmup_iterations: 5
   iterations: 1000
   concurrency: 1
+  repeats: 3
 
 report:
   description:

internal/pgcompare/bench.go

@@ -174,6 +174,86 @@ func (b *benchmark) ValidateMatchingQueryNames(beforeQueries, afterQueries []Que
 	return nil
 }
 
+func (b *benchmark) RunRepeats(ctx context.Context, queries []Query, repeats, iterations, concurrency, warmupIterations uint) ([]Stats, error) {
+	if repeats == 0 {
+		repeats = 1
+	}
+	samples := make([][]Stats, repeats)
+	for r := uint(0); r < repeats; r++ {
+		b.log.Info("Running repeat", "repeat", r+1, "total", repeats)
+		s, err := b.Run(ctx, queries, iterations, concurrency, warmupIterations)
+		if err != nil {
+			return nil, fmt.Errorf("repeat %d: %w", r+1, err)
+		}
+		samples[r] = s
+	}
+	return aggregateRepeatStats(queries, samples), nil
+}
+
+func aggregateRepeatStats(queries []Query, samples [][]Stats) []Stats {
+	out := make([]Stats, len(queries))
+	for qi, q := range queries {
+		durs := map[string][]time.Duration{
+			"Min": nil, "Max": nil, "P50": nil, "P95": nil, "P99": nil, "Mean": nil, "StdDev": nil,
+		}
+		var qps, errRate []float64
+		var errs []string
+		for _, rep := range samples {
+			s := rep[qi]
+			durs["Min"] = append(durs["Min"], s.Min)
+			durs["Max"] = append(durs["Max"], s.Max)
+			durs["P50"] = append(durs["P50"], s.P50)
+			durs["P95"] = append(durs["P95"], s.P95)
+			durs["P99"] = append(durs["P99"], s.P99)
+			durs["Mean"] = append(durs["Mean"], s.Mean)
+			durs["StdDev"] = append(durs["StdDev"], s.StdDev)
+			qps = append(qps, s.QPS)
+			errRate = append(errRate, s.ErrorRate)
+			errs = append(errs, s.Errors...)
+		}
+		out[qi] = Stats{
+			QueryName: q.Name,
+			Min:       medianDuration(durs["Min"]),
+			Max:       medianDuration(durs["Max"]),
+			P50:       medianDuration(durs["P50"]),
+			P95:       medianDuration(durs["P95"]),
+			P99:       medianDuration(durs["P99"]),
+			Mean:      medianDuration(durs["Mean"]),
+			StdDev:    medianDuration(durs["StdDev"]),
+			QPS:       medianFloat64(qps),
+			ErrorRate: medianFloat64(errRate),
+			Errors:    errs,
+		}
+	}
+	return out
+}
+
+func medianDuration(values []time.Duration) time.Duration {
+	if len(values) == 0 {
+		return 0
+	}
+	cp := append([]time.Duration(nil), values...)
+	sort.Slice(cp, func(i, j int) bool { return cp[i] < cp[j] })
+	n := len(cp)
+	if n%2 == 1 {
+		return cp[n/2]
+	}
+	return (cp[n/2-1] + cp[n/2]) / 2
+}
+
+func medianFloat64(values []float64) float64 {
+	if len(values) == 0 {
+		return 0
+	}
+	cp := append([]float64(nil), values...)
+	sort.Float64s(cp)
+	n := len(cp)
+	if n%2 == 1 {
+		return cp[n/2]
+	}
+	return (cp[n/2-1] + cp[n/2]) / 2
+}
+
 func (b *benchmark) Run(ctx context.Context, queries []Query, iterations, concurrency, warmupIterations uint) ([]Stats, error) {
 	b.log.Info("Running queries", "queries", queries, "iterations", iterations, "warmup_iterations", warmupIterations)
 

internal/pgcompare/config.go

@@ -36,6 +36,7 @@ type Config struct {
 		WarmupIterations int    `yaml:"warmup_iterations" default:"0"`
 		Iterations       int    `yaml:"iterations"`
 		Concurrency      int    `yaml:"concurrency"`
+		Repeats          int    `yaml:"repeats"`
 	} `yaml:"benchmark"`
 
 	Report struct {
@@ -65,6 +66,9 @@ func LoadConfig(configPath string) (*Config, error) {
 
 	cfg.ProjectDir = projectDir
 	cfg.Migration.EnvVar = normalizeMigrationEnvVar(cfg.Migration.EnvVar)
+	if cfg.Benchmark.Repeats == 0 {
+		cfg.Benchmark.Repeats = 1
+	}
 
 	if err := godotenv.Load(filepath.Join(projectDir, ".env")); err != nil {
 		return nil, fmt.Errorf("load .env: %w", err)
@@ -115,6 +119,9 @@ func (c *Config) validate() error {
 	if c.Benchmark.Concurrency <= 0 {
 		return fmt.Errorf("benchmark.concurrency must be positive")
 	}
+	if c.Benchmark.Repeats <= 0 {
+		return fmt.Errorf("benchmark.repeats must be positive")
+	}
 	return nil
 }
 

internal/pgcompare/config_test.go

@@ -236,6 +236,7 @@ func validConfigForTest() Config {
 	cfg.Benchmark.WarmupIterations = 0
 	cfg.Benchmark.Iterations = 10
 	cfg.Benchmark.Concurrency = 2
+	cfg.Benchmark.Repeats = 1
 	return cfg
 }
 

internal/pgcompare/model.go

@@ -42,6 +42,7 @@ type ReportData struct {
 	Iterations       int
 	WarmupIterations int
 	Concurrency      int
+	Repeats          int
 	Speedups         []float64
 	Before           *BenchResult
 	After            *BenchResult

internal/pgcompare/templates/report.html

@@ -134,6 +134,7 @@
             Iterations: {{ .Iterations }},
             WarmupIterations: {{ .WarmupIterations }},
             Concurrency: {{ .Concurrency }},
+            Repeats: {{ .Repeats }},
             Speedups: speedups,
             Description: description,
             Before: {Phase: "{{ .Before.Phase }}", Stats: beforeStats},
@@ -179,7 +180,8 @@
     var BENCH_PARAM_HINTS = {
         'iterations': 'Количество измеряемых выполнений каждого запроса в фазах before/after.',
         'warmup iterations': 'Количество предварительных прогонов перед замером каждого запроса.',
-        'concurrency': 'Количество конкурентных воркеров, выполняющих запросы во время измерений.'
+        'concurrency': 'Количество конкурентных воркеров, выполняющих запросы во время измерений.',
+        'repeats': 'Количество независимых повторов бенчмарка на фазу. Метрики агрегируются по медиане, что гасит случайные выбросы между прогонами.'
     };
 
     function hintIconFromText(text) {
@@ -214,6 +216,7 @@
                 + buildBenchParam('iterations', data.Iterations)
                 + buildBenchParam('warmup iterations', data.WarmupIterations)
                 + buildBenchParam('concurrency', data.Concurrency)
+                + (data.Repeats && data.Repeats > 1 ? buildBenchParam('repeats', data.Repeats) : '')
                 + '</div>';
         }
         return '<div class="header">'

main.go

@@ -108,9 +108,10 @@ func runBenchmark(_ *cobra.Command, _ []string) error {
 	}
 
 	fmt.Fprintln(os.Stderr, "Benchmarking 'before'...")
-	beforeStats, err := bench.Run(
+	beforeStats, err := bench.RunRepeats(
 		ctx,
 		beforeQueries,
+		uint(cfg.Benchmark.Repeats),
 		uint(cfg.Benchmark.Iterations),
 		uint(cfg.Benchmark.Concurrency),
 		uint(cfg.Benchmark.WarmupIterations),
@@ -133,9 +134,10 @@ func runBenchmark(_ *cobra.Command, _ []string) error {
 	}
 
 	fmt.Fprintln(os.Stderr, "Benchmarking 'after'...")
-	afterStats, err := bench.Run(
+	afterStats, err := bench.RunRepeats(
 		ctx,
 		afterQueries,
+		uint(cfg.Benchmark.Repeats),
 		uint(cfg.Benchmark.Iterations),
 		uint(cfg.Benchmark.Concurrency),
 		uint(cfg.Benchmark.WarmupIterations),
@@ -166,6 +168,7 @@ func runBenchmark(_ *cobra.Command, _ []string) error {
 		Iterations:       cfg.Benchmark.Iterations,
 		WarmupIterations: cfg.Benchmark.WarmupIterations,
 		Concurrency:      cfg.Benchmark.Concurrency,
+		Repeats:          cfg.Benchmark.Repeats,
 		Speedups:         speedups,
 		Before: &pgcompare.BenchResult{
 			Phase: "before",