From de5c4384007284bb9f1300591c097f2bf0cae0eb Mon Sep 17 00:00:00 2001 From: Event Horizon <772552754@qq.com> Date: Thu, 15 Jan 2026 20:42:51 +0800 Subject: [PATCH 1/4] optimize matrix enumeration --- Mocha.sln | 7 + .../Prometheus/PromQL/Engine/Evaluator.cs | 21 +-- .../PromQL/Engine/MatrixEnumerator.cs | 75 +++++++++ .../MatrixEnumerationBenchmark.cs | 69 +++++++++ .../Mocha.Query.Benchmarks.csproj | 20 +++ tests/Mocha.Query.Benchmarks/Program.cs | 12 ++ .../MatrixEnumeratorTests.cs | 143 ++++++++++++++++++ 7 files changed, 337 insertions(+), 10 deletions(-) create mode 100644 src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs create mode 100644 tests/Mocha.Query.Benchmarks/MatrixEnumerationBenchmark.cs create mode 100644 tests/Mocha.Query.Benchmarks/Mocha.Query.Benchmarks.csproj create mode 100644 tests/Mocha.Query.Benchmarks/Program.cs create mode 100644 tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs diff --git a/Mocha.sln b/Mocha.sln index c3090b3..61c1159 100644 --- a/Mocha.sln +++ b/Mocha.sln @@ -64,6 +64,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Antlr4.Generated", "s EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Storage.Benchmarks", "tests\Mocha.Storage.Benchmarks\Mocha.Storage.Benchmarks.csproj", "{4B63E7B0-C8C8-4206-A113-A3D4EFB7533E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mocha.Query.Benchmarks", "tests\Mocha.Query.Benchmarks\Mocha.Query.Benchmarks.csproj", "{89EF2B9E-24CF-4A80-8199-284E34C11277}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -85,6 +87,7 @@ Global {D56CA47A-A948-4FB5-9E16-C61E12535521} = {24F9E34A-D92A-4C0A-851F-1E864181BF97} {904CC523-A2D4-4982-8A7B-A6A0F5A5EB19} = {6983D239-07DA-4DFA-9AAA-F6876029FF8D} {4B63E7B0-C8C8-4206-A113-A3D4EFB7533E} = {24F9E34A-D92A-4C0A-851F-1E864181BF97} + {89EF2B9E-24CF-4A80-8199-284E34C11277} = {24F9E34A-D92A-4C0A-851F-1E864181BF97} EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {DCA600F0-4D6C-44DA-A493-F63097CCE74E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU @@ -135,5 +138,9 @@ Global {4B63E7B0-C8C8-4206-A113-A3D4EFB7533E}.Debug|Any CPU.Build.0 = Debug|Any CPU {4B63E7B0-C8C8-4206-A113-A3D4EFB7533E}.Release|Any CPU.ActiveCfg = Release|Any CPU {4B63E7B0-C8C8-4206-A113-A3D4EFB7533E}.Release|Any CPU.Build.0 = Release|Any CPU + {89EF2B9E-24CF-4A80-8199-284E34C11277}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {89EF2B9E-24CF-4A80-8199-284E34C11277}.Debug|Any CPU.Build.0 = Debug|Any CPU + {89EF2B9E-24CF-4A80-8199-284E34C11277}.Release|Any CPU.ActiveCfg = Release|Any CPU + {89EF2B9E-24CF-4A80-8199-284E34C11277}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs b/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs index 70b350a..c17a6ab 100644 --- a/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs +++ b/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs @@ -11,7 +11,6 @@ namespace Mocha.Query.Prometheus.PromQL.Engine; -// TODO: Use ArrayPool internal class Evaluator { public long StartTimestampUnixSec { get; init; } @@ -132,11 +131,11 @@ public MatrixResult Eval(Expression expr) var matrixSelector = (MatrixSelector)call.Args[matrixArgIndex]; result = new MatrixResult(matrixSelector.Series.Count()); var selectorOffsetSeconds = (long)matrixSelector.Offset.TotalSeconds; - var selectorRangeSeconds = matrixSelector.Range.TotalSeconds; + var selectorRangeSeconds = (long)matrixSelector.Range.TotalSeconds; var stepRangeSeconds = (long)Math.Min(selectorRangeSeconds, Interval.TotalSeconds); // Reuse objects across steps to save memory allocations. - // TODO: use ArrayPool + var points = new List(); var inMatrix = new MatrixResult(1) { new Series { Metric = Labels.Empty, Points = [] } }; inArgs[matrixArgIndex] = inMatrix; var enh = new EvalNodeHelper { Output = new VectorResult(1) }; @@ -144,6 +143,7 @@ public MatrixResult Eval(Expression expr) // Process all the calls for one time series at a time. foreach (var timeSeries in matrixSelector.Series) { + points.Clear(); var series = new Series { Metric = timeSeries.Labels.DropMetricName(), @@ -155,6 +155,7 @@ public MatrixResult Eval(Expression expr) var step = -1; var refTimeStart = StartTimestampUnixSec - selectorOffsetSeconds; var refTimeEnd = EndTimestampUnixSec - selectorOffsetSeconds; + using var matrixEnumerator = new MatrixEnumerator(timeSeries.Samples); for (var ts = refTimeStart; ts <= refTimeEnd; ts += stepRangeSeconds) { step++; @@ -172,18 +173,19 @@ public MatrixResult Eval(Expression expr) var maxTs = ts; var minTs = maxTs - selectorRangeSeconds; // Evaluate the matrix selector for this series for this step. - // TODO: optimize enumeration - var points = timeSeries.Samples - .Where(s => s.TimestampUnixSec >= minTs && s.TimestampUnixSec <= maxTs) - .Select(s => - new DoublePoint { TimestampUnixSec = s.TimestampUnixSec, Value = s.Value }) - .ToList(); + points = matrixEnumerator.Enumerate(minTs, maxTs, points); if (points.Count <= 0) { continue; } + _currentSamples += points.Count; + if (_currentSamples > MaxSamples) + { + throw new TooManySamplesException(); + } + inMatrix[0].Points = points; enh.TimestampUnixSec = ts; enh.Output.Clear(); @@ -427,7 +429,6 @@ public MatrixResult Eval(Expression expr) } } - /// /// Evaluates the given expressions, and then for each step calls /// the given function with the values computed for each expression at that step. diff --git a/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs b/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs new file mode 100644 index 0000000..d837bd0 --- /dev/null +++ b/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Core Community under one or more agreements. +// The .NET Core Community licenses this file to you under the MIT license. + +using Mocha.Core.Storage.Prometheus.Metrics; +using Mocha.Query.Prometheus.PromQL.Values; + +namespace Mocha.Query.Prometheus.PromQL.Engine; + +public class MatrixEnumerator(IEnumerable samples) : IDisposable +{ + private readonly IEnumerator _enumerator = samples.GetEnumerator(); + + public List Enumerate( + long minTs, + long maxTs, + List reusedPoints) + { + var keepFrom = 0; + while (keepFrom < reusedPoints.Count && reusedPoints[keepFrom].TimestampUnixSec < minTs) + { + keepFrom++; + } + + // If there is an overlap between previous and current ranges, keep the overlapping part. + // If keepFrom is 0, all points are within the range, so keep them all. + if (keepFrom > 0) + { + reusedPoints.RemoveRange(0, keepFrom); + } + else if (keepFrom == reusedPoints.Count) + { + // No overlap, clear all points. + reusedPoints.Clear(); + } + + while (true) + { + // Current is uninitialized or has been fully consumed + if (_enumerator.Current == null) + { + if (!_enumerator.MoveNext()) + { + break; + } + } + + var sample = _enumerator.Current; + + // Future data, leave it for the next step + if (sample!.TimestampUnixSec > maxTs) + { + break; + } + + // If the sample is within the range, add it to the points + if (sample.TimestampUnixSec >= minTs) + { + reusedPoints.Add(new DoublePoint { TimestampUnixSec = sample.TimestampUnixSec, Value = sample.Value }); + } + + // Move to the next sample + if (!_enumerator.MoveNext()) + { + break; + } + } + + return reusedPoints; + } + + public void Dispose() + { + _enumerator.Dispose(); + } +} diff --git a/tests/Mocha.Query.Benchmarks/MatrixEnumerationBenchmark.cs b/tests/Mocha.Query.Benchmarks/MatrixEnumerationBenchmark.cs new file mode 100644 index 0000000..b26314f --- /dev/null +++ b/tests/Mocha.Query.Benchmarks/MatrixEnumerationBenchmark.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Core Community under one or more agreements. +// The .NET Core Community licenses this file to you under the MIT license. + +using BenchmarkDotNet.Attributes; +using Mocha.Core.Storage.Prometheus.Metrics; +using Mocha.Query.Prometheus.PromQL.Engine; +using Mocha.Query.Prometheus.PromQL.Values; + +[MemoryDiagnoser] +public class MatrixEnumerationBenchmark +{ + [Params(1_000, 10_000)] public int SampleCount { get; set; } + + [Params(15)] public int SampleIntervalSeconds { get; set; } + + [Params(30, 60)] public int SelectorRangeSeconds { get; set; } + + private List _samples = null!; + private long _startTs; + private long _endTs; + + [GlobalSetup] + public void Setup() + { + _samples = new List(SampleCount); + + long ts = 0; + for (int i = 0; i < SampleCount; i++) + { + _samples.Add(new TimeSeriesSample { TimestampUnixSec = ts, Value = i }); + + ts += SampleIntervalSeconds; + } + + _startTs = _samples[0].TimestampUnixSec; + _endTs = _samples[^1].TimestampUnixSec; + } + + [Benchmark(Baseline = true)] + public void LinqEveryStep() + { + for (var ts = _startTs; ts <= _endTs; ts += SelectorRangeSeconds) + { + var maxTs = ts; + var minTs = maxTs - SelectorRangeSeconds; + + var points = _samples + .Where(s => s.TimestampUnixSec >= minTs && + s.TimestampUnixSec <= maxTs) + .Select(s => new DoublePoint { TimestampUnixSec = s.TimestampUnixSec, Value = s.Value }) + .ToList(); + } + } + + [Benchmark] + public void EnumeratorSlidingWindow() + { + var reusePoints = new List(); + using var enumerator = new MatrixEnumerator(_samples); + + for (var ts = _startTs; ts <= _endTs; ts += SelectorRangeSeconds) + { + var maxTs = ts; + var minTs = maxTs - SelectorRangeSeconds; + + enumerator.Enumerate(minTs, maxTs, reusePoints); + } + } +} diff --git a/tests/Mocha.Query.Benchmarks/Mocha.Query.Benchmarks.csproj b/tests/Mocha.Query.Benchmarks/Mocha.Query.Benchmarks.csproj new file mode 100644 index 0000000..74add26 --- /dev/null +++ b/tests/Mocha.Query.Benchmarks/Mocha.Query.Benchmarks.csproj @@ -0,0 +1,20 @@ + + + + Exe + net8.0 + enable + enable + true + + + + + + + + + + + + diff --git a/tests/Mocha.Query.Benchmarks/Program.cs b/tests/Mocha.Query.Benchmarks/Program.cs new file mode 100644 index 0000000..e5c3418 --- /dev/null +++ b/tests/Mocha.Query.Benchmarks/Program.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Core Community under one or more agreements. +// The .NET Core Community licenses this file to you under the MIT license. + +using BenchmarkDotNet.Running; + +var allBenchmarks = new[] +{ + typeof(MatrixEnumerationBenchmark) +}; + +new BenchmarkSwitcher(allBenchmarks).Run(args); + diff --git a/tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs b/tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs new file mode 100644 index 0000000..ba737db --- /dev/null +++ b/tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Core Community under one or more agreements. +// The .NET Core Community licenses this file to you under the MIT license. + +namespace Mocha.Query.Tests; + +using System; +using System.Collections.Generic; +using Mocha.Core.Storage.Prometheus.Metrics; +using Mocha.Query.Prometheus.PromQL.Engine; +using Mocha.Query.Prometheus.PromQL.Values; +using Xunit; + +public class MatrixEnumeratorTests +{ + [Fact] + public void Enumerate_FirstCall_FillsWindow() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + var reused = new List(); + + var result = enumerator.Enumerate( + minTs: 0, + maxTs: 20, + reusedPoints: reused); + + Assert.Equal(new[] { 0L, 10L, 20L }, result.ConvertAll(p => p.TimestampUnixSec)); + } + + [Fact] + public void Enumerate_OverlappingWindow_ReusesPoints() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + var reused = new List(); + + // First window + enumerator.Enumerate(0, 20, reused); + + // Overlapping window + var result = enumerator.Enumerate(10, 30, reused); + + // keepFrom > 0 path + Assert.Equal(new[] { 10L, 20L, 30L }, result.ConvertAll(p => p.TimestampUnixSec)); + } + + [Fact] + public void Enumerate_NoOverlap_ClearsReusedPoints() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + var reused = new List(); + + // First window + enumerator.Enumerate(0, 10, reused); + + // Non-overlapping window + var result = enumerator.Enumerate(30, 40, reused); + + // keepFrom == Count → Clear() + Assert.Equal(new[] { 30L, 40L }, result.ConvertAll(p => p.TimestampUnixSec)); + } + + [Fact] + public void Enumerate_WindowWithNoSamples_ReturnsEmpty() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + var reused = new List(); + + var result = enumerator.Enumerate( + minTs: 100, + maxTs: 200, + reusedPoints: reused); + + Assert.Empty(result); + } + + [Fact] + public void Enumerate_EnumeratorExhausted_DoesNotThrow() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + var reused = new List(); + + // Consume everything + enumerator.Enumerate(0, 100, reused); + + // Enumerator is exhausted + var result = enumerator.Enumerate(0, 100, reused); + + Assert.NotNull(result); + } + + [Fact] + public void Enumerate_FutureSampleStopsConsumption() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + var reused = new List(); + + // maxTs stops before 30 + var result = enumerator.Enumerate(0, 15, reused); + + // Should not consume sample at ts=20 + Assert.Equal(new[] { 0L, 10L }, result.ConvertAll(p => p.TimestampUnixSec)); + + // Next window should still see 20 + var result2 = enumerator.Enumerate(0, 25, reused); + + Assert.Contains(result2, p => p.TimestampUnixSec == 20); + } + + [Fact] + public void Enumerate_Should_Not_Skip_Future_Sample() + { + var samples = new List + { + new() { TimestampUnixSec = 0, Value = 0 }, + new() { TimestampUnixSec = 10, Value = 1 }, + new() { TimestampUnixSec = 20, Value = 2 } + }; + + using var enumerator = new MatrixEnumerator(samples); + var reused = new List(); + + // First window stops early + var p1 = enumerator.Enumerate(minTs: 0, maxTs: 5, reused); + Assert.Single(p1); + Assert.Equal(0, p1[0].TimestampUnixSec); + + // Second window must still see ts=10 + var p2 = enumerator.Enumerate(minTs: 0, maxTs: 15, reused); + Assert.Contains(p2, p => p.TimestampUnixSec == 10); + } + + private static List CreateSamples() + { + // ts: 0, 10, 20, 30, 40 + return + [ + new TimeSeriesSample { TimestampUnixSec = 0, Value = 0 }, + new() { TimestampUnixSec = 10, Value = 1 }, + new() { TimestampUnixSec = 20, Value = 2 }, + new() { TimestampUnixSec = 30, Value = 3 }, + new() { TimestampUnixSec = 40, Value = 4 } + ]; + } +} From ea4e86f72e40fcea182fcc3dadd2b301b9a5ee3a Mon Sep 17 00:00:00 2001 From: Event Horizon <772552754@qq.com> Date: Mon, 19 Jan 2026 21:32:24 +0800 Subject: [PATCH 2/4] fix for loop step --- src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs b/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs index c17a6ab..410b388 100644 --- a/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs +++ b/src/Mocha.Query/Prometheus/PromQL/Engine/Evaluator.cs @@ -25,7 +25,8 @@ internal class Evaluator public MatrixResult Eval(Expression expr) { - var numSteps = (int)((EndTimestampUnixSec - StartTimestampUnixSec) / Interval.TotalSeconds) + 1; + var intervalSeconds = (long)Interval.TotalSeconds; + var numSteps = (int)((EndTimestampUnixSec - StartTimestampUnixSec) / intervalSeconds) + 1; MatrixResult result; switch (expr) { @@ -132,7 +133,6 @@ public MatrixResult Eval(Expression expr) result = new MatrixResult(matrixSelector.Series.Count()); var selectorOffsetSeconds = (long)matrixSelector.Offset.TotalSeconds; var selectorRangeSeconds = (long)matrixSelector.Range.TotalSeconds; - var stepRangeSeconds = (long)Math.Min(selectorRangeSeconds, Interval.TotalSeconds); // Reuse objects across steps to save memory allocations. var points = new List(); @@ -156,7 +156,7 @@ public MatrixResult Eval(Expression expr) var refTimeStart = StartTimestampUnixSec - selectorOffsetSeconds; var refTimeEnd = EndTimestampUnixSec - selectorOffsetSeconds; using var matrixEnumerator = new MatrixEnumerator(timeSeries.Samples); - for (var ts = refTimeStart; ts <= refTimeEnd; ts += stepRangeSeconds) + for (var ts = refTimeStart; ts <= refTimeEnd; ts += intervalSeconds) { step++; // Set the non-matrix arguments. @@ -352,7 +352,6 @@ public MatrixResult Eval(Expression expr) // TODO: use ArrayPool Points = new List(numSteps) }; - var intervalSeconds = (long)Interval.TotalSeconds; var refTimeStart = StartTimestampUnixSec - offsetSeconds; var refTimeEnd = EndTimestampUnixSec - offsetSeconds; using var enumerator = timeSeries.Samples.Reverse().GetEnumerator(); From 7a845da97f889fbe8acb407d409210ffa9fad8c4 Mon Sep 17 00:00:00 2001 From: Event Horizon <772552754@qq.com> Date: Sat, 31 Jan 2026 10:12:46 +0800 Subject: [PATCH 3/4] add test --- .../PromQL/Engine/MatrixEnumerator.cs | 7 +++++ .../MatrixEnumeratorTests.cs | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs b/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs index d837bd0..bb24485 100644 --- a/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs +++ b/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs @@ -15,6 +15,13 @@ public List Enumerate( long maxTs, List reusedPoints) { + ArgumentNullException.ThrowIfNull(reusedPoints); + + if (minTs > maxTs) + { + throw new ArgumentException("minTs must be less than or equal to maxTs"); + } + var keepFrom = 0; while (keepFrom < reusedPoints.Count && reusedPoints[keepFrom].TimestampUnixSec < minTs) { diff --git a/tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs b/tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs index ba737db..733f562 100644 --- a/tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs +++ b/tests/Mocha.Query.Tests/MatrixEnumeratorTests.cs @@ -128,6 +128,34 @@ public void Enumerate_Should_Not_Skip_Future_Sample() Assert.Contains(p2, p => p.TimestampUnixSec == 10); } + [Fact] + public void Enumerate_InvalidWindow_Throws() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + var reused = new List(); + + Assert.Throws(() => + { + enumerator.Enumerate( + minTs: 20, + maxTs: 10, + reusedPoints: reused); + }); + } + + [Fact] + public void Enumerate_NullReusedPoints_Throws() + { + using var enumerator = new MatrixEnumerator(CreateSamples()); + Assert.Throws(() => + { + enumerator.Enumerate( + minTs: 0, + maxTs: 10, + reusedPoints: null!); + }); + } + private static List CreateSamples() { // ts: 0, 10, 20, 30, 40 From 6a159be59c3e3f38fde38ad7ec835652b200b5d4 Mon Sep 17 00:00:00 2001 From: Event Horizon <772552754@qq.com> Date: Sat, 31 Jan 2026 10:19:19 +0800 Subject: [PATCH 4/4] no message --- src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs b/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs index bb24485..e68446c 100644 --- a/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs +++ b/src/Mocha.Query/Prometheus/PromQL/Engine/MatrixEnumerator.cs @@ -17,9 +17,9 @@ public List Enumerate( { ArgumentNullException.ThrowIfNull(reusedPoints); - if (minTs > maxTs) + if (minTs >= maxTs) { - throw new ArgumentException("minTs must be less than or equal to maxTs"); + throw new ArgumentException("minTs must be less than maxTs"); } var keepFrom = 0;