Add GitHub Actions workflow for benchmark load tests with baseline comparison

.github/workflows/benchmark.yml

@@ -0,0 +1,217 @@
+# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: Benchmark Load Test
+
+on:
+  pull_request:
+    paths:
+      - 'websocket_server/**'
+      - 'tools/benchmarks/**'
+      - '.github/workflows/benchmark.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    name: Run Load Test
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Read package.json node and npm engines version
+        uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3
+        id: versions
+        with:
+          fallbackNode: '^20'
+          fallbackNpm: '^10'
+
+      - name: Set up node ${{ steps.versions.outputs.nodeVersion }}
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
+        with:
+          node-version: ${{ steps.versions.outputs.nodeVersion }}
+
+      - name: Set up npm ${{ steps.versions.outputs.npmVersion }}
+        run: npm i -g 'npm@${{ steps.versions.outputs.npmVersion }}'
+
+      - name: Install dependencies
+        env:
+          CYPRESS_INSTALL_BINARY: 0
+          PUPPETEER_SKIP_DOWNLOAD: true
+        run: npm ci
+
+      - name: Run benchmark load test
+        id: benchmark
+        run: |
+          # Run benchmark with smaller load for CI
+          export TLS=false
+          export JWT_SECRET_KEY=benchmark-secret
+          export NEXTCLOUD_URL=http://localhost
+          export LOAD_TEST_CONCURRENCY=50,100
+          export LOAD_TEST_DURATION=45
+          export NODE_OPTIONS=--max-old-space-size=8192
+
+          # Run the benchmark and capture output
+          node tools/benchmarks/runBenchmarks.mjs > benchmark_output.txt 2>&1
+
+          # Extract the JSON results from the output
+          # The script outputs JSON after the "=== Benchmark Summary ===" line
+          sed -n '/=== Benchmark Summary ===/,$ p' benchmark_output.txt | tail -n +2 > results.json
+
+          # Validate JSON format using Node.js
+          if ! node -e "JSON.parse(require('fs').readFileSync('results.json', 'utf8'))" 2>/dev/null; then
+            echo "Error: Failed to extract valid JSON from benchmark output"
+            cat benchmark_output.txt
+            exit 1
+          fi
+
+          # Display the full output
+          cat benchmark_output.txt
+
+      - name: Display benchmark results
+        run: |
+          echo "## Benchmark Results"
+          node -pe "JSON.stringify(JSON.parse(require('fs').readFileSync('results.json', 'utf8')), null, 2)"
+
+      - name: Comment PR with results
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const fs = require('fs');
+
+            // Read and parse current results with error handling
+            let results;
+            try {
+              const data = fs.readFileSync('results.json', 'utf8');
+              results = JSON.parse(data);
+            } catch (error) {
+              console.error('Failed to read or parse results.json:', error);
+              await github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: '## ❌ Benchmark Load Test Failed\n\nFailed to parse benchmark results. Please check the workflow logs for details.'
+              });
+              throw error;
+            }
+
+            // Read baseline results
+            let baseline = null;
+            try {
+              const baselineData = fs.readFileSync('tools/benchmarks/results.json', 'utf8');
+              const baselineJson = JSON.parse(baselineData);
+              if (baselineJson.scenarios) {
+                baseline = {};
+                for (const scenario of baselineJson.scenarios) {
+                  baseline[scenario.concurrency] = scenario;
+                }
+              }
+            } catch (error) {
+              console.log('No baseline found or error reading baseline:', error.message);
+            }
+
+            // Helper function to format numbers with thousand separators
+            const formatNumber = (num) => {
+              return Math.round(num).toLocaleString('en-US');
+            };
+
+            // Helper function to format diff
+            const formatDiff = (current, base, decimals = 2) => {
+              if (base === undefined || base === null) return current.toFixed(decimals);
+              const diff = current - base;
+              const sign = diff > 0 ? '+' : '';
+              const color = diff > 0 ? '🔴' : (diff < 0 ? '🟢' : '⚪');
+              return `${current.toFixed(decimals)} (${color}${sign}${diff.toFixed(decimals)})`;
+            };
+
+            const formatDiffInt = (current, base) => {
+              if (base === undefined || base === null) return formatNumber(current);
+              const diff = current - base;
+              const sign = diff > 0 ? '+' : '';
+              const color = diff > 0 ? '🔴' : (diff < 0 ? '🟢' : '⚪');
+              return `${formatNumber(current)} (${color}${sign}${formatNumber(diff)})`;
+            };
+
+            const formatPercent = (current, base, decimals = 2) => {
+              if (base === undefined || base === null) return `${current.toFixed(decimals)}%`;
+              const diff = current - base;
+              const sign = diff > 0 ? '+' : '';
+              const color = diff > 0 ? '🔴' : (diff < 0 ? '🟢' : '⚪');
+              return `${current.toFixed(decimals)}% (${color}${sign}${diff.toFixed(decimals)}%)`;
+            };
+
+            // Format the results as a markdown table
+            let comment = '## 🚀 Benchmark Load Test Results\n\n';
+            if (baseline) {
+              comment += 'Comparison with baseline from `tools/benchmarks/results.json`.\n';
+              comment += 'Legend: 🟢 improvement, 🔴 regression, ⚪ no change\n\n';
+            } else {
+              comment += 'These results show the performance characteristics of the websocket server under load.\n\n';
+            }
+
+            // Create a table for each concurrency level
+            comment += '| Concurrent Users | Avg CPU | Peak CPU | Avg Memory (MB) | Peak Memory (MB) | Bytes Sent | Bytes Received | Messages Sent | Messages Received | Dropped Connections |\n';
+            comment += '|------------------|---------|----------|-----------------|------------------|------------|----------------|---------------|-------------------|--------------------|\n';
+
+            for (const result of results) {
+              const { concurrency, cpu, memory, loadSummary } = result;
+              const base = baseline ? baseline[concurrency] : null;
+
+              const avgCpu = base ? formatPercent(cpu.average, base.cpu.average) : `${cpu.average.toFixed(2)}%`;
+              const peakCpu = base ? formatPercent(cpu.peak, base.cpu.peak) : `${cpu.peak.toFixed(2)}%`;
+              const avgMem = base ? formatDiff(memory.averageRssMb, base.memory.averageRssMb) : memory.averageRssMb.toFixed(2);
+              const peakMem = base ? formatDiff(memory.peakRssMb, base.memory.peakRssMb) : memory.peakRssMb.toFixed(2);
+
+              // Network/message metrics - use formatDiffInt for large numbers with thousand separators
+              // Note: baseline stores network bytes from server perspective (ingress=from clients, egress=to clients)
+              // Current results are from client perspective (sent=to server, received=from server)
+              // So: bytesSent maps to ingressBytes, bytesReceived maps to egressBytes
+              const bytesSent = base ? formatDiffInt(loadSummary.bytesSent, base.network?.ingressBytes) : formatNumber(loadSummary.bytesSent);
+              const bytesReceived = base ? formatDiffInt(loadSummary.bytesReceived, base.network?.egressBytes) : formatNumber(loadSummary.bytesReceived);
+              // Messages are new metrics, show without diff if baseline doesn't have them
+              const messagesSent = formatNumber(loadSummary.messagesSent);
+              const messagesReceived = formatNumber(loadSummary.messagesReceived);
+
+              // Note: baseline has droppedConnections at top level, not in loadSummary
+              const baseDropped = base ? base.droppedConnections : null;
+              const dropped = base ? formatDiff(loadSummary.droppedConnections, baseDropped, 0) : loadSummary.droppedConnections;
+
+              comment += `| ${concurrency} | ${avgCpu} | ${peakCpu} | ${avgMem} | ${peakMem} | ${bytesSent} | ${bytesReceived} | ${messagesSent} | ${messagesReceived} | ${dropped} |\n`;
+            }
+
+            comment += '\n### Details\n\n';
+            for (const result of results) {
+              const { concurrency } = result;
+              comment += `<details>\n<summary>📊 ${concurrency} concurrent users</summary>\n\n`;
+              comment += '```json\n';
+              comment += JSON.stringify(result, null, 2);
+              comment += '\n```\n\n';
+              comment += '</details>\n\n';
+            }
+
+            comment += '\n---\n';
+            comment += '*Note: These benchmarks run with reduced load (50, 100 users) for CI efficiency. ';
+            comment += `For full benchmarks, see the [README](https://github.com/${context.repo.owner}/${context.repo.repo}#benchmarking--capacity-planning).*`;
+
+            // Post the comment
+            await github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: comment
+            });
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        if: always()
+        with:
+          name: benchmark-results
+          path: |
+            results.json
+            benchmark_output.txt