diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..8baec310 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +name: Benchmark Load Test + +on: + pull_request: + paths: + - 'websocket_server/**' + - 'tools/benchmarks/**' + - '.github/workflows/benchmark.yml' + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +jobs: + benchmark: + runs-on: ubuntu-latest + name: Run Load Test + + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Read package.json node and npm engines version + uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3 + id: versions + with: + fallbackNode: '^20' + fallbackNpm: '^10' + + - name: Set up node ${{ steps.versions.outputs.nodeVersion }} + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 + with: + node-version: ${{ steps.versions.outputs.nodeVersion }} + + - name: Set up npm ${{ steps.versions.outputs.npmVersion }} + run: npm i -g 'npm@${{ steps.versions.outputs.npmVersion }}' + + - name: Install dependencies + env: + CYPRESS_INSTALL_BINARY: 0 + PUPPETEER_SKIP_DOWNLOAD: true + run: npm ci + + - name: Run benchmark load test + id: benchmark + run: | + # Run benchmark with smaller load for CI + export TLS=false + export JWT_SECRET_KEY=benchmark-secret + export NEXTCLOUD_URL=http://localhost + export LOAD_TEST_CONCURRENCY=50,100 + export LOAD_TEST_DURATION=45 + export NODE_OPTIONS=--max-old-space-size=8192 + + # Run the benchmark and capture output + node tools/benchmarks/runBenchmarks.mjs > benchmark_output.txt 2>&1 + + # Extract the JSON results from the output + # The script outputs JSON after the "=== Benchmark Summary ===" line + sed -n '/=== Benchmark Summary ===/,$ p' benchmark_output.txt | tail -n +2 > results.json + + # Validate JSON format using Node.js + if ! node -e "JSON.parse(require('fs').readFileSync('results.json', 'utf8'))" 2>/dev/null; then + echo "Error: Failed to extract valid JSON from benchmark output" + cat benchmark_output.txt + exit 1 + fi + + # Display the full output + cat benchmark_output.txt + + - name: Display benchmark results + run: | + echo "## Benchmark Results" + node -pe "JSON.stringify(JSON.parse(require('fs').readFileSync('results.json', 'utf8')), null, 2)" + + - name: Comment PR with results + if: github.event_name == 'pull_request' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const fs = require('fs'); + + // Read and parse current results with error handling + let results; + try { + const data = fs.readFileSync('results.json', 'utf8'); + results = JSON.parse(data); + } catch (error) { + console.error('Failed to read or parse results.json:', error); + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '## ❌ Benchmark Load Test Failed\n\nFailed to parse benchmark results. Please check the workflow logs for details.' + }); + throw error; + } + + // Read baseline results + let baseline = null; + try { + const baselineData = fs.readFileSync('tools/benchmarks/results.json', 'utf8'); + const baselineJson = JSON.parse(baselineData); + if (baselineJson.scenarios) { + baseline = {}; + for (const scenario of baselineJson.scenarios) { + baseline[scenario.concurrency] = scenario; + } + } + } catch (error) { + console.log('No baseline found or error reading baseline:', error.message); + } + + // Helper function to format numbers with thousand separators + const formatNumber = (num) => { + return Math.round(num).toLocaleString('en-US'); + }; + + // Helper function to format diff + const formatDiff = (current, base, decimals = 2) => { + if (base === undefined || base === null) return current.toFixed(decimals); + const diff = current - base; + const sign = diff > 0 ? '+' : ''; + const color = diff > 0 ? '🔴' : (diff < 0 ? '🟢' : '⚪'); + return `${current.toFixed(decimals)} (${color}${sign}${diff.toFixed(decimals)})`; + }; + + const formatDiffInt = (current, base) => { + if (base === undefined || base === null) return formatNumber(current); + const diff = current - base; + const sign = diff > 0 ? '+' : ''; + const color = diff > 0 ? '🔴' : (diff < 0 ? '🟢' : '⚪'); + return `${formatNumber(current)} (${color}${sign}${formatNumber(diff)})`; + }; + + const formatPercent = (current, base, decimals = 2) => { + if (base === undefined || base === null) return `${current.toFixed(decimals)}%`; + const diff = current - base; + const sign = diff > 0 ? '+' : ''; + const color = diff > 0 ? '🔴' : (diff < 0 ? '🟢' : '⚪'); + return `${current.toFixed(decimals)}% (${color}${sign}${diff.toFixed(decimals)}%)`; + }; + + // Format the results as a markdown table + let comment = '## 🚀 Benchmark Load Test Results\n\n'; + if (baseline) { + comment += 'Comparison with baseline from `tools/benchmarks/results.json`.\n'; + comment += 'Legend: 🟢 improvement, 🔴 regression, ⚪ no change\n\n'; + } else { + comment += 'These results show the performance characteristics of the websocket server under load.\n\n'; + } + + // Create a table for each concurrency level + comment += '| Concurrent Users | Avg CPU | Peak CPU | Avg Memory (MB) | Peak Memory (MB) | Bytes Sent | Bytes Received | Messages Sent | Messages Received | Dropped Connections |\n'; + comment += '|------------------|---------|----------|-----------------|------------------|------------|----------------|---------------|-------------------|--------------------|\n'; + + for (const result of results) { + const { concurrency, cpu, memory, loadSummary } = result; + const base = baseline ? baseline[concurrency] : null; + + const avgCpu = base ? formatPercent(cpu.average, base.cpu.average) : `${cpu.average.toFixed(2)}%`; + const peakCpu = base ? formatPercent(cpu.peak, base.cpu.peak) : `${cpu.peak.toFixed(2)}%`; + const avgMem = base ? formatDiff(memory.averageRssMb, base.memory.averageRssMb) : memory.averageRssMb.toFixed(2); + const peakMem = base ? formatDiff(memory.peakRssMb, base.memory.peakRssMb) : memory.peakRssMb.toFixed(2); + + // Network/message metrics - use formatDiffInt for large numbers with thousand separators + // Note: baseline stores network bytes from server perspective (ingress=from clients, egress=to clients) + // Current results are from client perspective (sent=to server, received=from server) + // So: bytesSent maps to ingressBytes, bytesReceived maps to egressBytes + const bytesSent = base ? formatDiffInt(loadSummary.bytesSent, base.network?.ingressBytes) : formatNumber(loadSummary.bytesSent); + const bytesReceived = base ? formatDiffInt(loadSummary.bytesReceived, base.network?.egressBytes) : formatNumber(loadSummary.bytesReceived); + // Messages are new metrics, show without diff if baseline doesn't have them + const messagesSent = formatNumber(loadSummary.messagesSent); + const messagesReceived = formatNumber(loadSummary.messagesReceived); + + // Note: baseline has droppedConnections at top level, not in loadSummary + const baseDropped = base ? base.droppedConnections : null; + const dropped = base ? formatDiff(loadSummary.droppedConnections, baseDropped, 0) : loadSummary.droppedConnections; + + comment += `| ${concurrency} | ${avgCpu} | ${peakCpu} | ${avgMem} | ${peakMem} | ${bytesSent} | ${bytesReceived} | ${messagesSent} | ${messagesReceived} | ${dropped} |\n`; + } + + comment += '\n### Details\n\n'; + for (const result of results) { + const { concurrency } = result; + comment += `
\n📊 ${concurrency} concurrent users\n\n`; + comment += '```json\n'; + comment += JSON.stringify(result, null, 2); + comment += '\n```\n\n'; + comment += '
\n\n'; + } + + comment += '\n---\n'; + comment += '*Note: These benchmarks run with reduced load (50, 100 users) for CI efficiency. '; + comment += `For full benchmarks, see the [README](https://github.com/${context.repo.owner}/${context.repo.repo}#benchmarking--capacity-planning).*`; + + // Post the comment + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + + - name: Upload benchmark results + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + if: always() + with: + name: benchmark-results + path: | + results.json + benchmark_output.txt