From 36c4a9600046e1234d62c20b61780e0db1028c15 Mon Sep 17 00:00:00 2001 From: Igor Braga <5835477+bragaigor@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:27:24 -0400 Subject: [PATCH 1/3] Assess NOSYNC mode via nitro-testnode Signed-off-by: Igor Braga <5835477+bragaigor@users.noreply.github.com> --- TIER3-BENCHMARK.md | 291 +++++++++++++++++++++++++++++++++++++ docker-compose.yaml | 5 + scripts/index.ts | 2 + scripts/mixedLoad.ts | 255 ++++++++++++++++++++++++++++++++ scripts/run-tier3-bench.sh | 139 ++++++++++++++++++ scripts/tier3-summarize.py | 128 ++++++++++++++++ test-node.bash | 15 ++ 7 files changed, 835 insertions(+) create mode 100644 TIER3-BENCHMARK.md create mode 100644 scripts/mixedLoad.ts create mode 100755 scripts/run-tier3-bench.sh create mode 100755 scripts/tier3-summarize.py diff --git a/TIER3-BENCHMARK.md b/TIER3-BENCHMARK.md new file mode 100644 index 00000000..abffc8f2 --- /dev/null +++ b/TIER3-BENCHMARK.md @@ -0,0 +1,291 @@ +# Tier-3 NoSync vs Sync Pebble Benchmark + +End-to-end benchmark to compare Pebble `--persistent.pebble.sync-mode=true` vs +`false` (the current default) inside the full nitro-testnode docker-compose +stack — real RPC ingress, real batch posting, real container disk semantics. + +This is **tier 3** of a 4-tier investigation. Tiers 1 (Pebble micro-bench) and +2 (system_tests block-validator with packed blocks) already showed NoSync wins +by ~9-15× on `arb/block/writetodb` p50/p95 and ~1.8-5.7× on +`arb/sequencer/block/creation` under realistic packed-block workloads. Tier 3 +exists to confirm those signals survive in the realistic stack, where +batch-poster I/O, RPC backpressure, and container disk semantics could change +the picture. + +## Components + +| File | Role | +| --- | --- | +| `docker-compose.yaml` | Sequencer wired to `${NITRO_PEBBLE_SYNC_MODE}` env var; metrics enabled on port 6070 | +| `test-node.bash` | New `--pebble-sync-mode {true\|false}` flag; exports `NITRO_PEBBLE_SYNC_MODE` | +| `scripts/mixedLoad.ts` | `gen-mixed-load` command — mixed L2 traffic generator (transfers / ERC20 / contract creations) | +| `scripts/scrape-metrics.sh` | Polls `/debug/metrics` once per second; emits one CSV row per sample | +| `scripts/tier3-summarize.py` | Reads the 4 cell CSVs, drops 60s warmup, emits a markdown comparison table | +| `scripts/run-tier3-bench.sh` | Driver — runs the 4-cell matrix `{sync, nosync} × {heavy, light}` | + +## Prerequisites + +- Docker + docker compose v2 +- `jq`, `python3` ≥ 3.10, `curl` (host side) +- Host ports free: `8547` (sequencer RPC), `6070` (metrics) +- ~75 min of wall time for the default matrix (`DURATION=900` × 4 cells + ~1 + min teardown each) + +## Quick start + +From `nitro-testnode/` root: + +```bash +# full matrix with defaults +./scripts/run-tier3-bench.sh + +# results land in: +# results/tier3/{nosync,sync}_{heavy,light}.csv +# results/tier3/summary.md +``` + +The driver handles testnode lifecycle for you — **don't pre-launch the +testnode**. Each cell starts with `docker compose down -v`, boots a fresh +chain, runs steady-state for `$DURATION`, then tears down. + +## Sanity check (run this first) + +Before committing to ~75 min of matrix runs, verify the generator hits the +intended density: + +```bash +# boot a fresh testnode +./test-node.bash --init --no-l2-traffic --pebble-sync-mode false --detach + +# wait ~30s for sequencer warmup, then run a 2-min generator +docker compose run --rm scripts gen-mixed-load --load=heavy --duration=120 + +# in another shell, watch tx counts per block: +while true; do + cast block latest --rpc-url http://localhost:8547 \ + | grep -E "number|transactions" ; echo "---" ; sleep 1 +done + +# tear down when done +docker compose down -v +``` + +Expected: heavy cell sustained at ~25-35 tx/block; light cell at ~5-10 tx/block. + +If density is consistently off: +- **Too low**: increase senders or lower delay (see Configuration below) +- **Too high**: opposite + +The generator's final line shows the achieved aggregate rate: +`done: sent=N errors=0 rate=XXX.X tx/s mix=...`. Multiply rate by 0.25 to get +tx/block at the 250ms block cadence. + +## Configuration + +### Sequencer flag + +`--pebble-sync-mode {true|false}` (default `false`). Sets +`--persistent.pebble.sync-mode` on the sequencer container. + +### `gen-mixed-load` CLI args + +| flag | default | description | +| --- | --- | --- | +| `--load` | `heavy` | profile: `heavy` (~30 tx/block) or `light` (~7 tx/block) | +| `--duration` | `900` | steady-state seconds | +| `--senders` | (profile) | override parallel sender accounts | +| `--delay-ms` | (profile) | override per-sender inter-tx delay (ms) | + +Profile defaults are in `scripts/mixedLoad.ts` `LOAD_PROFILES`: + +```typescript +heavy: { senders: 8, perSenderDelayMs: 60, fundEth: "100" } +light: { senders: 2, perSenderDelayMs: 60, fundEth: "100" } +``` + +Tx mix (also in `mixedLoad.ts`, `TX_MIX`): +- 60% ETH transfer (`gasLimit: 21000`) +- 30% ERC20 transfer (`gasLimit: 100000`) +- 10% small contract creation (`gasLimit: 300000`) + +### Driver env vars (`run-tier3-bench.sh`) + +| var | default | description | +| --- | --- | --- | +| `DURATION` | `900` | steady-state seconds per cell | +| `WARMUP_BLOCKS` | `5` | min block height before starting load | +| `RPC_URL` | `http://localhost:8547` | sequencer RPC for readiness probe | +| `METRICS_URL` | `http://localhost:6070/debug/metrics` | metrics endpoint to scrape | +| `HEAVY_SENDERS` | unset | override sender count for heavy cells | +| `HEAVY_DELAY_MS` | unset | override per-sender delay for heavy cells | +| `LIGHT_SENDERS` | unset | override sender count for light cells | +| `LIGHT_DELAY_MS` | unset | override per-sender delay for light cells | + +Examples: + +```bash +# tune for higher heavy density +HEAVY_SENDERS=12 HEAVY_DELAY_MS=40 ./scripts/run-tier3-bench.sh + +# faster iteration (5 min per cell instead of 15) +DURATION=300 ./scripts/run-tier3-bench.sh + +# all knobs +DURATION=600 HEAVY_SENDERS=10 HEAVY_DELAY_MS=50 \ + LIGHT_SENDERS=3 LIGHT_DELAY_MS=80 \ + ./scripts/run-tier3-bench.sh +``` + +## After editing the generator + +`mixedLoad.ts` is compiled into the `scripts` Docker image at build time. +**After any change to the .ts source, rebuild before re-running:** + +```bash +docker compose build scripts +``` + +The driver script triggers `--init` on each cell, which sets `build_utils=true` +internally — so source changes are picked up automatically when running the +full matrix. Only the manual sanity-check path needs an explicit rebuild. + +## Output + +### Per-cell CSV (`results/tier3/.csv`) + +One row per second of wall time, columns: + +``` +ts_unix, +arb/block/writetodb.count, +arb/block/writetodb.p50_ns, +arb/block/writetodb.p95_ns, +arb/block/writetodb.p99_ns, +arb/block/writetodb.mean_ns, +arb/sequencer/block/creation.count, +arb/sequencer/block/creation.p50_ns, +arb/sequencer/block/creation.p95_ns, +arb/sequencer/block/creation.p99_ns, +arb/sequencer/block/creation.mean_ns, +chain/inserts.count +``` + +All `_ns` values are nanoseconds (go-ethereum histogram default). `count` +columns are monotonic since sequencer start. Histogram percentiles come from +go-ethereum's `BoundedHistogramSample`, which is a recent-window sample so +percentiles smooth quickly. + +### Summary (`results/tier3/summary.md`) + +Markdown table with the same shape as the tier-2 result table: + +``` +| metric | NoSync heavy | Sync heavy | wins by | NoSync light | Sync light | wins by | +| --- | --- | --- | --- | --- | --- | --- | +| arb/block/writetodb count | 12345 | 12340 | — | 3120 | 3115 | — | +| arb/block/writetodb p50 | 0.412 ms | 6.105 ms | 14.8× | 0.401 ms | 5.890 ms | 14.7× | +| arb/block/writetodb p95 | 0.534 ms | 7.612 ms | 14.3× | 0.521 ms | 7.488 ms | 14.4× | +| arb/sequencer/block/creation p50 | 9.812 ms | 17.405 ms | 1.8× | 4.122 ms | 8.901 ms | 2.2× | +| arb/sequencer/block/creation p95 | 11.103 ms | 19.842 ms | 1.8× | 5.011 ms | 10.224 ms | 2.0× | +| chain/inserts count | 12345 | 12340 | — | 3120 | 3115 | — | +``` + +Aggregation rules (see `tier3-summarize.py`): +- First **60 seconds dropped** as warmup (`WARMUP_SEC`). +- Counts: `last - first` over the steady-state window — comparable activity + per cell. +- Percentiles: median of per-sample values across the steady-state window. + Histograms are pre-smoothed so this is stable. +- `wins by`: ratio of Sync over NoSync. Reported as `tie` if within 10%. + +## Expected results + +### Reference: tier 2 (already complete) + +Packed-block `testBlockValidatorComplex` workload (~30 tx/block): + +``` +arb/block/writetodb p50: NoSync 0.655ms vs Sync 6.151ms — 9.4× +arb/block/writetodb p95: NoSync 0.873ms vs Sync 10.030ms — 11.5× +arb/sequencer/block/creation p50: NoSync 9.316ms vs Sync 17.151ms — 1.8× +arb/sequencer/block/creation p95: NoSync 10.390ms vs Sync 19.379ms — 1.9× +``` + +(Simple `testBlockValidatorSimple` workload showed even larger gaps on +writetodb, ~15×, but the depleteGas execution swamped the block-creation +signal.) + +### Tier-3 decision rule + +**Confirms tier 2 (decision: keep NoSync default):** +- writeToDB p50/p95: NoSync wins by ≥ 5× in heavy +- block-creation p50/p95: NoSync wins by ≥ 1.5× in heavy +- light cells show similar (or larger) NoSync advantage + +**Surprises (investigate before deciding):** +- The gap collapses to < 2× — likely batch-poster or RPC backpressure + dominating; tier-3 has surfaced something the in-process test missed +- Sync wins anywhere — extremely unexpected; check for misconfiguration first + (e.g. `wal-bytes-per-sync` not at the 512KB default, batch-poster running + out-of-process) +- Heavy and light disagree directionally — points to a workload-dependent + effect worth understanding before shipping + +## Troubleshooting + +**`bad account name: [...]` from `gen-mixed-load`** — `mixedLoad.ts` uses +`user_mixedload_` accounts. If you see another name in the error, the +scripts image is stale; run `docker compose build scripts`. + +**Generator outputs `rate=` much lower than expected** — likely a stale image +not running the local-signing path. Confirm `mixedLoad.ts` line ~138 contains +`sender.signTransaction(tx)` followed by `sender.provider.sendTransaction`. +Then `docker compose build scripts`. + +**`metrics endpoint not reachable within 60s`** in the driver — check that +docker-compose maps port 6070 (`docker compose ps` should show +`127.0.0.1:6070->6070/tcp` for the sequencer). If missing, your +`docker-compose.yaml` was reverted at some point; re-apply the port mapping. + +**Sequencer doesn't reach the warmup block** — usually means the testnode +boot itself failed. Run `docker compose logs sequencer` to inspect. Most +common: stale `nitro-node-dev-testnode` image after a Nitro source change — +`./test-node.bash --build` rebuilds it. + +**Histogram values are zero in the CSV** — the metric exists but no samples +have arrived yet. The generator's first ~5 seconds may show empty rows; the +60s warmup discard handles this in the summarizer. + +## Customizing + +### Add a new tx kind to the mix + +1. Add to `TxKind` union and `TX_MIX` in `mixedLoad.ts`. +2. Extend the `switch (kind)` in `buildSignableTx` with the tx fields. +3. Extend the `mix` field in stats and the final log line. +4. `docker compose build scripts`. + +### Add a new metric to the scrape + +1. Add the metric name to either `metrics` (histograms) or `counters` + (gauges/counters) in `scrape-metrics.sh`. +2. Add the corresponding `(col, kind, label)` tuple to `METRICS` in + `tier3-summarize.py`. + +### Add another sweep dimension + +The matrix is built from the `cells` array in `run-tier3-bench.sh`. To add a +sweep over, e.g., `wal-bytes-per-sync`, extend the array and pass the new +flag through to `test-node.bash` (which would need a corresponding CLI arg +and env-var passthrough — same pattern as `--pebble-sync-mode`). + +## Known deviation from the original plan + +The plan included a Stylus call as 10% of the mix. nitro-testnode only has a +`StylusDeployer` *factory*, not a callable Stylus program. Activating a real +Stylus contract requires `cargo-stylus` + WASM activation which is significant +scope. Since the benchmark's purpose is Pebble write-path stress (state +writes per block / commit cadence), the Stylus slot was substituted with +another contract creation. If you want a real Stylus call, compile a small +Stylus program once, embed its activated bytecode + address in +`mixedLoad.ts`, and add a `stylus` case to `buildSignableTx`. diff --git a/docker-compose.yaml b/docker-compose.yaml index 896d7e38..3e7358f1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -162,6 +162,7 @@ services: - "127.0.0.1:8547:8547" - "127.0.0.1:8548:8548" - "127.0.0.1:9642:9642" + - "127.0.0.1:6070:6070" volumes: - "seqdata:/home/user/.arbitrum/local/nitro" - "l1keystore:/home/user/l1keystore" @@ -176,6 +177,10 @@ services: - --graphql.enable - --graphql.vhosts=* - --graphql.corsdomain=* + - --persistent.pebble.sync-mode=${NITRO_PEBBLE_SYNC_MODE:-false} + - --metrics + - --metrics-server.addr=0.0.0.0 + - --metrics-server.port=6070 depends_on: - geth diff --git a/scripts/index.ts b/scripts/index.ts index 3516e071..2f283836 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -44,6 +44,7 @@ import { createFeeTokenPricerCommand, grantFiltererRoleCommand, } from "./ethcommands"; +import { mixedLoadCommand } from "./mixedLoad"; async function main() { await Yargs(hideBin(process.argv)) @@ -94,6 +95,7 @@ async function main() { .command(redisReadCommand) .command(redisInitCommand) .command(waitForSyncCommand) + .command(mixedLoadCommand) .strict() .demandCommand(1, "a command must be specified") .epilogue(namedAccountHelpString) diff --git a/scripts/mixedLoad.ts b/scripts/mixedLoad.ts new file mode 100644 index 00000000..b61aeccc --- /dev/null +++ b/scripts/mixedLoad.ts @@ -0,0 +1,255 @@ +import { BigNumber, Contract, ContractFactory, ethers, Wallet } from "ethers"; +import { namedAccount } from "./accounts"; + +const ERC20_ABI = [ + "constructor(uint8 decimals_, address mintTo)", + "function transfer(address to, uint256 amount) returns (bool)", + "function balanceOf(address) view returns (uint256)", +]; + +// Same testnode ERC20 bytecode as deployERC20Contract in ethcommands.ts. +const ERC20_BYTECODE = + "0x60a06040523480156200001157600080fd5b5060405162000d4938038062000d49833981016040819052620000349162000195565b60405180604001604052806008815260200167746573746e6f646560c01b815250604051806040016040528060028152602001612a2760f11b815250816003908162000081919062000288565b50600462000090828262000288565b50505060ff8216608052620000c281620000ac84600a62000469565b620000bc90633b9aca0062000481565b620000ca565b5050620004b9565b6001600160a01b038216620001255760405162461bcd60e51b815260206004820152601f60248201527f45524332303a206d696e7420746f20746865207a65726f206164647265737300604482015260640160405180910390fd5b8060026000828254620001399190620004a3565b90915550506001600160a01b038216600081815260208181526040808320805486019055518481527fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef910160405180910390a35050565b505050565b60008060408385031215620001a957600080fd5b825160ff81168114620001bb57600080fd5b60208401519092506001600160a01b0381168114620001d957600080fd5b809150509250929050565b634e487b7160e01b600052604160045260246000fd5b600181811c908216806200020f57607f821691505b6020821081036200023057634e487b7160e01b600052602260045260246000fd5b50919050565b601f8211156200019057600081815260208120601f850160051c810160208610156200025f5750805b601f850160051c820191505b8181101562000280578281556001016200026b565b505050505050565b81516001600160401b03811115620002a457620002a4620001e4565b620002bc81620002b58454620001fa565b8462000236565b602080601f831160018114620002f45760008415620002db5750858301515b600019600386901b1c1916600185901b17855562000280565b600085815260208120601f198616915b82811015620003255788860151825594840194600190910190840162000304565b5085821015620003445787850151600019600388901b60f8161c191681555b5050505050600190811b01905550565b634e487b7160e01b600052601160045260246000fd5b600181815b80851115620003ab5781600019048211156200038f576200038f62000354565b808516156200039d57918102915b93841c93908002906200036f565b509250929050565b600082620003c45750600162000463565b81620003d35750600062000463565b8160018114620003ec5760028114620003f75762000417565b600191505062000463565b60ff8411156200040b576200040b62000354565b50506001821b62000463565b5060208310610133831016604e8410600b84101617156200043c575081810a62000463565b6200044883836200036a565b80600019048211156200045f576200045f62000354565b0290505b92915050565b60006200047a60ff841683620003b3565b9392505050565b60008160001904831182151516156200049e576200049e62000354565b500290565b8082018082111562000463576200046362000354565b608051610874620004d5600039600061011b01526108746000f3fe608060405234801561001057600080fd5b50600436106100a95760003560e01c80633950935111610071578063395093511461014557806370a082311461015857806395d89b4114610181578063a457c2d714610189578063a9059cbb1461019c578063dd62ed3e146101af57600080fd5b806306fdde03146100ae578063095ea7b3146100cc57806318160ddd146100ef57806323b872dd14610101578063313ce56714610114575b600080fd5b6100b66101c2565b6040516100c391906106be565b60405180910390f35b6100df6100da366004610728565b610254565b60405190151581526020016100c3565b6002545b6040519081526020016100c3565b6100df61010f366004610752565b61026e565b60405160ff7f00000000000000000000000000000000000000000000000000000000000000001681526020016100c3565b6100df610153366004610728565b610292565b6100f361016636600461078e565b6001600160a01b031660009081526020819052604090205490565b6100b66102b4565b6100df610197366004610728565b6102c3565b6100df6101aa366004610728565b610343565b6100f36101bd3660046107b0565b610351565b6060600380546101d1906107e3565b80601f01602080910402602001604051908101604052809291908181526020018280546101fd906107e3565b801561024a5780601f1061021f5761010080835404028352916020019161024a565b820191906000526020600020905b81548152906001019060200180831161022d57829003601f168201915b5050505050905090565b60003361026281858561037c565b60019150505b92915050565b60003361027c8582856104a0565b61028785858561051a565b506001949350505050565b6000336102628185856102a58383610351565b6102af919061081d565b61037c565b6060600480546101d1906107e3565b600033816102d18286610351565b9050838110156103365760405162461bcd60e51b815260206004820152602560248201527f45524332303a2064656372656173656420616c6c6f77616e63652062656c6f77604482015264207a65726f60d81b60648201526084015b60405180910390fd5b610287828686840361037c565b60003361026281858561051a565b6001600160a01b03918216600090815260016020908152604080832093909416825291909152205490565b6001600160a01b0383166103de5760405162461bcd60e51b8152602060048201526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b606482015260840161032d565b6001600160a01b03821661043f5760405162461bcd60e51b815260206004820152602260248201527f45524332303a20617070726f766520746f20746865207a65726f206164647265604482015261737360f01b606482015260840161032d565b6001600160a01b0383811660008181526001602090815260408083209487168084529482529182902085905590518481527f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925910160405180910390a3505050565b60006104ac8484610351565b9050600019811461051457818110156105075760405162461bcd60e51b815260206004820152601d60248201527f45524332303a20696e73756666696369656e7420616c6c6f77616e6365000000604482015260640161032d565b610514848484840361037c565b50505050565b6001600160a01b03831661057e5760405162461bcd60e51b815260206004820152602560248201527f45524332303a207472616e736665722066726f6d20746865207a65726f206164604482015264647265737360d81b606482015260840161032d565b6001600160a01b0382166105e05760405162461bcd60e51b815260206004820152602360248201527f45524332303a207472616e7366657220746f20746865207a65726f206164647260448201526265737360e81b606482015260840161032d565b6001600160a01b038316600090815260208190526040902054818110156106585760405162461bcd60e51b815260206004820152602660248201527f45524332303a207472616e7366657220616d6f756e7420657863656564732062604482015265616c616e636560d01b606482015260840161032d565b6001600160a01b03848116600081815260208181526040808320878703905593871680835291849020805487019055925185815290927fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef910160405180910390a3610514565b600060208083528351808285015260005b818110156106eb578581018301518582016040015282016106cf565b506000604082860101526040601f19601f8301168501019250505092915050565b80356001600160a01b038116811461072357600080fd5b919050565b6000806040838503121561073b57600080fd5b6107448361070c565b946020939093013593505050565b60008060006060848603121561076757600080fd5b6107708461070c565b925061077e6020850161070c565b9150604084013590509250925092565b6000602082840312156107a057600080fd5b6107a98261070c565b9392505050565b600080604083850312156107c357600080fd5b6107cc8361070c565b91506107da6020840161070c565b90509250929050565b600181811c908216806107f757607f821691505b60208210810361081757634e487b7160e01b600052602260045260246000fd5b50919050565b8082018082111561026857634e487b7160e01b600052601160045260246000fdfea2646970667358221220257f3d763bae7b8c0189ed676531d85a1046e0bea68722f67c2616d46f01c02964736f6c63430008100033"; + +// Returns 5 zero bytes as runtime code; minimal valid contract creation init code. +const SMALL_CONTRACT_INIT_CODE = "0x6005600c60003960056000f30000000000"; + +interface LoadProfile { + senders: number; + perSenderDelayMs: number; + fundEth: string; +} + +// Targets at 250ms blocks (4 blocks/sec): +// heavy: ~120 tx/sec aggregate -> ~30 tx/block +// light: ~30 tx/sec aggregate -> ~7 tx/block +const LOAD_PROFILES: { [k: string]: LoadProfile } = { + heavy: { senders: 8, perSenderDelayMs: 60, fundEth: "100" }, + light: { senders: 2, perSenderDelayMs: 60, fundEth: "100" }, +}; + +const TX_MIX = [ + { kind: "transfer", weight: 60 }, + { kind: "erc20", weight: 30 }, + { kind: "create", weight: 10 }, +] as const; +type TxKind = typeof TX_MIX[number]["kind"]; + +function pickKind(): TxKind { + let r = Math.random() * TX_MIX.reduce((s, e) => s + e.weight, 0); + for (const e of TX_MIX) { + r -= e.weight; + if (r <= 0) return e.kind; + } + return TX_MIX[0].kind; +} + +function senderName(threadId: number): string { + return `user_mixedload_${threadId}`; +} + +async function deployErc20(funnel: Wallet): Promise { + const factory = new ContractFactory(ERC20_ABI, ERC20_BYTECODE, funnel); + const decimals = 18; + const token = await factory.deploy(decimals, funnel.address); + await token.deployTransaction.wait(); + return new Contract(token.address, ERC20_ABI, funnel); +} + +async function fundSenders( + funnel: Wallet, + senders: Wallet[], + fundEth: string, + erc20: Contract, +): Promise { + // Sequential funding txs from funnel — nonce ordering matters and sender count is small. + let nonce = await funnel.getTransactionCount("pending"); + const ethAmount = ethers.utils.parseEther(fundEth); + const erc20Amount = ethers.utils.parseUnits("1000000", 18); // 1M tokens per sender + const pending: Promise[] = []; + for (const s of senders) { + pending.push( + funnel.sendTransaction({ to: s.address, value: ethAmount, nonce: nonce++ }), + ); + pending.push( + erc20.transfer(s.address, erc20Amount, { nonce: nonce++ }), + ); + } + // Wait for the last one to confirm so balances are observable before steady state. + const last = await pending[pending.length - 1]; + await last.wait(); +} + +interface ChainCtx { + chainId: number; + gasPrice: BigNumber; + erc20Address: string; +} + +function buildSignableTx( + ctx: ChainCtx, + nonce: number, + kind: TxKind, +): ethers.providers.TransactionRequest { + const recipient = ethers.Wallet.createRandom().address; + const base = { chainId: ctx.chainId, gasPrice: ctx.gasPrice, nonce }; + switch (kind) { + case "transfer": + return { + ...base, + to: recipient, + value: BigNumber.from(1), + gasLimit: 21000, + }; + case "erc20": { + const iface = new ethers.utils.Interface(ERC20_ABI); + return { + ...base, + to: ctx.erc20Address, + data: iface.encodeFunctionData("transfer", [recipient, 1]), + gasLimit: 100000, + }; + } + case "create": + return { + ...base, + data: SMALL_CONTRACT_INIT_CODE, + gasLimit: 300000, + }; + } +} + +async function runSender( + sender: Wallet, + ctx: ChainCtx, + deadline: number, + perSenderDelayMs: number, + stats: { sent: number; errors: number; mix: { [k in TxKind]: number } }, +): Promise { + // Sign locally with all fields explicit, then submit raw — bypasses ethers' + // populateTransaction(), which would otherwise add a getFeeData() RPC per tx. + let nonce = await sender.getTransactionCount("pending"); + let consecutiveErrors = 0; + while (Date.now() < deadline) { + const kind = pickKind(); + try { + const tx = buildSignableTx(ctx, nonce, kind); + const signed = await sender.signTransaction(tx); + await sender.provider.sendTransaction(signed); + nonce++; + stats.sent++; + stats.mix[kind]++; + consecutiveErrors = 0; + } catch (e: any) { + stats.errors++; + consecutiveErrors++; + if (stats.errors <= 5) { + console.error(`sender ${sender.address} tx error (${kind}):`, e?.message ?? e); + } + // Recover from nonce drift by re-syncing with the node. + if (consecutiveErrors >= 3) { + nonce = await sender.getTransactionCount("pending"); + consecutiveErrors = 0; + } + } + if (perSenderDelayMs > 0) { + await new Promise((f) => setTimeout(f, perSenderDelayMs)); + } + } +} + +export const mixedLoadCommand = { + command: "gen-mixed-load", + describe: + "generates mixed L2 traffic (transfers / ERC20 transfers / contract creations) for benchmarks", + builder: { + load: { + string: true, + describe: "load profile: heavy (~30 tx/block) or light (~7 tx/block)", + default: "heavy", + }, + duration: { + number: true, + describe: "steady-state duration in seconds", + default: 900, + }, + senders: { + number: true, + describe: "override number of parallel sender accounts", + }, + delayMs: { + number: true, + describe: "override per-sender inter-tx delay in ms", + }, + }, + handler: async (argv: any) => { + const baseProfile = LOAD_PROFILES[argv.load]; + if (!baseProfile) { + console.error(`unknown load profile: ${argv.load}; expected heavy|light`); + process.exit(1); + } + const profile: LoadProfile = { + ...baseProfile, + senders: argv.senders ?? baseProfile.senders, + perSenderDelayMs: argv.delayMs ?? baseProfile.perSenderDelayMs, + }; + + const provider = new ethers.providers.WebSocketProvider(argv.l2url); + const funnel = namedAccount("funnel").connect(provider); + + console.log( + `gen-mixed-load: profile=${argv.load} senders=${profile.senders} ` + + `delay=${profile.perSenderDelayMs}ms duration=${argv.duration}s`, + ); + + const senders: Wallet[] = []; + for (let i = 0; i < profile.senders; i++) { + senders.push(namedAccount(senderName(i)).connect(provider)); + } + + console.log("deploying ERC20 token..."); + const erc20 = await deployErc20(funnel); + console.log(`ERC20 at ${erc20.address}`); + + console.log(`funding ${senders.length} senders...`); + await fundSenders(funnel, senders, profile.fundEth, erc20); + + // Cache chainId + gasPrice once. Each tx then signs locally and submits + // raw, avoiding ethers' per-tx getFeeData() RPC overhead (~3 round-trips + // per tx -> 1). + const network = await provider.getNetwork(); + const baseGasPrice = await provider.getGasPrice(); + const ctx: ChainCtx = { + chainId: network.chainId, + gasPrice: baseGasPrice.mul(2), // 2x buffer for inclusion under load + erc20Address: erc20.address, + }; + console.log( + `chainId=${ctx.chainId} gasPrice=${ctx.gasPrice.toString()} wei`, + ); + + const deadline = Date.now() + argv.duration * 1000; + const stats = { + sent: 0, + errors: 0, + mix: { transfer: 0, erc20: 0, create: 0 } as { [k in TxKind]: number }, + }; + + console.log("entering steady state..."); + const startedAt = Date.now(); + await Promise.all( + senders.map((s) => + runSender(s, ctx, deadline, profile.perSenderDelayMs, stats), + ), + ); + const elapsedSec = (Date.now() - startedAt) / 1000; + + console.log( + `done: sent=${stats.sent} errors=${stats.errors} ` + + `rate=${(stats.sent / elapsedSec).toFixed(1)} tx/s ` + + `mix=transfer:${stats.mix.transfer},erc20:${stats.mix.erc20},create:${stats.mix.create}`, + ); + + provider.destroy(); + }, +}; diff --git a/scripts/run-tier3-bench.sh b/scripts/run-tier3-bench.sh new file mode 100755 index 00000000..0fbed09f --- /dev/null +++ b/scripts/run-tier3-bench.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Drives the tier-3 NoSync vs Sync benchmark matrix. +# +# Matrix: {sync, nosync} x {heavy, light} = 4 cells. +# Per cell: +# 1. Tear down any existing testnode state. +# 2. Boot testnode (init + detach) with the cell's sync-mode flag. +# 3. Wait for sequencer readiness. +# 4. Run mixed-load generator + metrics scraper concurrently for $DURATION. +# 5. Stop scraper, tear down testnode. +# After the matrix, render the comparison markdown table. +# +# Env overrides: +# DURATION steady-state seconds per cell (default 900 = 15 min) +# WARMUP_BLOCKS blocks to wait for after boot before starting load (default 5) +# HEAVY_SENDERS override sender count for heavy cells (default: mixedLoad.ts default) +# HEAVY_DELAY_MS override per-sender delay (ms) for heavy cells +# LIGHT_SENDERS override sender count for light cells +# LIGHT_DELAY_MS override per-sender delay (ms) for light cells +# +# Run from nitro-testnode/ root: +# ./scripts/run-tier3-bench.sh +# HEAVY_SENDERS=12 HEAVY_DELAY_MS=30 ./scripts/run-tier3-bench.sh + +set -eu + +mydir="$(cd "$(dirname "$0")" && pwd)" +cd "${mydir}/.." + +DURATION="${DURATION:-900}" +WARMUP_BLOCKS="${WARMUP_BLOCKS:-5}" +RPC_URL="${RPC_URL:-http://localhost:8547}" +METRICS_URL="${METRICS_URL:-http://localhost:6070/debug/metrics}" +HEAVY_SENDERS="${HEAVY_SENDERS:-}" +HEAVY_DELAY_MS="${HEAVY_DELAY_MS:-}" +LIGHT_SENDERS="${LIGHT_SENDERS:-}" +LIGHT_DELAY_MS="${LIGHT_DELAY_MS:-}" + +results_dir="${PWD}/results/tier3" +mkdir -p "$results_dir" + +cells=( + "false:heavy:nosync_heavy" + "true:heavy:sync_heavy" + "false:light:nosync_light" + "true:light:sync_light" +) + +wait_for_block() { + local target="$1" + local deadline=$(( $(date +%s) + 180 )) + while [[ $(date +%s) -lt $deadline ]]; do + local hex + hex=$(curl -fsS --max-time 2 -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + "$RPC_URL" 2>/dev/null | jq -r '.result // empty' || true) + if [[ -n "$hex" && "$hex" != "null" ]]; then + local n=$((hex)) + if [[ $n -ge $target ]]; then + echo "sequencer ready at block $n" + return 0 + fi + fi + sleep 2 + done + echo "ERROR: sequencer did not reach block $target within 180s" >&2 + return 1 +} + +wait_for_metrics() { + local deadline=$(( $(date +%s) + 60 )) + while [[ $(date +%s) -lt $deadline ]]; do + if curl -fsS --max-time 2 "$METRICS_URL" >/dev/null 2>&1; then + echo "metrics endpoint ready at $METRICS_URL" + return 0 + fi + sleep 1 + done + echo "ERROR: metrics endpoint $METRICS_URL not reachable within 60s" >&2 + return 1 +} + +run_cell() { + local sync_mode="$1" + local load="$2" + local run_id="$3" + + echo "============================================================" + echo "cell: ${run_id} (sync-mode=${sync_mode}, load=${load})" + echo "============================================================" + + echo "[$run_id] tearing down any existing testnode state..." + docker compose down -v --remove-orphans >/dev/null 2>&1 || true + + echo "[$run_id] booting testnode..." + ./test-node.bash --init --no-l2-traffic --pebble-sync-mode "$sync_mode" --detach + + wait_for_block "$WARMUP_BLOCKS" + wait_for_metrics + + echo "[$run_id] starting metrics scraper..." + ./scripts/scrape-metrics.sh "$run_id" "$METRICS_URL" 1 & + local scraper_pid=$! + + local gen_args=(--load="$load" --duration="$DURATION") + if [[ "$load" == "heavy" ]]; then + [[ -n "$HEAVY_SENDERS" ]] && gen_args+=(--senders="$HEAVY_SENDERS") + [[ -n "$HEAVY_DELAY_MS" ]] && gen_args+=(--delay-ms="$HEAVY_DELAY_MS") + else + [[ -n "$LIGHT_SENDERS" ]] && gen_args+=(--senders="$LIGHT_SENDERS") + [[ -n "$LIGHT_DELAY_MS" ]] && gen_args+=(--delay-ms="$LIGHT_DELAY_MS") + fi + + echo "[$run_id] starting mixed-load generator (${gen_args[*]})..." + docker compose run --rm scripts gen-mixed-load "${gen_args[@]}" || { + echo "[$run_id] WARNING: generator exited non-zero" >&2 + } + + echo "[$run_id] stopping scraper..." + kill -TERM "$scraper_pid" 2>/dev/null || true + wait "$scraper_pid" 2>/dev/null || true + + echo "[$run_id] tearing down testnode..." + docker compose down -v --remove-orphans >/dev/null 2>&1 || true + + echo "[$run_id] done. CSV: ${results_dir}/${run_id}.csv" +} + +for cell in "${cells[@]}"; do + IFS=':' read -r sync_mode load run_id <<< "$cell" + run_cell "$sync_mode" "$load" "$run_id" +done + +echo "============================================================" +echo "matrix complete. summarizing..." +echo "============================================================" +python3 "${mydir}/tier3-summarize.py" "$results_dir" | tee "${results_dir}/summary.md" +echo +echo "summary written to ${results_dir}/summary.md" diff --git a/scripts/tier3-summarize.py b/scripts/tier3-summarize.py new file mode 100755 index 00000000..974b8f1f --- /dev/null +++ b/scripts/tier3-summarize.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Reads the 4 tier-3 CSVs produced by scrape-metrics.sh and emits a markdown +table comparing NoSync vs Sync at the {heavy, light} workload densities. + +Filename convention (set by run-tier3-bench.sh): + {nosync,sync}_{heavy,light}.csv + +Aggregation rules: + - Skip the first WARMUP_SEC seconds of each run (sequencer cold cache, etc.). + - For percentiles: take the median of the per-sample values across the + steady-state window. Histograms are sample-based and already smoothed, + so per-sample medians are stable representatives. + - For counts: report (last - first) over the steady-state window — this is + the activity that happened in that window, comparable across cells. + - Times are reported in milliseconds (raw values are nanoseconds). +""" + +import csv +import statistics +import sys +from pathlib import Path + +WARMUP_SEC = 60 +NS_PER_MS = 1_000_000.0 + +METRICS = [ + ("arb/block/writetodb.count", "count", "arb/block/writetodb count"), + ("arb/block/writetodb.p50_ns", "p_ms", "arb/block/writetodb p50"), + ("arb/block/writetodb.p95_ns", "p_ms", "arb/block/writetodb p95"), + ("arb/sequencer/block/creation.count", "count", "arb/sequencer/block/creation count"), + ("arb/sequencer/block/creation.p50_ns", "p_ms", "arb/sequencer/block/creation p50"), + ("arb/sequencer/block/creation.p95_ns", "p_ms", "arb/sequencer/block/creation p95"), + ("chain/inserts.count", "count", "chain/inserts count"), +] + +CELLS = [ + ("nosync_heavy", "NoSync heavy"), + ("sync_heavy", "Sync heavy"), + ("nosync_light", "NoSync light"), + ("sync_light", "Sync light"), +] + + +def load_cell(path: Path) -> dict[str, list[float]]: + """Reads a CSV and returns column -> list of floats (warmup skipped).""" + cols: dict[str, list[float]] = {} + with path.open() as f: + reader = csv.DictReader(f) + rows = list(reader) + if not rows: + return cols + t0 = float(rows[0]["ts_unix"]) + for row in rows: + if float(row["ts_unix"]) - t0 < WARMUP_SEC: + continue + for k, v in row.items(): + if k == "ts_unix" or v == "": + continue + try: + cols.setdefault(k, []).append(float(v)) + except ValueError: + pass + return cols + + +def aggregate(values: list[float], kind: str) -> float | None: + if not values: + return None + if kind == "count": + return values[-1] - values[0] + if kind == "p_ms": + return statistics.median(values) / NS_PER_MS + raise ValueError(kind) + + +def fmt(v: float | None, kind: str) -> str: + if v is None: + return "n/a" + if kind == "count": + return f"{int(v)}" + if kind == "p_ms": + return f"{v:.3f} ms" + return str(v) + + +def wins(nosync: float | None, sync: float | None, kind: str) -> str: + if kind == "count": + return "—" + if not nosync or not sync: + return "n/a" + ratio = sync / nosync if nosync > 0 else 0 + if ratio < 1.1 and ratio > 0.9: + return "tie" + return f"{ratio:.1f}×" + + +def main(csv_dir: Path) -> int: + cells: dict[str, dict[str, list[float]]] = {} + for cell_id, _ in CELLS: + path = csv_dir / f"{cell_id}.csv" + if not path.exists(): + print(f"warning: missing {path}", file=sys.stderr) + cells[cell_id] = {} + continue + cells[cell_id] = load_cell(path) + + # Markdown table. + print("| metric | NoSync heavy | Sync heavy | wins by | NoSync light | Sync light | wins by |") + print("|---|---|---|---|---|---|---|") + for col, kind, label in METRICS: + vals = { + cell_id: aggregate(cells[cell_id].get(col, []), kind) + for cell_id, _ in CELLS + } + print( + f"| {label} | {fmt(vals['nosync_heavy'], kind)} | {fmt(vals['sync_heavy'], kind)} | " + f"{wins(vals['nosync_heavy'], vals['sync_heavy'], kind)} | " + f"{fmt(vals['nosync_light'], kind)} | {fmt(vals['sync_light'], kind)} | " + f"{wins(vals['nosync_light'], vals['sync_light'], kind)} |" + ) + return 0 + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + sys.exit(main(Path(sys.argv[1]))) diff --git a/test-node.bash b/test-node.bash index 3c80c67a..eff4a581 100755 --- a/test-node.bash +++ b/test-node.bash @@ -80,6 +80,9 @@ build_node_images=false l2_traffic=true l3_traffic=true +# Pebble sync mode for the sequencer (false = NoSync / async WAL) +pebble_sync_mode=false + while [[ $# -gt 0 ]]; do case $1 in --init) @@ -306,6 +309,15 @@ while [[ $# -gt 0 ]]; do l3_traffic=false shift ;; + --pebble-sync-mode|--pebble-sync-mode=*) + if [[ "$1" == --pebble-sync-mode=* ]]; then + pebble_sync_mode="${1#*=}" + shift + else + pebble_sync_mode="$2" + shift 2 + fi + ;; *) echo Usage: $0 \[OPTIONS..] echo $0 script [SCRIPT-ARGS] @@ -336,6 +348,7 @@ while [[ $# -gt 0 ]]; do echo --no-run does not launch nodes \(useful with build or init\) echo --no-l2-traffic disables L2 spam transaction traffic \(default: enabled\) echo --no-l3-traffic disables L3 spam transaction traffic \(default: enabled\) + echo --pebble-sync-mode VAL set sequencer pebble sync-mode \(true=fsync per commit, false=async WAL\) \(default: false\) echo --no-simple run a full configuration with separate sequencer/batch-poster/validator/relayer echo --build-dev-nitro rebuild dev nitro docker image echo --no-build-dev-nitro don\'t rebuild dev nitro docker image @@ -350,6 +363,8 @@ while [[ $# -gt 0 ]]; do esac done +export NITRO_PEBBLE_SYNC_MODE="$pebble_sync_mode" + NODES="sequencer" INITIAL_SEQ_NODES="sequencer" From 608bec8a091eacce98448f14d0a8d54d5a4f893f Mon Sep 17 00:00:00 2001 From: Igor Braga <5835477+bragaigor@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:00:35 -0400 Subject: [PATCH 2/3] add missing scrape-metrics.sh Signed-off-by: Igor Braga <5835477+bragaigor@users.noreply.github.com> --- scripts/scrape-metrics.sh | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 scripts/scrape-metrics.sh diff --git a/scripts/scrape-metrics.sh b/scripts/scrape-metrics.sh new file mode 100755 index 00000000..aec370f7 --- /dev/null +++ b/scripts/scrape-metrics.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Polls the sequencer's expvar metrics endpoint at a fixed interval and emits a +# CSV row per sample. Designed to be killed with SIGTERM by the driver script; +# on exit it leaves the CSV closed cleanly. +# +# Usage: scrape-metrics.sh [endpoint] [interval-sec] +# run-id - identifier used in the output filename (e.g. "nosync_heavy") +# endpoint - default http://localhost:6070/debug/metrics +# interval-sec - default 1 + +set -eu + +run_id="${1:?usage: $0 [endpoint] [interval-sec]}" +endpoint="${2:-http://localhost:6070/debug/metrics}" +interval="${3:-1}" + +mydir="$(cd "$(dirname "$0")" && pwd)" +results_dir="${mydir}/../results/tier3" +mkdir -p "$results_dir" +out="${results_dir}/${run_id}.csv" + +# Histograms we care about. Counters and percentiles are extracted from each. +metrics=( + "arb/block/writetodb" + "arb/sequencer/block/creation" +) +# Counters/gauges (just .count or scalar value). +counters=( + "chain/inserts" +) + +# Header. +{ + printf "ts_unix" + for m in "${metrics[@]}"; do + printf ",%s.count,%s.p50_ns,%s.p95_ns,%s.p99_ns,%s.mean_ns" "$m" "$m" "$m" "$m" "$m" + done + for c in "${counters[@]}"; do + printf ",%s.count" "$c" + done + printf "\n" +} > "$out" + +# Build the jq filter once. Missing keys yield empty fields (no error). +jq_filter='[(now | floor)' +for m in "${metrics[@]}"; do + jq_filter+=", (.[\"${m}.count\"] // \"\"), (.[\"${m}.50-percentile\"] // \"\"), (.[\"${m}.95-percentile\"] // \"\"), (.[\"${m}.99-percentile\"] // \"\"), (.[\"${m}.mean\"] // \"\")" +done +for c in "${counters[@]}"; do + jq_filter+=", (.[\"${c}.count\"] // .[\"${c}\"] // \"\")" +done +jq_filter+='] | @csv' + +trap 'echo "scrape-metrics: stopping, wrote $(( $(wc -l < "'"$out"'") - 1 )) samples to '"$out"'"; exit 0' TERM INT + +while true; do + if row=$(curl -fsS --max-time 5 "$endpoint" | jq -r "$jq_filter" 2>/dev/null); then + # jq @csv quotes everything as strings; strip quotes for numeric parsing. + echo "$row" | tr -d '"' >> "$out" + fi + sleep "$interval" +done From c4228d60cb297ca8ec8fedbdaaa81683a8d02808 Mon Sep 17 00:00:00 2001 From: Igor Braga <5835477+bragaigor@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:30:30 -0400 Subject: [PATCH 3/3] display light and heavy mode results separately Signed-off-by: Igor Braga <5835477+bragaigor@users.noreply.github.com> --- scripts/tier3-summarize.py | 52 ++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/scripts/tier3-summarize.py b/scripts/tier3-summarize.py index 974b8f1f..232963e2 100755 --- a/scripts/tier3-summarize.py +++ b/scripts/tier3-summarize.py @@ -94,6 +94,39 @@ def wins(nosync: float | None, sync: float | None, kind: str) -> str: return f"{ratio:.1f}×" +def print_combined_table(cells: dict[str, dict[str, list[float]]]) -> None: + print("## Combined matrix") + print() + print("| metric | NoSync heavy | Sync heavy | wins by | NoSync light | Sync light | wins by |") + print("|---|---|---|---|---|---|---|") + for col, kind, label in METRICS: + vals = {cell_id: aggregate(cells[cell_id].get(col, []), kind) for cell_id, _ in CELLS} + print( + f"| {label} | {fmt(vals['nosync_heavy'], kind)} | {fmt(vals['sync_heavy'], kind)} | " + f"{wins(vals['nosync_heavy'], vals['sync_heavy'], kind)} | " + f"{fmt(vals['nosync_light'], kind)} | {fmt(vals['sync_light'], kind)} | " + f"{wins(vals['nosync_light'], vals['sync_light'], kind)} |" + ) + + +def print_split_table( + cells: dict[str, dict[str, list[float]]], + nosync_id: str, + sync_id: str, + title: str, +) -> None: + print(f"## {title}") + print() + print("| metric | NoSync | Sync | wins by |") + print("|---|---|---|---|") + for col, kind, label in METRICS: + nosync = aggregate(cells[nosync_id].get(col, []), kind) + sync = aggregate(cells[sync_id].get(col, []), kind) + print( + f"| {label} | {fmt(nosync, kind)} | {fmt(sync, kind)} | {wins(nosync, sync, kind)} |" + ) + + def main(csv_dir: Path) -> int: cells: dict[str, dict[str, list[float]]] = {} for cell_id, _ in CELLS: @@ -104,20 +137,11 @@ def main(csv_dir: Path) -> int: continue cells[cell_id] = load_cell(path) - # Markdown table. - print("| metric | NoSync heavy | Sync heavy | wins by | NoSync light | Sync light | wins by |") - print("|---|---|---|---|---|---|---|") - for col, kind, label in METRICS: - vals = { - cell_id: aggregate(cells[cell_id].get(col, []), kind) - for cell_id, _ in CELLS - } - print( - f"| {label} | {fmt(vals['nosync_heavy'], kind)} | {fmt(vals['sync_heavy'], kind)} | " - f"{wins(vals['nosync_heavy'], vals['sync_heavy'], kind)} | " - f"{fmt(vals['nosync_light'], kind)} | {fmt(vals['sync_light'], kind)} | " - f"{wins(vals['nosync_light'], vals['sync_light'], kind)} |" - ) + print_combined_table(cells) + print() + print_split_table(cells, "nosync_heavy", "sync_heavy", "Heavy workload (NoSync vs Sync)") + print() + print_split_table(cells, "nosync_light", "sync_light", "Light workload (NoSync vs Sync)") return 0