From f3337326a24af58cece4b47d14e8720da8227e88 Mon Sep 17 00:00:00 2001 From: FlowmemoryAI <283694809+FlowmemoryAI@users.noreply.github.com> Date: Wed, 13 May 2026 15:00:11 -0500 Subject: [PATCH] Add guarded Base canary reader --- docs/CURRENT_STATE.md | 9 +- docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md | 50 +++- docs/ROADMAP.md | 5 +- package.json | 2 + services/indexer/README.md | 11 + services/indexer/package.json | 1 + services/indexer/src/base-canary.ts | 215 +++++++++++++++ services/indexer/src/base-sepolia.ts | 52 +--- services/indexer/src/indexer.ts | 2 +- services/indexer/src/persistence.ts | 67 +++++ services/indexer/src/reader-utils.ts | 39 +++ services/indexer/src/rpc.ts | 10 + services/indexer/test/indexer.test.ts | 259 +++++++++++++++++- 13 files changed, 664 insertions(+), 58 deletions(-) create mode 100644 services/indexer/src/base-canary.ts create mode 100644 services/indexer/src/reader-utils.ts diff --git a/docs/CURRENT_STATE.md b/docs/CURRENT_STATE.md index bd4a9696..09838602 100644 --- a/docs/CURRENT_STATE.md +++ b/docs/CURRENT_STATE.md @@ -10,7 +10,7 @@ FlowMemory is in launch-candidate V0 hardening. The bootstrap repository operating system, contracts V0 foundation, crypto V0 foundation, local indexer/verifier fixture package, dashboard V0, FlowRouter hardware POC, local no-value devnet prototype, launch-core contract-event spine, and pre-production hardening guardrails have merged into `main`. The launch-candidate work added swap-derived memory signals, stricter launch validation, and Base Sepolia testnet deploy/read commands. -On 2026-05-13 a small Base mainnet canary deployment was broadcast for V0 testing. It is documented in `docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md`. This is not a production launch and does not change the production/mainnet-readiness guardrails. +On 2026-05-13 a small Base mainnet canary deployment was broadcast for V0 testing. It is documented in `docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md`. A guarded Base mainnet canary reader now exists for those known canary addresses and small explicit block ranges. This is not a production launch and does not change the production/mainnet-readiness guardrails. The launch-core V0 stack now has a single runnable local command that connects contract fixtures, local indexing/verifier outputs, crypto schema vocabulary, Rootflow transitions, Flow Memory objects, generated dashboard state, local no-value devnet output, and hardware POC output without production deployment. @@ -55,6 +55,9 @@ Indexer/verifier local package: - The verifier supports local fixture checks for rootfield registration, root commitments, and swap-derived memory-signal commitments. - `npm run index:base-sepolia -- --rpc-url --address --from-block --to-block ` provides a constrained Base Sepolia reader path. - The Base Sepolia reader requires an explicit RPC URL, rejects non-Base-Sepolia chain ids, and persists both canonical state and a durable checkpoint without storing RPC URLs or keys. +- `npm run index:base-canary -- --acknowledge-mainnet-canary --rpc-url --address --from-block --to-block ` provides a guarded Base mainnet canary reader path for the documented V0 canary deployment only. +- The Base canary reader requires explicit acknowledgement, RPC URL, addresses, and block range; rejects non-Base-mainnet chain ids; refuses scans wider than 5,000 blocks; persists canonical state plus a durable canary checkpoint; and marks the checkpoint as not production-ready. +- A live canary read over blocks `45955500` to `45955540` observed 4 FlowPulse logs from the documented `RootfieldRegistry` and `FlowMemoryHookAdapter` canary addresses with 0 rejected logs and 0 duplicates. - `npm run deploy:base-sepolia` and `npm run deploy:base-sepolia:broadcast` provide Foundry deploy commands for the current V0 Base Sepolia testnet contract set. They require local env values and do not commit credentials. - A Base mainnet V0 canary deployment exists for testing only; deployed addresses and smoke transactions are recorded in `docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md`. @@ -114,7 +117,7 @@ Launch-core specifications: - Hosted launch-core services. - Production indexer or verifier service runtime. - Production persistence layer, production live RPC reader, production APIs, or hosted services. -- Base mainnet reader. +- Broad Base mainnet reader. - Dashboard ingestion of the Base mainnet canary deployment. - Contract source verification automation for the deployed canary contracts. - Explorer or hardware console implementation. @@ -176,7 +179,7 @@ Before assigning agents, check for dirty worktrees and avoid overlapping folders ## Current Operator Priorities 1. Keep the generated launch-core command stable in CI. -2. Build a guarded Base canary reader path for the deployed V0 canary addresses. +2. Exercise the guarded Base canary reader against the documented V0 canary addresses and feed its output into the next dashboard canary-ingestion issue. 3. Exercise the Base Sepolia deploy/read path on explicit testnet contract addresses only. 4. Continue contracts hardening without production mainnet deployment or token mechanics. 5. Keep dashboard work fixture-backed until a production API is explicitly scoped. diff --git a/docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md b/docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md index 9211486a..7cdf868d 100644 --- a/docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md +++ b/docs/DEPLOYMENTS/2026-05-13-base-canary-v0.md @@ -70,6 +70,18 @@ Rootfield id: - Commitment: `0x30055afe075a7c6ea8557ea3a2d3c7012d9d558ebda95803726179355f98ede9` +### Additional Swap-Memory Signal Observed By Reader + +The guarded canary reader also observed an earlier swap-memory signal emitted +by the hook adapter during smoke testing: + +- Transaction: `0x5f81dc48c5d172ff3f44a333a33598f23c82be2614f4156d5dd3257a16806cc7` +- Block: `45955507` +- FlowPulse pulse id: `0x2d436d766f9777b7f9925d57d8b2d57def3fdfae405017104f21795e20eacef7` +- Pulse type: `4` / `SWAP_MEMORY_SIGNAL` +- Commitment: + `0x30055afe075a7c6ea8557ea3a2d3c7012d9d558ebda95803726179355f98ede9` + ## State Readback `RootfieldRegistry.getRootfield(rootfieldId)` returned: @@ -84,19 +96,41 @@ rootCount: 1 active: true ``` +## Guarded Reader Command + +The repo now includes a guarded canary reader for these live V0 canary logs. It +requires explicit acknowledgement, explicit addresses, and a small explicit +block range: + +```powershell +npm run index:base-canary -- --acknowledge-mainnet-canary --rpc-url https://mainnet.base.org --address 0x2a7ADd68a1d45C3251E2F92fFe4926124654a97C --address 0x179Df6d52e9DeF5D02704583a2E4E5a9FF427245 --from-block 45955500 --to-block 45955540 --finalized-block 45955540 +``` + +Observed canary smoke read for the range above: + +- `RootfieldRegistry`: rootfield registration and root submission FlowPulse logs. +- `FlowMemoryHookAdapter`: two swap-memory signal FlowPulse logs. +- Observation count: `4`. +- Rejected log count: `0`. +- Duplicate count: `0`. +- Last indexed block: `45955535`. +- Output state: `services/indexer/out/base-canary-indexer-state.json` by default. +- Output checkpoint: `services/indexer/out/base-canary-indexer-checkpoint.json` by default. + +The canary reader refuses non-Base-mainnet RPC endpoints, refuses scans wider +than 5,000 blocks, stores no RPC URLs or private keys, and marks checkpoint +output as not production-ready. + ## Important Gaps Found -1. The checked-in live reader is Base Sepolia-only and intentionally rejects - Base mainnet chain id `8453`. A guarded Base canary reader is needed before - the dashboard can ingest live mainnet canary logs. -2. The dashboard still consumes generated fixtures. It does not yet ingest a +1. The dashboard still consumes generated fixtures. It does not yet ingest a deployment artifact plus live read output. -3. Contract source verification is not automated for all deployed contracts. -4. `FlowMemoryHookAdapter` is still an adapter scaffold. It is not a production +2. Contract source verification is not automated for all deployed contracts. +3. `FlowMemoryHookAdapter` is still an adapter scaffold. It is not a production Uniswap v4 hook wired into PoolManager permissions. -5. Ownership is still direct deployer ownership where applicable. There is no +4. Ownership is still direct deployer ownership where applicable. There is no multisig, governance, recovery, or operational key policy. -6. Verifier and worker registry flows are deployed, but live verifier report +5. Verifier and worker registry flows are deployed, but live verifier report submission, report signing, and verifier economics are not built. ## Notes diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d7cc48f8..5e39b264 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -65,8 +65,9 @@ Status: implemented as fixture-first services plus generated launch-core state; - Flow Memory schemas for MemorySignal, MemoryReceipt, RootfieldBundle, and AgentMemoryView exist under `schemas/flowmemory/`. - Generated MemorySignal and RootflowTransition fixtures expose contract-event linkage through `contractEvent` and `contractEventRef`. - Fixture-based parser and reorg-state tests exist in the indexer/verifier packages. -- Deterministic persistence exists for fixture state and the constrained Base Sepolia reader checkpoint. +- Deterministic persistence exists for fixture state, the constrained Base Sepolia reader checkpoint, and the guarded Base mainnet canary checkpoint. - A Base Sepolia reader path exists for explicit RPC URLs and explicit FlowPulse contract addresses; it rejects non-Base-Sepolia chain ids. +- A guarded Base mainnet canary reader exists for explicit RPC URLs, explicit known canary addresses, and small explicit block ranges; it rejects non-Base-mainnet chain ids and marks output as canary-only. - Base Sepolia deploy/read commands exist for the current V0 testnet contract set. - A Base mainnet V0 canary deployment has been performed for testing only and is documented under `docs/DEPLOYMENTS/`. - Runtime schema validation and generated fixture drift checks exist for launch-core outputs. @@ -142,7 +143,7 @@ The initial merge sequence has completed for repo OS, contracts foundation, cryp Next merge preference: -1. Guarded Base canary reader and deployment-artifact ingestion. +1. Deployment-artifact ingestion for the guarded Base canary reader output. 2. Base Sepolia reader soak tests against explicit testnet deployments. 3. Dashboard live/canary mode separation from generated fixtures. 4. Static analysis follow-up findings triaged for any public testnet deployment. diff --git a/package.json b/package.json index ef7ffe50..2e194da5 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "test": "npm test --prefix services/shared && npm test --prefix services/indexer && npm test --prefix services/verifier && npm test --prefix services/flowmemory", "contracts:hardening": "node infra/scripts/run-contract-hardening.mjs", "contracts:hardening:slither": "node infra/scripts/run-contract-hardening.mjs --require-slither", + "index:base-canary": "npm run index:base-canary --prefix services/indexer", + "read:base-canary": "npm run index:base-canary --prefix services/indexer", "index:base-sepolia": "npm run index:base-sepolia --prefix services/indexer", "read:base-sepolia": "npm run index:base-sepolia --prefix services/indexer", "deploy:base-sepolia": "node infra/scripts/run-base-sepolia-deploy.mjs", diff --git a/services/indexer/README.md b/services/indexer/README.md index c75685a1..1c20ff07 100644 --- a/services/indexer/README.md +++ b/services/indexer/README.md @@ -9,6 +9,7 @@ From the repository root: ```powershell npm run index:fixtures npm run index:base-sepolia -- --rpc-url --address --from-block --to-block +npm run index:base-canary -- --acknowledge-mainnet-canary --rpc-url --address --from-block --to-block npm run demo:indexer npm test --prefix services/indexer ``` @@ -177,6 +178,14 @@ flowmemory.indexer.base_sepolia_checkpoint.v0 The checkpoint records the network, chain id, emitting addresses, scan range, finality threshold, state path, counts, and latest indexed block. It intentionally does not store RPC URLs or private keys. +The Base mainnet canary reader writes: + +```text +flowmemory.indexer.base_canary_checkpoint.v0 +``` + +It is for the documented V0 canary deployment only. It requires `--acknowledge-mainnet-canary`, an explicit RPC URL, explicit emitting addresses, and an explicit block range. It rejects non-Base-mainnet endpoints and refuses scans wider than 5,000 blocks. The checkpoint marks `productionReady: false` and intentionally does not store RPC URLs or private keys. + The JSON schema fixture lives at: ```text @@ -189,4 +198,6 @@ services/indexer/fixtures/indexer-state.schema.json `readBaseSepoliaFlowPulseLogs` is the current live reader boundary. It requires an explicit RPC URL and refuses endpoints unless `eth_chainId` returns Base Sepolia (`84532`). It is not a Base mainnet reader and does not make production-mainnet readiness claims. +`readBaseMainnetCanaryFlowPulseLogs` is the narrow Base mainnet canary boundary. It requires an explicit RPC URL and refuses endpoints unless `eth_chainId` returns Base mainnet (`8453`). It is not a broad production indexer and should only be used against known canary contract addresses and small block ranges. + See [docs/INDEXER_VERIFIER_MVP.md](../../docs/INDEXER_VERIFIER_MVP.md) for the full pipeline. diff --git a/services/indexer/package.json b/services/indexer/package.json index 2ceedfd9..aa4fa97c 100644 --- a/services/indexer/package.json +++ b/services/indexer/package.json @@ -4,6 +4,7 @@ "type": "module", "scripts": { "demo": "node src/demo.ts", + "index:base-canary": "node src/base-canary.ts", "index:base-sepolia": "node src/base-sepolia.ts", "index:fixtures": "node src/index-fixtures.ts", "test": "node --test test/*.test.ts" diff --git a/services/indexer/src/base-canary.ts b/services/indexer/src/base-canary.ts new file mode 100644 index 00000000..cd7a2aa9 --- /dev/null +++ b/services/indexer/src/base-canary.ts @@ -0,0 +1,215 @@ +import { resolve } from "node:path"; + +import { indexFlowPulseLogs, type IndexerState } from "./indexer.ts"; +import { + baseCanaryIndexerCheckpoint, + type BaseCanaryIndexerCheckpoint, + writeBaseCanaryIndexerCheckpoint, + writeIndexerState, +} from "./persistence.ts"; +import { + blockArgumentToDecimalString, + blockArgumentToRpcQuantity, + normalizeEvmAddresses, + readArgValue, +} from "./reader-utils.ts"; +import { BASE_MAINNET_CHAIN_ID, readBaseMainnetCanaryFlowPulseLogs } from "./rpc.ts"; + +export const BASE_CANARY_MAX_BLOCK_SPAN = 5_000n; + +export interface BaseCanaryReaderOptions { + rpcUrl: string; + addresses: string[]; + fromBlock: string; + toBlock: string; + outPath?: string; + checkpointPath?: string; + finalizedBlockNumber?: string; + generatedAt?: string; + acknowledgeMainnetCanary?: boolean; + fetchImpl?: typeof fetch; +} + +export interface BaseCanaryReaderResult { + state: IndexerState; + checkpoint: BaseCanaryIndexerCheckpoint; + statePath: string; + checkpointPath: string; +} + +interface CliOptions extends BaseCanaryReaderOptions { + outPath: string; + checkpointPath: string; + acknowledgeMainnetCanary: true; +} + +function assertCanaryAcknowledged(acknowledgeMainnetCanary?: boolean): void { + if (acknowledgeMainnetCanary !== true) { + throw new Error("--acknowledge-mainnet-canary is required for the Base mainnet canary reader"); + } +} + +function assertCanaryBlockRange(fromBlock: string, toBlock: string): void { + if (BigInt(toBlock) < BigInt(fromBlock)) { + throw new Error("--to-block must be greater than or equal to --from-block"); + } + + const span = BigInt(toBlock) - BigInt(fromBlock); + if (span > BASE_CANARY_MAX_BLOCK_SPAN) { + throw new Error( + `Base canary reader refuses broad scans; block span ${span.toString()} exceeds ${BASE_CANARY_MAX_BLOCK_SPAN.toString()}`, + ); + } +} + +export function parseBaseCanaryReaderArgs(args: string[]): CliOptions { + let rpcUrl = ""; + let fromBlock = ""; + let toBlock = ""; + let finalizedBlockNumber: string | undefined; + let acknowledgeMainnetCanary = false; + const addresses: string[] = []; + let outPath = "out/base-canary-indexer-state.json"; + let checkpointPath = "out/base-canary-indexer-checkpoint.json"; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--acknowledge-mainnet-canary") { + acknowledgeMainnetCanary = true; + } else if (arg === "--rpc-url") { + rpcUrl = readArgValue(args, index, arg); + index += 1; + } else if (arg === "--address" || arg === "--addresses") { + addresses.push(readArgValue(args, index, arg)); + index += 1; + } else if (arg === "--from-block") { + fromBlock = readArgValue(args, index, arg); + index += 1; + } else if (arg === "--to-block") { + toBlock = readArgValue(args, index, arg); + index += 1; + } else if (arg === "--finalized-block") { + finalizedBlockNumber = blockArgumentToDecimalString(readArgValue(args, index, arg)); + index += 1; + } else if (arg === "--out") { + outPath = readArgValue(args, index, arg); + index += 1; + } else if (arg === "--checkpoint-out") { + checkpointPath = readArgValue(args, index, arg); + index += 1; + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + + if (rpcUrl.trim() === "") { + throw new Error("--rpc-url is required; FlowMemory does not ship a default RPC endpoint"); + } + if (fromBlock.trim() === "") { + throw new Error("--from-block is required"); + } + if (toBlock.trim() === "") { + throw new Error("--to-block is required"); + } + + assertCanaryAcknowledged(acknowledgeMainnetCanary); + + const normalizedFromBlock = blockArgumentToDecimalString(fromBlock); + const normalizedToBlock = blockArgumentToDecimalString(toBlock); + assertCanaryBlockRange(normalizedFromBlock, normalizedToBlock); + + return { + rpcUrl, + addresses: normalizeEvmAddresses(addresses), + fromBlock: normalizedFromBlock, + toBlock: normalizedToBlock, + finalizedBlockNumber, + outPath, + checkpointPath, + acknowledgeMainnetCanary: true, + }; +} + +export async function runBaseCanaryReader(options: BaseCanaryReaderOptions): Promise { + assertCanaryAcknowledged(options.acknowledgeMainnetCanary); + + const addresses = normalizeEvmAddresses(options.addresses); + const fromBlock = blockArgumentToDecimalString(options.fromBlock); + const toBlock = blockArgumentToDecimalString(options.toBlock); + const outPath = resolve(options.outPath ?? "out/base-canary-indexer-state.json"); + const checkpointPath = resolve(options.checkpointPath ?? "out/base-canary-indexer-checkpoint.json"); + + assertCanaryBlockRange(fromBlock, toBlock); + + const readResult = await readBaseMainnetCanaryFlowPulseLogs({ + rpcUrl: options.rpcUrl, + addresses, + fromBlock: blockArgumentToRpcQuantity(fromBlock), + toBlock: blockArgumentToRpcQuantity(toBlock), + fetchImpl: options.fetchImpl, + }); + + const finalizedBlockNumber = options.finalizedBlockNumber === undefined + ? undefined + : blockArgumentToDecimalString(options.finalizedBlockNumber); + + const state = indexFlowPulseLogs(readResult.logs, { + chainId: BASE_MAINNET_CHAIN_ID, + finalizedBlockNumber, + source: "base-mainnet-canary-rpc", + sourceAddresses: addresses, + }); + const checkpoint = baseCanaryIndexerCheckpoint({ + addresses, + fromBlock, + toBlock, + finalizedBlockNumber, + statePath: outPath, + state, + generatedAt: options.generatedAt, + }); + + writeIndexerState(outPath, state); + writeBaseCanaryIndexerCheckpoint(checkpointPath, checkpoint); + + return { + state, + checkpoint, + statePath: outPath, + checkpointPath, + }; +} + +function usage(): string { + return [ + "Usage:", + " node src/base-canary.ts --acknowledge-mainnet-canary --rpc-url --address <0x...> --from-block --to-block [--finalized-block ] [--out ] [--checkpoint-out ]", + "", + "Boundary:", + ` This reader only accepts Base mainnet chainId ${BASE_MAINNET_CHAIN_ID} and is canary-only.`, + ` It refuses scans wider than ${BASE_CANARY_MAX_BLOCK_SPAN.toString()} blocks and stores no RPC URLs or keys.`, + ].join("\n"); +} + +if (process.argv[1]?.replaceAll("\\", "/").endsWith("/base-canary.ts")) { + runBaseCanaryReader(parseBaseCanaryReaderArgs(process.argv.slice(2))) + .then((result) => { + console.log(JSON.stringify({ + schema: "flowmemory.indexer.base_canary_reader_summary.v0", + network: result.checkpoint.network, + chainId: result.checkpoint.chainId, + statePath: result.statePath, + checkpointPath: result.checkpointPath, + observationCount: result.checkpoint.observationCount, + rejectedLogCount: result.checkpoint.rejectedLogCount, + duplicateCount: result.checkpoint.duplicateCount, + lastIndexedBlock: result.checkpoint.lastIndexedBlock, + productionReady: result.checkpoint.safety.productionReady, + }, null, 2)); + }) + .catch((error) => { + console.error(error instanceof Error ? error.message : error); + console.error(usage()); + process.exitCode = 1; + }); +} diff --git a/services/indexer/src/base-sepolia.ts b/services/indexer/src/base-sepolia.ts index d45d2952..ca8787f5 100644 --- a/services/indexer/src/base-sepolia.ts +++ b/services/indexer/src/base-sepolia.ts @@ -7,8 +7,16 @@ import { writeBaseSepoliaIndexerCheckpoint, writeIndexerState, } from "./persistence.ts"; +import { + blockArgumentToDecimalString, + blockArgumentToRpcQuantity, + normalizeEvmAddresses, + readArgValue, +} from "./reader-utils.ts"; import { BASE_SEPOLIA_CHAIN_ID, readBaseSepoliaFlowPulseLogs } from "./rpc.ts"; +export { blockArgumentToDecimalString, blockArgumentToRpcQuantity } from "./reader-utils.ts"; + export interface BaseSepoliaReaderOptions { rpcUrl: string; addresses: string[]; @@ -33,46 +41,6 @@ interface CliOptions extends BaseSepoliaReaderOptions { checkpointPath: string; } -function normalizeAddress(address: string): string { - const normalized = address.trim().toLowerCase(); - if (!/^0x[0-9a-f]{40}$/.test(normalized)) { - throw new Error(`invalid EVM address: ${address}`); - } - return normalized; -} - -function normalizeAddresses(addresses: string[]): string[] { - const normalized = addresses.flatMap((entry) => entry.split(",")).map(normalizeAddress); - const unique = [...new Set(normalized)].sort((left, right) => left.localeCompare(right)); - if (unique.length === 0) { - throw new Error("at least one FlowPulse contract address is required"); - } - return unique; -} - -export function blockArgumentToDecimalString(value: string): string { - const trimmed = value.trim().toLowerCase(); - if (/^0x[0-9a-f]+$/.test(trimmed)) { - return BigInt(trimmed).toString(); - } - if (/^[0-9]+$/.test(trimmed)) { - return BigInt(trimmed).toString(); - } - throw new Error(`block value must be a decimal or 0x quantity, received: ${value}`); -} - -export function blockArgumentToRpcQuantity(value: string): string { - return `0x${BigInt(blockArgumentToDecimalString(value)).toString(16)}`; -} - -function readArgValue(args: string[], index: number, name: string): string { - const value = args[index + 1]; - if (value === undefined || value.startsWith("--")) { - throw new Error(`${name} requires a value`); - } - return value; -} - export function parseBaseSepoliaReaderArgs(args: string[]): CliOptions { let rpcUrl = ""; let fromBlock = ""; @@ -122,7 +90,7 @@ export function parseBaseSepoliaReaderArgs(args: string[]): CliOptions { return { rpcUrl, - addresses: normalizeAddresses(addresses), + addresses: normalizeEvmAddresses(addresses), fromBlock: blockArgumentToDecimalString(fromBlock), toBlock: blockArgumentToDecimalString(toBlock), finalizedBlockNumber, @@ -132,7 +100,7 @@ export function parseBaseSepoliaReaderArgs(args: string[]): CliOptions { } export async function runBaseSepoliaReader(options: BaseSepoliaReaderOptions): Promise { - const addresses = normalizeAddresses(options.addresses); + const addresses = normalizeEvmAddresses(options.addresses); const fromBlock = blockArgumentToDecimalString(options.fromBlock); const toBlock = blockArgumentToDecimalString(options.toBlock); const outPath = resolve(options.outPath ?? "out/base-sepolia-indexer-state.json"); diff --git a/services/indexer/src/indexer.ts b/services/indexer/src/indexer.ts index 0a36db76..9d4ebf75 100644 --- a/services/indexer/src/indexer.ts +++ b/services/indexer/src/indexer.ts @@ -71,7 +71,7 @@ export interface IndexedRootfield { pulseCount: number; } -export type IndexerStateSource = "fixture" | "local-rpc-placeholder" | "base-sepolia-rpc"; +export type IndexerStateSource = "fixture" | "local-rpc-placeholder" | "base-sepolia-rpc" | "base-mainnet-canary-rpc"; export interface IndexerState { schema: "flowmemory.indexer.state.v0"; diff --git a/services/indexer/src/persistence.ts b/services/indexer/src/persistence.ts index 765d5138..a40fe53c 100644 --- a/services/indexer/src/persistence.ts +++ b/services/indexer/src/persistence.ts @@ -27,6 +27,28 @@ export interface BaseSepoliaIndexerCheckpoint { generatedAt: string; } +export interface BaseCanaryIndexerCheckpoint { + schema: "flowmemory.indexer.base_canary_checkpoint.v0"; + network: "base-mainnet-canary"; + chainId: "8453"; + source: "base-mainnet-canary-rpc"; + addresses: string[]; + fromBlock: string; + toBlock: string; + finalizedBlockNumber?: string; + statePath: string; + observationCount: number; + cursorCount: number; + rejectedLogCount: number; + duplicateCount: number; + lastIndexedBlock: string; + generatedAt: string; + safety: { + acknowledgement: "base-mainnet-canary-only"; + productionReady: false; + }; +} + export function persistedIndexerState(state: IndexerState): PersistedIndexerState { return { schema: "flowmemory.indexer.persistence.v0", @@ -66,6 +88,42 @@ export function baseSepoliaIndexerCheckpoint(input: { }; } +export function baseCanaryIndexerCheckpoint(input: { + addresses: string[]; + fromBlock: string; + toBlock: string; + finalizedBlockNumber?: string; + statePath: string; + state: IndexerState; + generatedAt?: string; +}): BaseCanaryIndexerCheckpoint { + const lastIndexedBlock = input.state.cursors.reduce((latest, cursor) => { + return BigInt(cursor.blockNumber) > BigInt(latest) ? cursor.blockNumber : latest; + }, input.fromBlock); + + return { + schema: "flowmemory.indexer.base_canary_checkpoint.v0", + network: "base-mainnet-canary", + chainId: "8453", + source: "base-mainnet-canary-rpc", + addresses: [...input.addresses].sort((left, right) => left.localeCompare(right)), + fromBlock: input.fromBlock, + toBlock: input.toBlock, + finalizedBlockNumber: input.finalizedBlockNumber, + statePath: input.statePath, + observationCount: input.state.observations.length, + cursorCount: input.state.cursors.length, + rejectedLogCount: input.state.rejectedLogs.length, + duplicateCount: input.state.duplicates.length, + lastIndexedBlock, + generatedAt: input.generatedAt ?? new Date().toISOString(), + safety: { + acknowledgement: "base-mainnet-canary-only", + productionReady: false, + }, + }; +} + export function writeIndexerState(path: string, state: IndexerState): void { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, `${canonicalJson(persistedIndexerState(state))}\n`, "utf8"); @@ -83,3 +141,12 @@ export function writeBaseSepoliaIndexerCheckpoint(path: string, checkpoint: Base export function readBaseSepoliaIndexerCheckpoint(path: string): BaseSepoliaIndexerCheckpoint { return JSON.parse(readFileSync(path, "utf8")) as BaseSepoliaIndexerCheckpoint; } + +export function writeBaseCanaryIndexerCheckpoint(path: string, checkpoint: BaseCanaryIndexerCheckpoint): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${canonicalJson(checkpoint)}\n`, "utf8"); +} + +export function readBaseCanaryIndexerCheckpoint(path: string): BaseCanaryIndexerCheckpoint { + return JSON.parse(readFileSync(path, "utf8")) as BaseCanaryIndexerCheckpoint; +} diff --git a/services/indexer/src/reader-utils.ts b/services/indexer/src/reader-utils.ts new file mode 100644 index 00000000..621385cd --- /dev/null +++ b/services/indexer/src/reader-utils.ts @@ -0,0 +1,39 @@ +export function normalizeEvmAddress(address: string): string { + const normalized = address.trim().toLowerCase(); + if (!/^0x[0-9a-f]{40}$/.test(normalized)) { + throw new Error(`invalid EVM address: ${address}`); + } + return normalized; +} + +export function normalizeEvmAddresses(addresses: string[]): string[] { + const normalized = addresses.flatMap((entry) => entry.split(",")).map(normalizeEvmAddress); + const unique = [...new Set(normalized)].sort((left, right) => left.localeCompare(right)); + if (unique.length === 0) { + throw new Error("at least one FlowPulse contract address is required"); + } + return unique; +} + +export function blockArgumentToDecimalString(value: string): string { + const trimmed = value.trim().toLowerCase(); + if (/^0x[0-9a-f]+$/.test(trimmed)) { + return BigInt(trimmed).toString(); + } + if (/^[0-9]+$/.test(trimmed)) { + return BigInt(trimmed).toString(); + } + throw new Error(`block value must be a decimal or 0x quantity, received: ${value}`); +} + +export function blockArgumentToRpcQuantity(value: string): string { + return `0x${BigInt(blockArgumentToDecimalString(value)).toString(16)}`; +} + +export function readArgValue(args: string[], index: number, name: string): string { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`${name} requires a value`); + } + return value; +} diff --git a/services/indexer/src/rpc.ts b/services/indexer/src/rpc.ts index 487b133f..9be21dc8 100644 --- a/services/indexer/src/rpc.ts +++ b/services/indexer/src/rpc.ts @@ -1,5 +1,6 @@ import { FLOWPULSE_EVENT_TOPIC0, type RawFlowPulseLogFixture } from "../../shared/src/index.ts"; +export const BASE_MAINNET_CHAIN_ID = "8453"; export const BASE_SEPOLIA_CHAIN_ID = "84532"; export interface LocalRpcReadOptions { @@ -133,3 +134,12 @@ export async function readBaseSepoliaFlowPulseLogs(options: LocalRpcReadOptions) } return readRpcFlowPulseLogSetWithChainId(options, chainId, fetchImpl); } + +export async function readBaseMainnetCanaryFlowPulseLogs(options: LocalRpcReadOptions): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const chainId = await readRpcChainId(fetchImpl, options.rpcUrl); + if (chainId !== BASE_MAINNET_CHAIN_ID) { + throw new Error(`expected Base mainnet chainId ${BASE_MAINNET_CHAIN_ID}, received ${chainId}`); + } + return readRpcFlowPulseLogSetWithChainId(options, chainId, fetchImpl); +} diff --git a/services/indexer/test/indexer.test.ts b/services/indexer/test/indexer.test.ts index ce331b36..17aaba92 100644 --- a/services/indexer/test/indexer.test.ts +++ b/services/indexer/test/indexer.test.ts @@ -4,11 +4,21 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; +import { parseBaseCanaryReaderArgs, runBaseCanaryReader } from "../src/base-canary.ts"; import { parseBaseSepoliaReaderArgs, runBaseSepoliaReader } from "../src/base-sepolia.ts"; import { indexFlowPulseLogs, indexFlowPulseReceipts } from "../src/indexer.ts"; import { loadIndexerFixtureLogs, loadIndexerFixtureReceipts } from "../src/fixtures.ts"; -import { readBaseSepoliaIndexerCheckpoint, readIndexerState, writeIndexerState } from "../src/persistence.ts"; -import { readBaseSepoliaFlowPulseLogs, readLocalRpcFlowPulseLogs } from "../src/rpc.ts"; +import { + readBaseCanaryIndexerCheckpoint, + readBaseSepoliaIndexerCheckpoint, + readIndexerState, + writeIndexerState, +} from "../src/persistence.ts"; +import { + readBaseMainnetCanaryFlowPulseLogs, + readBaseSepoliaFlowPulseLogs, + readLocalRpcFlowPulseLogs, +} from "../src/rpc.ts"; test("indexes FlowPulse fixture logs into canonical observations", () => { const state = indexFlowPulseLogs(loadIndexerFixtureLogs()); @@ -167,6 +177,34 @@ test("rejects non Base Sepolia RPC endpoints for live reads", async () => { assert.deepEqual(calls, ["eth_chainId"]); }); +test("rejects non Base mainnet RPC endpoints for canary reads", async () => { + const [fixtureLog] = loadIndexerFixtureLogs(); + const calls: string[] = []; + const fetchImpl = async (_url: string, init?: RequestInit): Promise => { + const body = JSON.parse(String(init?.body)) as { method: string }; + calls.push(body.method); + if (body.method === "eth_chainId") { + return Response.json({ jsonrpc: "2.0", id: 1, result: "0x14a34" }); + } + if (body.method === "eth_getLogs") { + return Response.json({ jsonrpc: "2.0", id: 1, result: [] }); + } + return Response.json({ jsonrpc: "2.0", id: 1, error: { code: -32601, message: "not found" } }); + }; + + await assert.rejects( + () => readBaseMainnetCanaryFlowPulseLogs({ + rpcUrl: "https://example.invalid", + addresses: [fixtureLog.address], + fromBlock: "0x1", + toBlock: "0x2", + fetchImpl, + }), + /expected Base mainnet chainId 8453, received 84532/, + ); + assert.deepEqual(calls, ["eth_chainId"]); +}); + test("runs Base Sepolia reader and persists durable state plus checkpoint", async () => { const [fixtureLog] = loadIndexerFixtureLogs(); const dir = mkdtempSync(join(tmpdir(), "flowmemory-base-sepolia-")); @@ -228,6 +266,173 @@ test("runs Base Sepolia reader and persists durable state plus checkpoint", asyn } }); +test("runs Base mainnet canary reader and persists empty scans safely", async () => { + const [fixtureLog] = loadIndexerFixtureLogs(); + const dir = mkdtempSync(join(tmpdir(), "flowmemory-base-canary-empty-")); + const statePath = join(dir, "state.json"); + const checkpointPath = join(dir, "checkpoint.json"); + const calls: string[] = []; + const fetchImpl = async (_url: string, init?: RequestInit): Promise => { + const body = JSON.parse(String(init?.body)) as { method: string }; + calls.push(body.method); + if (body.method === "eth_chainId") { + return Response.json({ jsonrpc: "2.0", id: 1, result: "0x2105" }); + } + if (body.method === "eth_getLogs") { + return Response.json({ jsonrpc: "2.0", id: 1, result: [] }); + } + return Response.json({ jsonrpc: "2.0", id: 1, error: { code: -32601, message: "not found" } }); + }; + + try { + const result = await runBaseCanaryReader({ + acknowledgeMainnetCanary: true, + rpcUrl: "https://example.invalid", + addresses: [fixtureLog.address], + fromBlock: "45955500", + toBlock: "45955500", + finalizedBlockNumber: "45955500", + outPath: statePath, + checkpointPath, + generatedAt: "2026-05-13T00:00:00.000Z", + fetchImpl, + }); + const persisted = readIndexerState(statePath); + const checkpoint = readBaseCanaryIndexerCheckpoint(checkpointPath); + + assert.deepEqual(calls, ["eth_chainId", "eth_getLogs"]); + assert.equal(result.state.source, "base-mainnet-canary-rpc"); + assert.equal(persisted.state.source, "base-mainnet-canary-rpc"); + assert.equal(checkpoint.schema, "flowmemory.indexer.base_canary_checkpoint.v0"); + assert.equal(checkpoint.network, "base-mainnet-canary"); + assert.equal(checkpoint.chainId, "8453"); + assert.equal(checkpoint.safety.productionReady, false); + assert.equal(checkpoint.observationCount, 0); + assert.equal(checkpoint.rejectedLogCount, 0); + assert.equal(checkpoint.lastIndexedBlock, "45955500"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("Base mainnet canary reader surfaces malformed logs without dropping checkpoints", async () => { + const [fixtureLog] = loadIndexerFixtureLogs(); + const dir = mkdtempSync(join(tmpdir(), "flowmemory-base-canary-malformed-")); + const statePath = join(dir, "state.json"); + const checkpointPath = join(dir, "checkpoint.json"); + const fetchImpl = async (_url: string, init?: RequestInit): Promise => { + const body = JSON.parse(String(init?.body)) as { method: string }; + if (body.method === "eth_chainId") { + return Response.json({ jsonrpc: "2.0", id: 1, result: "0x2105" }); + } + if (body.method === "eth_getLogs") { + return Response.json({ + jsonrpc: "2.0", + id: 1, + result: [{ + address: fixtureLog.address, + topics: fixtureLog.topics.slice(0, 3), + data: fixtureLog.data, + blockNumber: "0x2bd270c", + blockHash: fixtureLog.blockHash, + transactionHash: fixtureLog.transactionHash, + transactionIndex: "0x0", + logIndex: "0x0", + }], + }); + } + if (body.method === "eth_getTransactionReceipt") { + return Response.json({ jsonrpc: "2.0", id: 1, result: { status: "0x1" } }); + } + return Response.json({ jsonrpc: "2.0", id: 1, error: { code: -32601, message: "not found" } }); + }; + + try { + const result = await runBaseCanaryReader({ + acknowledgeMainnetCanary: true, + rpcUrl: "https://example.invalid", + addresses: [fixtureLog.address], + fromBlock: "45950732", + toBlock: "45950732", + outPath: statePath, + checkpointPath, + generatedAt: "2026-05-13T00:00:00.000Z", + fetchImpl, + }); + const checkpoint = readBaseCanaryIndexerCheckpoint(checkpointPath); + + assert.equal(result.state.observations.length, 0); + assert.equal(result.state.rejectedLogs.length, 1); + assert.equal(result.state.rejectedLogs[0].reasonCode, "log.malformed"); + assert.equal(checkpoint.rejectedLogCount, 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("Base mainnet canary reader preserves exact duplicates and reorg replacements", async () => { + const [fixtureLog] = loadIndexerFixtureLogs(); + const dir = mkdtempSync(join(tmpdir(), "flowmemory-base-canary-duplicates-")); + const statePath = join(dir, "state.json"); + const checkpointPath = join(dir, "checkpoint.json"); + const reorgLog = { + address: fixtureLog.address, + topics: fixtureLog.topics, + data: fixtureLog.data, + blockNumber: "0x2bd270d", + blockHash: "0x1111111111111111111111111111111111111111111111111111111111111111", + transactionHash: "0x2222222222222222222222222222222222222222222222222222222222222222", + transactionIndex: "0x1", + logIndex: "0x0", + }; + const baseLog = { + address: fixtureLog.address, + topics: fixtureLog.topics, + data: fixtureLog.data, + blockNumber: "0x2bd270c", + blockHash: fixtureLog.blockHash, + transactionHash: fixtureLog.transactionHash, + transactionIndex: "0x0", + logIndex: "0x0", + }; + const fetchImpl = async (_url: string, init?: RequestInit): Promise => { + const body = JSON.parse(String(init?.body)) as { method: string }; + if (body.method === "eth_chainId") { + return Response.json({ jsonrpc: "2.0", id: 1, result: "0x2105" }); + } + if (body.method === "eth_getLogs") { + return Response.json({ jsonrpc: "2.0", id: 1, result: [baseLog, baseLog, reorgLog] }); + } + if (body.method === "eth_getTransactionReceipt") { + return Response.json({ jsonrpc: "2.0", id: 1, result: { status: "0x1" } }); + } + return Response.json({ jsonrpc: "2.0", id: 1, error: { code: -32601, message: "not found" } }); + }; + + try { + const result = await runBaseCanaryReader({ + acknowledgeMainnetCanary: true, + rpcUrl: "https://example.invalid", + addresses: [fixtureLog.address], + fromBlock: "45950732", + toBlock: "45950733", + outPath: statePath, + checkpointPath, + generatedAt: "2026-05-13T00:00:00.000Z", + fetchImpl, + }); + + assert.equal(result.state.observations.length, 3); + assert.deepEqual(result.state.duplicates.map((duplicate) => duplicate.kind), [ + "exactDuplicate", + "reorgReplacement", + ]); + assert.equal(result.checkpoint.duplicateCount, 2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test("parses Base Sepolia reader CLI args without defaulting to a public RPC", () => { const [fixtureLog] = loadIndexerFixtureLogs(); const parsed = parseBaseSepoliaReaderArgs([ @@ -252,3 +457,53 @@ test("parses Base Sepolia reader CLI args without defaulting to a public RPC", ( /--rpc-url is required/, ); }); + +test("parses guarded Base mainnet canary reader CLI args", () => { + const [fixtureLog] = loadIndexerFixtureLogs(); + const parsed = parseBaseCanaryReaderArgs([ + "--acknowledge-mainnet-canary", + "--rpc-url", + "https://example.invalid", + "--addresses", + `${fixtureLog.address},${fixtureLog.address.toUpperCase()}`, + "--from-block", + "0x2bd270c", + "--to-block", + "45950740", + "--finalized-block", + "0x2bd270c", + ]); + + assert.equal(parsed.fromBlock, "45950732"); + assert.equal(parsed.toBlock, "45950740"); + assert.equal(parsed.finalizedBlockNumber, "45950732"); + assert.equal(parsed.addresses.length, 1); + assert.equal(parsed.acknowledgeMainnetCanary, true); + assert.throws( + () => parseBaseCanaryReaderArgs([ + "--rpc-url", + "https://example.invalid", + "--address", + fixtureLog.address, + "--from-block", + "1", + "--to-block", + "2", + ]), + /--acknowledge-mainnet-canary is required/, + ); + assert.throws( + () => parseBaseCanaryReaderArgs([ + "--acknowledge-mainnet-canary", + "--rpc-url", + "https://example.invalid", + "--address", + fixtureLog.address, + "--from-block", + "1", + "--to-block", + "5002", + ]), + /refuses broad scans/, + ); +});