Prepare nitro-test node for txn filter reporting feature - NIT-4675

docker-compose.yaml

@@ -511,6 +511,39 @@ services:
     depends_on:
       - sequencer
 
+  elasticmq:
+    image: softwaremill/elasticmq-native:1.6.9
+    ports:
+      - "127.0.0.1:9324:9324"
+    volumes:
+      - "config:/config"
+    command: ["-Dconfig.file=/config/elasticmq.conf"]
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O /dev/null http://localhost:9325 || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  filtering-report:
+    pid: host
+    image: nitro-node-dev-testnode
+    entrypoint: /usr/local/bin/filtering-report
+    ports:
+      - "127.0.0.1:8550:8547"
+    volumes:
+      - "config:/config"
+    command:
+      - --conf.file=/config/filtering_report_config.json
+    depends_on:
+      - elasticmq
+      - sequencer
+
+  report-receiver:
+    build: scripts/
+    command: ["serve-report-receiver"]
+    ports:
+      - "127.0.0.1:9081:8080"
+
 volumes:
   l1data:
   consensus:

scripts/config.ts

@@ -211,7 +211,7 @@ function createDataAvailabilityConfig(argv: any, anytrust: boolean) {
 }
 
 function applyTxFilteringConfig(config: any) {
-    config.execution.sequencer["transaction-filtering"] = {
+    config.execution["transaction-filtering"] = {
         "address-filter": {
             "enable": true,
             "s3": {
@@ -387,6 +387,9 @@ function writeConfigs(argv: any) {
         if (argv.txfiltering) {
             applyTxFilteringConfig(simpleConfig);
         }
+        if (argv.filteringreport) {
+            applyFilteringReportConfig(simpleConfig);
+        }
         fs.writeFileSync(path.join(consts.configpath, "sequencer_config.json"), JSON.stringify(simpleConfig))
     } else {
         let validatorConfig = JSON.parse(baseConfJSON)
@@ -413,6 +416,9 @@ function writeConfigs(argv: any) {
         if (argv.txfiltering) {
             applyTxFilteringConfig(sequencerConfig);
         }
+        if (argv.filteringreport) {
+            applyFilteringReportConfig(sequencerConfig);
+        }
         fs.writeFileSync(path.join(consts.configpath, "sequencer_config.json"), JSON.stringify(sequencerConfig))
 
         let posterConfig = JSON.parse(baseConfJSON)
@@ -740,8 +746,16 @@ export const writeConfigCommand = {
             describe: "enable transaction filtering mode",
             default: false
         },
+        filteringreport: {
+            boolean: true,
+            describe: "enable filtering report framework",
+            default: false
+        },
     },
     handler: (argv: any) => {
+        if (argv.filteringreport) {
+            argv.txfiltering = true;
+        }
         writeConfigs(argv)
     }
 }
@@ -873,8 +887,10 @@ export const initTxFilteringMinioCommand = {
     command: "init-tx-filtering-minio",
     describe: "initializes MinIO bucket and empty address hash list",
     handler: async () => {
+        const id = crypto.randomUUID();
         const salt = crypto.randomUUID();
         const initialAddressList = {
+            "id": id,
             "salt": salt,
             "hashing_scheme": "Sha256",
             "address_hashes": []
@@ -1014,3 +1030,137 @@ export const removeFilteredAddressCommand = {
         }
     }
 }
+
+function writeElasticMQConfig() {
+    const elasticmqConf = `include classpath("application.conf")
+
+node-address {
+  protocol = http
+  host = "*"
+  port = 9324
+  context-path = ""
+}
+
+rest-sqs {
+  enabled = true
+  bind-port = 9324
+  bind-hostname = "0.0.0.0"
+  sqs-limits = strict
+}
+
+queues {
+  filtering-reports {
+    defaultVisibilityTimeout = 30 seconds
+    delay = 0 seconds
+    receiveMessageWait = 0 seconds
+  }
+}
+`;
+    fs.writeFileSync(path.join(consts.configpath, "elasticmq.conf"), elasticmqConf);
+}
+
+export const writeElasticMQConfigCommand = {
+    command: "write-elasticmq-config",
+    describe: "writes ElasticMQ config file for SQS emulation",
+    handler: () => {
+        writeElasticMQConfig();
+    }
+}
+
+function writeFilteringReportConfig() {
+    const config = {
+        "http": {
+            "addr": "0.0.0.0",
+            "port": 8547,
+            "vhosts": "*",
+            "corsdomain": "*",
+            "api": ["filteringreport"]
+        },
+        "ws": {
+            "addr": "0.0.0.0",
+            "port": 8548
+        },
+        "queue": {
+            "queue-url": "http://elasticmq:9324/000000000000/filtering-reports",
+            "sqs-client": {
+                "region": "us-east-1",
+                "endpoint": "http://elasticmq:9324",
+                "access-key": "elasticmq",
+                "secret-key": "elasticmq"
+            }
+        },
+        "report-forwarder": {
+            "workers": 1,
+            "poll-interval": "5s",
+            "sqs-wait-time-seconds": 5,
+            "external-endpoint": {
+                "url": "http://report-receiver:8080",
+                "timeout": "10s"
+            }
+        }
+    };
+    fs.writeFileSync(
+        path.join(consts.configpath, "filtering_report_config.json"),
+        JSON.stringify(config)
+    );
+}
+
+export const writeFilteringReportConfigCommand = {
+    command: "write-filtering-report-config",
+    describe: "writes filtering-report service config file",
+    handler: () => {
+        writeFilteringReportConfig();
+    }
+}
+
+function applyFilteringReportConfig(config: any) {
+    if (!config.execution["transaction-filtering"]) {
+        config.execution["transaction-filtering"] = {};
+    }
+    config.execution["transaction-filtering"]["filtering-report-rpc-client"] = {
+        "url": "http://filtering-report:8547"
+    };
+}
+
+export const serveReportReceiverCommand = {
+    command: "serve-report-receiver",
+    describe: "starts an HTTP server that receives and logs filtering reports",
+    handler: async () => {
+        const http = require('http');
+        const reports: any[] = [];
+        const server = http.createServer((req: any, res: any) => {
+            if (req.method === 'POST') {
+                let body = '';
+                req.on('data', (chunk: string) => body += chunk);
+                req.on('end', () => {
+                    console.log('Received report:', body);
+                    try {
+                        reports.push(JSON.parse(body));
+                        res.writeHead(200, {'Content-Type': 'application/json'});
+                        res.end(JSON.stringify({status: 'ok'}));
+                    } catch (err) {
+                        console.error('Failed to parse report body:', err);
+                        res.writeHead(400, {'Content-Type': 'application/json'});
+                        res.end(JSON.stringify({status: 'error', message: 'invalid JSON'}));
+                    }
+                });
+            } else if (req.method === 'GET' && req.url === '/reports') {
+                res.writeHead(200, {'Content-Type': 'application/json'});
+                res.end(JSON.stringify(reports));
+            } else {
+                res.writeHead(200);
+                res.end('OK');
+            }
+        });
+        server.listen(8080, '0.0.0.0', () => {
+            console.log('Report receiver listening on :8080');
+        });
+        // Handler must be async and await a never-resolving promise so yargs
+        // does not return early. Without this, index.ts's
+        // `main().then(() => process.exit(0))` would terminate the process
+        // immediately after server.listen() starts, killing the server.
+        // SIGINT / SIGTERM from `docker compose down` will still terminate
+        // the container cleanly.
+        await new Promise<void>(() => {});
+    }
+}

scripts/index.ts

@@ -18,6 +18,9 @@ import {
   hashAddressCommand,
   addFilteredAddressCommand,
   removeFilteredAddressCommand,
+  writeElasticMQConfigCommand,
+  writeFilteringReportConfigCommand,
+  serveReportReceiverCommand,
 } from "./config";
 import {
   printAddressCommand,
@@ -88,6 +91,9 @@ async function main() {
     .command(hashAddressCommand)
     .command(addFilteredAddressCommand)
     .command(removeFilteredAddressCommand)
+    .command(writeElasticMQConfigCommand)
+    .command(writeFilteringReportConfigCommand)
+    .command(serveReportReceiverCommand)
     .command(grantFiltererRoleCommand)
     .command(printAddressCommand)
     .command(printPrivateKeyCommand)
@@ -97,7 +103,8 @@ async function main() {
     .strict()
     .demandCommand(1, "a command must be specified")
     .epilogue(namedAccountHelpString)
-    .help().argv;
+    .help()
+    .parseAsync();
 }
 
 main()

test-node.bash

@@ -63,6 +63,7 @@ l2anytrust=false
 l2referenceda=false
 l2timeboost=false
 l2txfiltering=false
+l2filteringreport=false
 
 # Use the dev versions of nitro/blockscout
 dev_nitro=false
@@ -280,6 +281,11 @@ while [[ $# -gt 0 ]]; do
             l2txfiltering=true
             shift
             ;;
+        --l2-filtering-report)
+            l2filteringreport=true
+            l2txfiltering=true
+            shift
+            ;;
         --redundantsequencers)
             simple=false
             redundantsequencers=$2
@@ -326,6 +332,7 @@ while [[ $# -gt 0 ]]; do
             echo --l2-referenceda  run the L2 with reference external data availability provider
             echo --l2-timeboost    run the L2 with Timeboost enabled, including auctioneer and bid validator
             echo --l2-tx-filtering  run the L2 with transaction filtering enabled
+            echo --l2-filtering-report  run the L2 with filtering report framework enabled \(implies --l2-tx-filtering\)
             echo --batchposters    batch posters [0-3]
             echo --redundantsequencers redundant sequencers [0-3]
             echo --detach          detach from nodes after running them
@@ -402,6 +409,10 @@ if $l2txfiltering; then
     NODES="$NODES minio transaction-filterer"
 fi
 
+if $l2filteringreport; then
+    NODES="$NODES elasticmq filtering-report report-receiver"
+fi
+
 if $dev_nitro && $build_dev_nitro; then
   echo == Building Nitro
   if ! [ -n "${NITRO_SRC+set}" ]; then
@@ -512,6 +523,14 @@ if $force_init; then
         run_script init-tx-filtering-minio
     fi
 
+    if $l2filteringreport; then
+        echo == Writing ElasticMQ config
+        run_script write-elasticmq-config
+
+        echo == Starting ElasticMQ
+        docker compose up --wait elasticmq
+    fi
+
     echo == Funding validator, sequencer and l2owner
     run_script send-l1 --ethamount 1000 --to validator --wait
     run_script send-l1 --ethamount 1000 --to sequencer --wait
@@ -568,6 +587,7 @@ anytrustNodeConfigLine=""
 referenceDaNodeConfigLine=""
 timeboostNodeConfigLine=""
 txFilteringNodeConfigLine=""
+filteringReportNodeConfigLine=""
 
 # Remaining init may require AnyTrust committee/mirrors to have been started
 if $l2anytrust; then
@@ -611,12 +631,15 @@ if $force_init; then
     if $l2txfiltering; then
         txFilteringNodeConfigLine="--txfiltering"
     fi
+    if $l2filteringreport; then
+        filteringReportNodeConfigLine="--filteringreport"
+    fi
 
     echo "== Writing configs"
     if $simple; then
-        run_script write-config --simple $anytrustNodeConfigLine $referenceDaNodeConfigLine $timeboostNodeConfigLine $txFilteringNodeConfigLine
+        run_script write-config --simple $anytrustNodeConfigLine $referenceDaNodeConfigLine $timeboostNodeConfigLine $txFilteringNodeConfigLine $filteringReportNodeConfigLine
     else
-        run_script write-config $anytrustNodeConfigLine $referenceDaNodeConfigLine $timeboostNodeConfigLine $txFilteringNodeConfigLine
+        run_script write-config $anytrustNodeConfigLine $referenceDaNodeConfigLine $timeboostNodeConfigLine $txFilteringNodeConfigLine $filteringReportNodeConfigLine
 
         echo == Initializing redis
         docker compose up --wait redis
@@ -661,6 +684,14 @@ if $force_init; then
         run_script write-tx-filterer-config
     fi
 
+    if $l2filteringreport; then
+        echo == Starting report receiver
+        docker compose up --wait report-receiver
+
+        echo == Writing filtering-report service config
+        run_script write-filtering-report-config
+    fi
+
     if $tokenbridge; then
         echo == Deploying L1-L2 token bridge
         sleep 10 # no idea why this sleep is needed but without it the deploy fails randomly