Add evm-ingest option to fix log index DATA-85

evm/evm-data-service/src/data-source/setup.ts

@@ -27,6 +27,7 @@ export interface DataSourceOptions {
     verifyReceiptsRoot?: boolean
     verifyLogsBloom?: boolean
     useGasUsedForReceiptsRoot?: boolean
+    fixLogIndex?: boolean
 }
 
 
@@ -48,7 +49,8 @@ export function createDataSource(options: DataSourceOptions): DataSource<Block>
         verifyTxSender: options.verifyTxSender,
         verifyReceiptsRoot: options.verifyReceiptsRoot,
         verifyLogsBloom: options.verifyLogsBloom,
-        useGasUsedForReceiptsRoot: options.useGasUsedForReceiptsRoot
+        useGasUsedForReceiptsRoot: options.useGasUsedForReceiptsRoot,
+        fixLogIndex: options.fixLogIndex
     })
     let rpcSource = new EvmRpcDataSource({
         rpc: httpRpc,

evm/evm-data-service/src/main.ts

@@ -35,6 +35,7 @@ runProgram(async () => {
     program.option('--verify-receipts-root', 'Verify block receipts against receipts root')
     program.option('--verify-logs-bloom', 'Verify block logs against logs bloom')
     program.option('--use-gas-used-for-receipts-root', 'Use gasUsed instead of cumulativeGasUsed for receipts root calculation')
+    program.option('--fix-log-index', 'Renumber log indices sequentially (workaround for chains with broken logIndex)')
     program.parse()
 
     let args = program.opts() as {
@@ -58,6 +59,7 @@ runProgram(async () => {
         verifyReceiptsRoot?: boolean
         verifyLogsBloom?: boolean
         useGasUsedForReceiptsRoot?: boolean
+        fixLogIndex?: boolean
     }
 
     let dataSourceOptions: DataSourceOptions = {
@@ -78,7 +80,8 @@ runProgram(async () => {
         verifyTxRoot: args.verifyTxRoot,
         verifyReceiptsRoot: args.verifyReceiptsRoot,
         verifyLogsBloom: args.verifyLogsBloom,
-        useGasUsedForReceiptsRoot: args.useGasUsedForReceiptsRoot
+        useGasUsedForReceiptsRoot: args.useGasUsedForReceiptsRoot,
+        fixLogIndex: args.fixLogIndex
     }
 
     let mainWorker = new WorkerClient(dataSourceOptions)

evm/evm-ingest/src/ingest.ts

@@ -7,6 +7,7 @@ import {toJSON} from '@subsquid/util-internal-json'
 interface Options extends IngestOptions {
     withTraces?: boolean
     withStatediffs?: boolean
+    fixLogIndex?: boolean
 }
 
 
@@ -28,16 +29,18 @@ export class EvmIngest extends Ingest<Options> {
         })
         program.option('--with-traces', 'Include EVM call traces')
         program.option('--with-statediffs', 'Include EVM state updates')
+        program.option('--fix-log-index', 'Renumber log indices sequentially (workaround for chains with broken logIndex)')
     }
 
     protected async *getBlocks(range: Range): AsyncIterable<object[]> {
         let withTraces = this.options().withTraces
         let withStateDiffs = this.options().withStatediffs
+        let fixLogIndex = this.options().fixLogIndex
 
         for await (let batch of this.archive().getRawBlocks<RawBlock>(range)) {
             yield batch.map(raw => {
                 try {
-                    let block = mapRawBlock(raw, withTraces, withStateDiffs)
+                    let block = mapRawBlock(raw, withTraces, withStateDiffs, fixLogIndex)
                     return toJSON(block)
                 } catch(err: any) {
                     throw addErrorContext(err, {

evm/evm-normalization/src/mapping.ts

@@ -700,7 +700,7 @@ export function mapRpcBlock(src: rpc.Block, withTraces?: boolean, withStateDiffs
 }
 
 
-export function mapRawBlock(raw: RawBlock, withTraces?: boolean, withStateDiffs?: boolean): Block {
+export function mapRawBlock(raw: RawBlock, withTraces?: boolean, withStateDiffs?: boolean, fixLogIndex?: boolean): Block {
     let block: Block = {
         header: mapBlockHeader(raw),
         transactions: [],
@@ -721,7 +721,11 @@ export function mapRawBlock(raw: RawBlock, withTraces?: boolean, withStateDiffs?
         if (tx.receipt_) {
             for (let log of tx.receipt_.logs) {
                 let normalized = mapLog(log)
-                assert.equal(normalized.logIndex, logIndex++)
+                if (fixLogIndex) {
+                    normalized.logIndex = logIndex++
+                } else {
+                    assert.equal(normalized.logIndex, logIndex++)
+                }
                 block.logs.push(normalized)
             }
         }
@@ -757,7 +761,11 @@ export function mapRawBlock(raw: RawBlock, withTraces?: boolean, withStateDiffs?
         assert(block.logs.length == 0)
         for (let log of raw.logs_) {
             let normalized = mapLog(log)
-            assert.equal(normalized.logIndex, logIndex++)
+            if (fixLogIndex) {
+                normalized.logIndex = logIndex++
+            } else {
+                assert.equal(normalized.logIndex, logIndex++)
+            }
             block.logs.push(normalized)
         }
     }

evm/evm-rpc/src/rpc.ts

@@ -40,6 +40,7 @@ export interface RpcOptions {
     verifyReceiptsRoot?: boolean
     verifyLogsBloom?: boolean
     useGasUsedForReceiptsRoot?: boolean
+    fixLogIndex?: boolean
 }
 
 
@@ -52,6 +53,7 @@ export class Rpc {
     private verifyReceiptsRoot?: boolean
     private verifyLogsBloom?: boolean
     private useGasUsedForReceiptsRoot?: boolean
+    private fixLogIndex?: boolean
     private log: Logger
     private receiptsMethod?: GetReceiptsMethod
     private chainUtils?: ChainUtils
@@ -65,6 +67,7 @@ export class Rpc {
         this.verifyReceiptsRoot = options.verifyReceiptsRoot
         this.verifyLogsBloom = options.verifyLogsBloom
         this.useGasUsedForReceiptsRoot = options.useGasUsedForReceiptsRoot
+        this.fixLogIndex = options.fixLogIndex
         this.log = createLogger('sqd:evm-rpc')
     }
 
@@ -239,7 +242,11 @@ export class Rpc {
             try {
                 let logIndex = 0
                 for (let log of logs) {
-                    assert.equal(qty2Int(log.logIndex), logIndex++)
+                    if (this.fixLogIndex) {
+                        log.logIndex = toQty(logIndex++)
+                    } else {
+                        assert.equal(qty2Int(log.logIndex), logIndex++)
+                    }
                 }
 
                 if (this.verifyLogsBloom) {
@@ -317,7 +324,11 @@ export class Rpc {
             try {
                 let logIndex = 0
                 for (let log of logs) {
-                    assert.equal(qty2Int(log.logIndex), logIndex++)
+                    if (this.fixLogIndex) {
+                        log.logIndex = toQty(logIndex++)
+                    } else {
+                        assert.equal(qty2Int(log.logIndex), logIndex++)
+                    }
                 }
 
                 if (this.verifyLogsBloom) {
@@ -377,7 +388,11 @@ export class Rpc {
             try {
                 let logIndex = 0
                 for (let log of logs) {
-                    assert.equal(qty2Int(log.logIndex), logIndex++)
+                    if (this.fixLogIndex) {
+                        log.logIndex = toQty(logIndex++)
+                    } else {
+                        assert.equal(qty2Int(log.logIndex), logIndex++)
+                    }
                 }
 
                 if (this.verifyLogsBloom) {