Got thousand of messages like this (few in every ms) on node leading to its death due to OOM
2025-08-28 17:14:12.793 INFO - �[37mSyncing *** Slot: 260021, Head slot: 215059, Remaining slots: 44962, Target chain: de97fe..2ca2 (260009) with 2 peers, Connected peers: 10�[0m
2025-08-28 17:14:12.793 INFO - �[90mSync Info *** Range: 259956 - 260009 (3 batches), Downloading: none, Ready: 54 slots (3 batches), Batch import: in progress�[0m
2025-08-28 17:14:15.489 INFO - �[33mBeacon chain syncing complete, waiting for Execution Client�[0m
2025-08-28 17:14:24.509 INFO - Completed regeneration of state for block 0xedb49fb45d3382fa46cd1b3e6e2cd49c7f6032c9df6a3cf9efcb563d6ee0333d at slot 214943 by replaying 21 blocks. Took 481ms
2025-08-28 17:14:24.668 INFO - �[33mReorg Event *** New Head: de97fe9419fe0ed1933783a200d4b8ebd2d9c05286841472c788680027e42ca2 (260009), Previous Head: 252f0931cece4f5a2502108a507c6b771a5ab4753c70a1c38815b0da5ae3b244 (215059), Common Ancestor: 9aea8b88faba5ee273f09580d44ec437c730ce4f45d876d3146b1bdcc27a74b3 (215016)�[0m
2025-08-28 17:14:24.669 INFO - �[37mSyncing *** Slot: 260022, Head slot: 260009, Waiting for execution layer sync, Connected peers: 10�[0m
2025-08-28 17:14:24.967 WARN - Failed to remove operations from pools for now canonical block: 0x56a77d5441122a90ec22f935374c2093a62cf9413b1b7bf92d053c8acae71da7
java.util.concurrent.CompletionException: java.util.concurrent.RejectedExecutionException: Rejected execution on task queue - storage_query
at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.uniApplyNow(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.uniApplyStage(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.thenApply(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.SafeFuture.thenApply(SafeFuture.java:570) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.dataproviders.lookup.BlockProvider.lambda$combined$4(BlockProvider.java:75) ~[teku-ethereum-dataproviders-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.CompletableFuture.uniComposeStage(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.thenCompose(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.SafeFuture.thenCompose(SafeFuture.java:651) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.dataproviders.lookup.BlockProvider.lambda$combined$5(BlockProvider.java:67) ~[teku-ethereum-dataproviders-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.dataproviders.lookup.BlockProvider.getBlock(BlockProvider.java:93) ~[teku-ethereum-dataproviders-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.storage.store.Store.retrieveSignedBlock(Store.java:663) ~[teku-storage-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.spec.datastructures.forkchoice.ReadOnlyStore.retrieveBlock(ReadOnlyStore.java:110) ~[teku-ethereum-spec-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.storage.client.RecentChainData.retrieveBlockByRoot(RecentChainData.java:637) ~[teku-storage-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.lambda$processCanonicalBlockOperations$14(OperationsReOrgManager.java:157) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.lang.Iterable.forEach(Unknown Source) ~[?:?]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.processCanonicalBlockOperations(OperationsReOrgManager.java:154) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.lambda$chainHeadUpdated$0(OperationsReOrgManager.java:89) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.Optional.ifPresent(Unknown Source) ~[?:?]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.chainHeadUpdated(OperationsReOrgManager.java:76) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown Source) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.executeMethod(DirectEventDeliverer.java:74) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.deliverTo(DirectEventDeliverer.java:54) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer.lambda$deliverTo$0(AsyncEventDeliverer.java:66) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer$QueueReader.deliverNextEvent(AsyncEventDeliverer.java:125) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer$QueueReader.run(AsyncEventDeliverer.java:116) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:?]
at java.base/java.lang.Thread.run(Unknown Source) [?:?]
Caused by: java.util.concurrent.RejectedExecutionException: Rejected execution on task queue - storage_query
at tech.pegasys.teku.infrastructure.async.MetricTrackingExecutorFactory.onRejectedExecution(MetricTrackingExecutorFactory.java:116) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.async.MetricTrackingExecutorFactory.lambda$newCachedThreadPool$0(MetricTrackingExecutorFactory.java:94) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.ThreadPoolExecutor.reject(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor.execute(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.ScheduledExecutorAsyncRunner.runAsync(ScheduledExecutorAsyncRunner.java:73) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.storage.server.CombinedStorageChannelSplitter.getHotBlocksByRoot(CombinedStorageChannelSplitter.java:156) ~[teku-storage-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown Source) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.executeMethod(DirectEventDeliverer.java:74) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.deliverToWithResponse(DirectEventDeliverer.java:67) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer.lambda$deliverToWithResponse$1(AsyncEventDeliverer.java:80) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
... 5 more
2025-08-28 17:14:24.984 WARN - Failed to remove operations from pools for now canonical block: 0x5a6ec5f789ee286027055efc77949db8d24c92b8e2791563f9100c4d2db40f61
java.util.concurrent.CompletionException: java.util.concurrent.RejectedExecutionException: Rejected execution on task queue - storage_query
at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.uniApplyNow(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.uniApplyStage(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.thenApply(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.SafeFuture.thenApply(SafeFuture.java:570) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.dataproviders.lookup.BlockProvider.lambda$combined$4(BlockProvider.java:75) ~[teku-ethereum-dataproviders-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.CompletableFuture.uniComposeStage(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.thenCompose(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.SafeFuture.thenCompose(SafeFuture.java:651) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.dataproviders.lookup.BlockProvider.lambda$combined$5(BlockProvider.java:67) ~[teku-ethereum-dataproviders-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.dataproviders.lookup.BlockProvider.getBlock(BlockProvider.java:93) ~[teku-ethereum-dataproviders-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.storage.store.Store.retrieveSignedBlock(Store.java:663) ~[teku-storage-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.spec.datastructures.forkchoice.ReadOnlyStore.retrieveBlock(ReadOnlyStore.java:110) ~[teku-ethereum-spec-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.storage.client.RecentChainData.retrieveBlockByRoot(RecentChainData.java:637) ~[teku-storage-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.lambda$processCanonicalBlockOperations$14(OperationsReOrgManager.java:157) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.lang.Iterable.forEach(Unknown Source) ~[?:?]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.processCanonicalBlockOperations(OperationsReOrgManager.java:154) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.lambda$chainHeadUpdated$0(OperationsReOrgManager.java:89) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.Optional.ifPresent(Unknown Source) ~[?:?]
at tech.pegasys.teku.statetransition.OperationsReOrgManager.chainHeadUpdated(OperationsReOrgManager.java:76) ~[teku-ethereum-statetransition-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown Source) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.executeMethod(DirectEventDeliverer.java:74) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.deliverTo(DirectEventDeliverer.java:54) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer.lambda$deliverTo$0(AsyncEventDeliverer.java:66) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer$QueueReader.deliverNextEvent(AsyncEventDeliverer.java:125) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer$QueueReader.run(AsyncEventDeliverer.java:116) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:?]
at java.base/java.lang.Thread.run(Unknown Source) [?:?]
Caused by: java.util.concurrent.RejectedExecutionException: Rejected execution on task queue - storage_query
at tech.pegasys.teku.infrastructure.async.MetricTrackingExecutorFactory.onRejectedExecution(MetricTrackingExecutorFactory.java:116) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.async.MetricTrackingExecutorFactory.lambda$newCachedThreadPool$0(MetricTrackingExecutorFactory.java:94) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.ThreadPoolExecutor.reject(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor.execute(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.ScheduledExecutorAsyncRunner.runAsync(ScheduledExecutorAsyncRunner.java:73) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.storage.server.CombinedStorageChannelSplitter.getHotBlocksByRoot(CombinedStorageChannelSplitter.java:156) ~[teku-storage-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown Source) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.executeMethod(DirectEventDeliverer.java:74) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.deliverToWithResponse(DirectEventDeliverer.java:67) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer.lambda$deliverToWithResponse$1(AsyncEventDeliverer.java:80) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
... 5 more
2025-08-28 17:14:24.985 WARN - Failed to remove operations from pools for now canonical block: 0x38333991a1ce830790965b43d084576451d51c002bd0a51c9d48f2502c863185
java.util.concurrent.CompletionException: java.util.concurrent.RejectedExecutionException: Rejected execution on task queue - storage_query
at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.completeThrowable(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.SafeFuture.lambda$propagateResult$2(SafeFuture.java:147) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.SafeFuture.lambda$propagateToAsync$30(SafeFuture.java:437) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.async.SafeFuture.of(SafeFuture.java:81) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.async.AsyncRunner.lambda$runAsync$2(AsyncRunner.java:47) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.async.SafeFuture.of(SafeFuture.java:73) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.async.ScheduledExecutorAsyncRunner.lambda$createRunnableForAction$1(ScheduledExecutorAsyncRunner.java:124) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:?]
at java.base/java.lang.Thread.run(Unknown Source) [?:?]
Caused by: java.util.concurrent.RejectedExecutionException: Rejected execution on task queue - storage_query
at tech.pegasys.teku.infrastructure.async.MetricTrackingExecutorFactory.onRejectedExecution(MetricTrackingExecutorFactory.java:116) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.async.MetricTrackingExecutorFactory.lambda$newCachedThreadPool$0(MetricTrackingExecutorFactory.java:94) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/java.util.concurrent.ThreadPoolExecutor.reject(Unknown Source) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor.execute(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.async.ScheduledExecutorAsyncRunner.runAsync(ScheduledExecutorAsyncRunner.java:73) ~[teku-infrastructure-async-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.storage.server.CombinedStorageChannelSplitter.getHotBlocksByRoot(CombinedStorageChannelSplitter.java:156) ~[teku-storage-develop.jar:25.7.1+81-g50be3d3e31]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(Unknown Source) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.executeMethod(DirectEventDeliverer.java:74) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.DirectEventDeliverer.deliverToWithResponse(DirectEventDeliverer.java:67) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer.lambda$deliverToWithResponse$1(AsyncEventDeliverer.java:80) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer$QueueReader.deliverNextEvent(AsyncEventDeliverer.java:125) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
at tech.pegasys.teku.infrastructure.events.AsyncEventDeliverer$QueueReader.run(AsyncEventDeliverer.java:116) ~[teku-infrastructure-events-develop.jar:25.7.1+81-g50be3d3e31]
... 3 more
Got thousand of messages like this (few in every ms) on node leading to its death due to OOM