From 161084531fdf69b422ed800293404f15a6ae1c8d Mon Sep 17 00:00:00 2001 From: ldt1996 Date: Tue, 31 Mar 2026 14:39:50 +0300 Subject: [PATCH 1/4] make read txn timeout configurable and set default to 1min --- resources/RecordEncoder.ts | 21 +++++++-- unitTests/resources/txn-tracking.test.js | 55 ++++++++++++++++++++++++ utility/hdbTerms.ts | 1 + 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index ac4a7f41d..83e277d84 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -21,6 +21,8 @@ import { blobsWereEncoded, decodeFromDatabase, deleteBlobsInObject, encodeBlobsW import { recordAction } from './analytics/write.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; import { when } from '../utility/when.ts'; +import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; +import * as envMngr from '../utility/environment/environmentManager.js'; export type Entry = { key: any; value: any; @@ -485,16 +487,20 @@ export function handleLocalTimeForGets(store, rootStore) { return store; } const trackedTxns: WeakRef[] = []; -setInterval(() => { +const configValue = envMngr.get(CONFIG_PARAMS.STORAGE_MAXREADTRANSACTIONOPENTIME) ?? 60000; +let READ_TXN_TIMEOUT_TICKS = Math.round( + Math.min(Math.max(configValue, 15000), 300000) / 15000 +); // clamp between 15s and 5min +export function checkReadTxnTimeouts() { for (let i = 0; i < trackedTxns.length; i++) { const txn = trackedTxns[i].deref(); if (!txn || txn.isDone || txn.isCommitted) trackedTxns.splice(i--, 1); else if (txn.notCurrent) { if (txn.openTimer) { if (txn.openTimer > 3) { - if (txn.openTimer > 60) { + if (txn.openTimer > READ_TXN_TIMEOUT_TICKS) { harperLogger.error( - 'Read transaction detected that has been open too long (over 15 minutes), ending transaction', + `Read transaction detected that has been open too long (over ${Math.round(READ_TXN_TIMEOUT_TICKS * 15)} seconds), ending transaction`, txn ); txn.done(); @@ -508,7 +514,14 @@ setInterval(() => { } else txn.openTimer = 1; } } -}, 15000).unref(); +} +setInterval(checkReadTxnTimeouts, 15000).unref(); +export function setReadTxnExpiration(ms: number) { + READ_TXN_TIMEOUT_TICKS = Math.round( + Math.min(Math.max(ms, 15000), 300000) / 15000 + ); +} + export function recordUpdater(store, tableId, auditStore) { return function ( id, diff --git a/unitTests/resources/txn-tracking.test.js b/unitTests/resources/txn-tracking.test.js index b8b05a00d..4719a41d3 100644 --- a/unitTests/resources/txn-tracking.test.js +++ b/unitTests/resources/txn-tracking.test.js @@ -3,6 +3,7 @@ const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); const { setTxnExpiration } = require('#src/resources/DatabaseTransaction'); const { setTxnExpiration: setLMDBTxnExpiration } = require('#src/resources/LMDBTransaction'); +const { setReadTxnExpiration, checkReadTxnTimeouts } = require('#src/resources/RecordEncoder'); const { setMainIsWorker } = require('#js/server/threads/manageThreads'); const { table } = require('#src/resources/databases'); const { setTimeout: delay } = require('node:timers/promises'); @@ -54,3 +55,57 @@ describe('Txn Expiration', () => { setTxnExpiration(30000); }); }); + +describe('Read Txn Expiration', () => { + let SlowReadResource; + before(async function () { + setupTestDBPath(); + setMainIsWorker(true); + let BasicTable = table({ + table: 'ReadTxnTable', + database: 'test', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + SlowReadResource = class extends BasicTable { + async get(query) { + const result = super.get(query); + await delay(50); + return result; + } + }; + }); + + it('Read txn will be ended after timeout', async function () { + await SlowReadResource.put(1, { name: 'one' }); + + // set timeout to minimum, 15s = 1 tick, openTimer > 1 means txn is expired + setReadTxnExpiration(15000); + + const readPromise = SlowReadResource.get(1); + await delay(20); + + // simulate timer ticks + checkReadTxnTimeouts(); + checkReadTxnTimeouts(); + + await readPromise; + }); + + it('Read txn below threshold is not expired', async function () { + setReadTxnExpiration(60000); + + await SlowReadResource.put(2, { name: 'two' }); + const readPromise = SlowReadResource.get(2); + await delay(20); + + // only 2 ticks + checkReadTxnTimeouts(); + + const result = await readPromise; + assert.equal(result.name, 'two'); + }); + + after(function () { + setReadTxnExpiration(60000); + }); +}); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 134d368fd..fa4e6b220 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -559,6 +559,7 @@ export const CONFIG_PARAMS = { STORAGE_ENCRYPTION: 'storage_encryption', STORAGE_MAXTRANSACTIONQUEUETIME: 'storage_maxTransactionQueueTime', STORAGE_MAXTRANSACTIONOPENTIME: 'storage_maxTransactionOpenTime', + STORAGE_MAXREADTRANSACTIONOPENTIME: 'storage_maxReadTransactionOpenTime', STORAGE_DEBUGLONGTRANSACTIONS: 'storage_debugLongTransactions', STORAGE_PATH: 'storage_path', STORAGE_BLOBPATHS: 'storage_blobPaths', From 2527d4415cb36e7ca8f03728462401379f73397a Mon Sep 17 00:00:00 2001 From: ldt1996 Date: Tue, 31 Mar 2026 14:47:41 +0300 Subject: [PATCH 2/4] fix formatting --- resources/RecordEncoder.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index 83e277d84..a7e261257 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -488,9 +488,7 @@ export function handleLocalTimeForGets(store, rootStore) { } const trackedTxns: WeakRef[] = []; const configValue = envMngr.get(CONFIG_PARAMS.STORAGE_MAXREADTRANSACTIONOPENTIME) ?? 60000; -let READ_TXN_TIMEOUT_TICKS = Math.round( - Math.min(Math.max(configValue, 15000), 300000) / 15000 -); // clamp between 15s and 5min +let READ_TXN_TIMEOUT_TICKS = Math.round(Math.min(Math.max(configValue, 15000), 300000) / 15000); // clamp between 15s and 5min export function checkReadTxnTimeouts() { for (let i = 0; i < trackedTxns.length; i++) { const txn = trackedTxns[i].deref(); @@ -517,9 +515,7 @@ export function checkReadTxnTimeouts() { } setInterval(checkReadTxnTimeouts, 15000).unref(); export function setReadTxnExpiration(ms: number) { - READ_TXN_TIMEOUT_TICKS = Math.round( - Math.min(Math.max(ms, 15000), 300000) / 15000 - ); + READ_TXN_TIMEOUT_TICKS = Math.round(Math.min(Math.max(ms, 15000), 300000) / 15000); } export function recordUpdater(store, tableId, auditStore) { From 93cafaada91354a6824f8b0a051dd199fa7bd1cd Mon Sep 17 00:00:00 2001 From: ldt1996 Date: Sun, 12 Apr 2026 10:21:33 +0300 Subject: [PATCH 3/4] remove clamping + increase exp time --- resources/RecordEncoder.ts | 6 +++--- unitTests/resources/txn-tracking.test.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index a7e261257..e53be5860 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -487,8 +487,8 @@ export function handleLocalTimeForGets(store, rootStore) { return store; } const trackedTxns: WeakRef[] = []; -const configValue = envMngr.get(CONFIG_PARAMS.STORAGE_MAXREADTRANSACTIONOPENTIME) ?? 60000; -let READ_TXN_TIMEOUT_TICKS = Math.round(Math.min(Math.max(configValue, 15000), 300000) / 15000); // clamp between 15s and 5min +const configValue = envMngr.get(CONFIG_PARAMS.STORAGE_MAXREADTRANSACTIONOPENTIME) ?? 300000; +let READ_TXN_TIMEOUT_TICKS = Math.round(configValue / 15000); export function checkReadTxnTimeouts() { for (let i = 0; i < trackedTxns.length; i++) { const txn = trackedTxns[i].deref(); @@ -515,7 +515,7 @@ export function checkReadTxnTimeouts() { } setInterval(checkReadTxnTimeouts, 15000).unref(); export function setReadTxnExpiration(ms: number) { - READ_TXN_TIMEOUT_TICKS = Math.round(Math.min(Math.max(ms, 15000), 300000) / 15000); + READ_TXN_TIMEOUT_TICKS = Math.round(ms / 15000); } export function recordUpdater(store, tableId, auditStore) { diff --git a/unitTests/resources/txn-tracking.test.js b/unitTests/resources/txn-tracking.test.js index 4719a41d3..fd96ce075 100644 --- a/unitTests/resources/txn-tracking.test.js +++ b/unitTests/resources/txn-tracking.test.js @@ -106,6 +106,6 @@ describe('Read Txn Expiration', () => { }); after(function () { - setReadTxnExpiration(60000); + setReadTxnExpiration(300000); }); }); From 5cc1b4193a67c82e5aa938b32ca86a9bfa3c7f46 Mon Sep 17 00:00:00 2001 From: ldt1996 Date: Tue, 12 May 2026 17:35:59 +0300 Subject: [PATCH 4/4] Update utility/hdbTerms.ts Co-authored-by: Chris Barber --- utility/hdbTerms.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 5ac7ee2ec..e67def85c 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -560,7 +560,7 @@ export const CONFIG_PARAMS = { STORAGE_ENCRYPTION: 'storage_encryption', STORAGE_MAXTRANSACTIONQUEUETIME: 'storage_maxTransactionQueueTime', STORAGE_MAXTRANSACTIONOPENTIME: 'storage_maxTransactionOpenTime', - STORAGE_MAXREADTRANSACTIONOPENTIME: 'storage_maxReadTransactionOpenTime', + STORAGE_MAX_READ_TRANSACTION_OPEN_TIME: 'storage_maxReadTransactionOpenTime', STORAGE_DEBUGLONGTRANSACTIONS: 'storage_debugLongTransactions', STORAGE_PATH: 'storage_path', STORAGE_BLOBPATHS: 'storage_blobPaths',