From fff85fb61cbce3c247594aa8c64df52d914efe16 Mon Sep 17 00:00:00 2001 From: abdevil Date: Wed, 18 Mar 2026 10:56:58 +0530 Subject: [PATCH 1/2] optimize scanning time --- src/blockchain_db/blockchain_db.h | 2 +- src/blockchain_db/lmdb/db_lmdb.cpp | 23 +- src/blockchain_db/lmdb/db_lmdb.h | 4 +- src/cryptonote_core/beldex_name_system.cpp | 5 +- src/cryptonote_core/blockchain.cpp | 493 ++++++++++++++++++--- src/cryptonote_core/blockchain.h | 12 +- 6 files changed, 463 insertions(+), 76 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index d0d50e6957a..ca3b997aaf7 100755 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -934,7 +934,7 @@ class BlockchainDB * * @return the block */ - virtual block get_block_from_height(uint64_t height) const = 0; + virtual block get_block_from_height(uint64_t height , size_t* size = nullptr) const = 0; /** * @brief fetch a block's timestamp diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 04bf274019d..072d8b69df8 100755 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -707,10 +707,13 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const // additional size needed. uint64_t size_used = mst.ms_psize * mei.me_last_pgno; - LOG_PRINT_L3("DB map size: " << mei.me_mapsize); - LOG_PRINT_L3("Space used: " << size_used); - LOG_PRINT_L3("Space remaining: " << mei.me_mapsize - size_used); - LOG_PRINT_L3("Size threshold: " << threshold_size); + LOG_PRINT_L3("DB map size: " << tools::get_human_readable_bytes(mei.me_mapsize)); + LOG_PRINT_L3("Space used: " << tools::get_human_readable_bytes(size_used)); + LOG_PRINT_L3("Space remaining: " + << tools::get_human_readable_bytes(mei.me_mapsize - size_used)); + LOG_PRINT_L3("Size threshold: " + << tools::get_human_readable_bytes(threshold_size)); + float resize_percent = RESIZE_PERCENT; LOG_PRINT_L3(fmt::format("Percent used: {:.04f} Percent threshold: {:.04f}", 100. * size_used / mei.me_mapsize, 100. * resize_percent)); @@ -1470,7 +1473,7 @@ void BlockchainLMDB::open(const fs::path& filename, cryptonote::network_type net throw0(DB_ERROR(lmdb_error("Failed to set max memory map size: ", result).c_str())); mdb_env_info(m_env, &mei); cur_mapsize = (uint64_t)mei.me_mapsize; - LOG_PRINT_L1("LMDB memory map size: " << cur_mapsize); + LOG_PRINT_L1("LMDB memory map size: " << tools::get_human_readable_bytes(cur_mapsize)); } if (need_resize()) @@ -2532,7 +2535,7 @@ template || std::is_same_v || std::is_same_v, int>> -T BlockchainLMDB::get_and_convert_block_blob_from_height(uint64_t height) const +T BlockchainLMDB::get_and_convert_block_blob_from_height(uint64_t height , size_t* size) const { // NOTE: Avoid any intermediary functions like taking a blob, then converting // to block which incurs a copy into blobdata then conversion, and prefer @@ -2573,12 +2576,15 @@ T BlockchainLMDB::get_and_convert_block_blob_from_height(uint64_t height) const result = blob; } + if (size) + *size = blob.size(); + return result; } -block BlockchainLMDB::get_block_from_height(uint64_t height) const +block BlockchainLMDB::get_block_from_height(uint64_t height, size_t* size) const { - block result = get_and_convert_block_blob_from_height(height); + block result = get_and_convert_block_blob_from_height(height , size); return result; } @@ -6121,6 +6127,7 @@ bool BlockchainLMDB::get_master_node_data(std::string& data, bool long_term) con } } + data.clear(); data.assign(reinterpret_cast(v.mv_data), v.mv_size); return true; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index fc43e6e92fd..1b4f5f6a676 100755 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -204,7 +204,7 @@ class BlockchainLMDB : public BlockchainDB uint64_t get_block_height(const crypto::hash& h) const override; - block get_block_from_height(uint64_t height) const override; + block get_block_from_height(uint64_t height, size_t* size = nullptr) const override; block_header get_block_header_from_height(uint64_t height) const override; @@ -447,7 +447,7 @@ class BlockchainLMDB : public BlockchainDB std::enable_if_t || std::is_same_v || std::is_same_v, int> = 0> - T get_and_convert_block_blob_from_height(uint64_t height) const; + T get_and_convert_block_blob_from_height(uint64_t height , size_t* size = nullptr) const; MDB_env* m_env; diff --git a/src/cryptonote_core/beldex_name_system.cpp b/src/cryptonote_core/beldex_name_system.cpp index d19e73fcfd3..c7b2bd9ec33 100755 --- a/src/cryptonote_core/beldex_name_system.cpp +++ b/src/cryptonote_core/beldex_name_system.cpp @@ -2224,10 +2224,11 @@ bool name_system_db::add_block(const cryptonote::block &block, const std::vector last_processed_height = height; last_processed_hash = cryptonote::get_block_hash(block); - if (bns_parsed_from_block) + bool do_commit = (height % 16384 == 0) || bns_parsed_from_block; + if (do_commit) { save_settings(last_processed_height, last_processed_hash, static_cast(DB_VERSION)); - db_transaction.commit = bns_parsed_from_block; + db_transaction.commit = true; } return true; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index c470f68859b..1839ecb4886 100755 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -299,71 +299,330 @@ uint64_t Blockchain::get_current_blockchain_height(bool lock) const if (lock) lock_.lock(); return m_db->height(); } +// --- types used for threaded loader --- +struct block_data +{ + uint64_t height = 0; + std::vector blocks; + std::vector> txs; + size_t size = 0; + bool failed = false; +}; +struct block_load_context +{ + static constexpr uint64_t CHUNK_SIZE = 300; // tuneable chunk size + static constexpr size_t MAX_QUEUE_SIZE = 12; // tuneable queue depth + std::mutex block_mut; + std::condition_variable block_cv; + std::queue next_blocks; + std::thread thread; + bool finished = false; + bool failed = false; + uint64_t height = 0; +}; //------------------------------------------------------------------ -bool Blockchain::load_missing_blocks_into_beldex_subsystems() +bool Blockchain::load_missing_blocks_into_beldex_subsystems(const std::atomic *abort, bool use_threaded_load) { - uint64_t const mnl_height = std::max(hard_fork_begins(m_nettype, hf::hf9_master_nodes).value_or(0), m_master_node_list.height() + 1); - uint64_t const bns_height = std::max(hard_fork_begins(m_nettype, hf::hf18_bns).value_or(0), m_bns_db.height() + 1); - uint64_t const end_height = m_db->height(); - uint64_t const start_height = std::min(end_height, std::min(bns_height, mnl_height)); - + + // Heights for subsystems (Beldex) + uint64_t const mnl_height = std::max(hard_fork_begins(m_nettype, hf::hf9_master_nodes).value_or(0), m_master_node_list.height() + 1); + uint64_t const bns_height = std::max(hard_fork_begins(m_nettype, hf::hf18_bns).value_or(0), m_bns_db.height() + 1); + + const uint64_t end_height = m_db->height(); + const uint64_t start_height = std::min(end_height, std::min(bns_height, mnl_height)); int64_t const total_blocks = static_cast(end_height) - static_cast(start_height); + if (total_blocks <= 0) return true; - if (total_blocks > 1) - MGINFO("Loading blocks into beldex subsystems, scanning blockchain from height: " << start_height << " to: " << end_height << " (mnl: " << mnl_height << ", bns: " << bns_height << ")"); - using clock = std::chrono::steady_clock; - using dseconds = std::chrono::duration; - int64_t constexpr BLOCK_COUNT = 1000; - auto work_start = clock::now(); - auto scan_start = work_start; - dseconds bns_duration{}, mnl_duration{}, bns_iteration_duration{}, mnl_iteration_duration{}; + MGINFO("Loading blocks into beldex subsystems, scanning blockchain from height: " + << start_height << " to: " << end_height << " (mnl: " << mnl_height + << ", bns: " << bns_height << ")"); + + // Initialize load context + block_load_context load_context = {}; + load_context.height = start_height; - for (int64_t block_count = total_blocks, - index = 0; - block_count > 0; - block_count -= BLOCK_COUNT, index++) + // Helper: fetch chunk of blocks + parallel fetch of txs + auto get_block_data = [&](uint64_t height, uint64_t end_height) -> block_data { - auto duration = dseconds{clock::now() - work_start}; - if (duration >= 10s) + block_data next_chunk{}; + next_chunk.height = height; + + // 1) Load blocks { - m_master_node_list.store(); - MGINFO(fmt::format("... scanning height {} ({:.3f}s) (mnl: {:.3f}s, bns: {:.3f}s)", - start_height + (index * BLOCK_COUNT), - duration.count(), - mnl_iteration_duration.count(), - bns_iteration_duration.count())); -#ifdef ENABLE_SYSTEMD - // Tell systemd that we're doing something so that it should let us continue starting up - // (giving us 120s until we have to send the next notification): - sd_notify(0, ("EXTEND_TIMEOUT_USEC=120000000\nSTATUS=Recanning blockchain; height " + std::to_string(start_height + (index * BLOCK_COUNT))).c_str()); -#endif - work_start = clock::now(); + size_t blocks_size; - bns_duration += bns_iteration_duration; - mnl_duration += mnl_iteration_duration; - bns_iteration_duration = mnl_iteration_duration = {}; + if (!_get_blocks_only(height, block_load_context::CHUNK_SIZE, next_chunk.blocks, &blocks_size)) + { + LOG_ERROR("Unable to get checkpointed historical blocks [" << height << "-" << std::min(height + block_load_context::CHUNK_SIZE - 1, end_height) + << "] for updating beldex subsystems"); + next_chunk.failed = true; + return next_chunk; + } + next_chunk.size += blocks_size; } - std::vector blocks; - uint64_t height = start_height + (index * BLOCK_COUNT); - if (!get_blocks_only(height, static_cast(BLOCK_COUNT), blocks)) + if (next_chunk.blocks.empty()) { - LOG_ERROR("Unable to get checkpointed historical blocks for updating beldex subsystems"); - return false; + return {}; } - for (cryptonote::block const &blk : blocks) + // 2) Parallelise TX loading via threadpool (one wait per block - keeps memory modest) + tools::threadpool::waiter tpool_waiter; + tools::threadpool &tpool = tools::threadpool::getInstance(); + { - uint64_t block_height = get_block_height(blk); + std::atomic bytes_loaded_for_block{0}; + std::atomic failed_height{0}; + next_chunk.txs.resize(next_chunk.blocks.size()); + + for (size_t blk_index = 0; blk_index < next_chunk.blocks.size(); blk_index++) + { + const auto &blk = next_chunk.blocks[blk_index]; + uint64_t blk_height = get_block_height(blk); + auto &txs = next_chunk.txs[blk_index]; + txs.resize(blk.tx_hashes.size()); + + for (size_t tx_index = 0; tx_index < blk.tx_hashes.size(); ++tx_index) + { + const crypto::hash &tx_hash = blk.tx_hashes[tx_index]; + tpool.submit(&tpool_waiter, [this, &txs, tx_index, tx_hash, + &bytes_loaded_for_block, blk_height, &failed_height]() + { + std::vector get_tx_result; + const std::vector single_hash{tx_hash}; + if (!_get_transactions(single_hash, get_tx_result, nullptr, nullptr)) { + if (failed_height == 0) failed_height = blk_height; + return; + } + if (get_tx_result.empty()) { + if (failed_height == 0) failed_height = blk_height; + return; + } + bytes_loaded_for_block += get_tx_result[0].blob_size; + txs[tx_index] = std::move(get_tx_result[0]); }); + + if (failed_height) + break; + } + + //CRITICAL FIX: Wait after EACH block, not after entire chunk + tpool_waiter.wait(&tpool); + + if (failed_height) + { + LOG_ERROR("Unable to get all transactions for subsystem updating from block: " << failed_height); + next_chunk.blocks.clear(); + next_chunk.txs.clear(); + next_chunk.size = 0; + next_chunk.failed = true; + return next_chunk; + } + + // Move block processing here + next_chunk.size += bytes_loaded_for_block.load(); + bytes_loaded_for_block = 0; // Reset for next block + cryptonote::get_block_hash(blk); + } + } + return next_chunk; + }; + + // If using a threaded loader, spawn loader thread that preloads chunks + if (use_threaded_load) + { + load_context.thread = std::thread{[&] + { + // Deferred callback that gets fired if we return early (or throw) that makes sure the + // processing thread gets notified about the failure. + auto failure_propagator = beldex::defer([&] + { + std::unique_lock lock{load_context.block_mut}; + load_context.failed = true; + load_context.block_cv.notify_all(); + }); + + for (; load_context.height < end_height; + load_context.height += block_load_context::CHUNK_SIZE) + { + { + std::unique_lock lock{load_context.block_mut}; + load_context.block_cv.wait( + lock, + [&] + { + return load_context.failed || (abort && *abort) || + load_context.next_blocks.size() < block_load_context::MAX_QUEUE_SIZE; + }); + + if (load_context.failed || (abort && *abort)) + return; + + assert(load_context.next_blocks.size() < block_load_context::MAX_QUEUE_SIZE); + } + + // Load the block data (may be slow) + block_data next_chunk = get_block_data(load_context.height, end_height); + + if (next_chunk.failed) + { + std::unique_lock lock{load_context.block_mut}; + load_context.failed = true; + load_context.block_cv.notify_all(); + return; + } + + { + std::unique_lock lock{load_context.block_mut}; + load_context.next_blocks.push(std::move(next_chunk)); + } + load_context.block_cv.notify_all(); + } - std::vector txs; - if (!get_transactions(blk.tx_hashes, txs)) + // Disarm the failure transmitter, then signal the processing thread that we finished + // loading everything. + failure_propagator.cancel(); + + { + std::unique_lock lock{load_context.block_mut}; + load_context.finished = true; + } + load_context.block_cv.notify_all(); + }}; + } + + // If we bail out of this function in any way before the very end successful `return true` (just + // before which we cancel this deferred call) then make sure we signal the loader thread and + // rejoin the thread on our way out. + auto failure_rejoiner = beldex::defer([&]{ + if (use_threaded_load) { + { + std::unique_lock lock{load_context.block_mut}; + load_context.failed = true; + } + load_context.block_cv.notify_all(); + load_context.thread.join(); + } }); + + // Timers / stats + using clock = std::chrono::steady_clock; + using dseconds = std::chrono::duration; + auto work_start = clock::now(); + auto scan_start = work_start; + dseconds bns_duration{}, bns_interval_duration{}; + dseconds mnl_duration{}, mnl_interval_duration{}; + dseconds get_block_data_duration{}, get_block_data_interval_duration{}; + + // Periodic store setup (matching Oxen's approach) + constexpr auto store_interval = std::chrono::minutes(5); + auto next_store = + work_start + std::chrono::milliseconds{ + crypto::rand() % std::chrono::milliseconds{store_interval}.count()}; + + uint64_t work_blocks = 0; + uint64_t total_bytes = 0, work_bytes = 0; + + // Main processing loop: pull chunks and feed to subsystems + while (true) + { + auto get_block_data_start = clock::now(); + block_data chunk; + + if (use_threaded_load) + { { - MERROR("Unable to get transactions for block for updating BNS DB: " << cryptonote::get_block_hash(blk)); + std::unique_lock lock{load_context.block_mut}; + load_context.block_cv.wait(lock, [&] + { return load_context.failed || (abort && *abort) || load_context.finished || + !load_context.next_blocks.empty(); }); + + if (load_context.failed || (abort && *abort)) + return false; + + if (load_context.finished && load_context.next_blocks.empty()) + break; + + chunk = std::move(load_context.next_blocks.front()); + load_context.next_blocks.pop(); + } + load_context.block_cv.notify_all(); // Notify the loader that we've removed a block + } + else + { + chunk = get_block_data(load_context.height, end_height); + if (chunk.failed) return false; + if (load_context.height >= end_height || chunk.blocks.empty() || (abort && *abort)) + break; + load_context.height += block_load_context::CHUNK_SIZE; + } + + auto now = clock::now(); + get_block_data_interval_duration += now - get_block_data_start; + dseconds interval_duration = now - work_start; + + // Periodic store + if (now >= next_store) + { + auto store_start = clock::now(); + m_master_node_list.store(); + auto store_end = clock::now(); + auto elapsed = store_end - store_start; + + if (elapsed >= 1s) + { + MGINFO(fmt::format("... stored MN state snapshot @ {} in {:.2f}s", + chunk.height, + std::chrono::duration(elapsed).count())); } + now = store_end; + next_store = now + store_interval; + } + + bool every_10s = interval_duration >= 10s; + uint64_t height = chunk.height; + + if (height + chunk.blocks.size() >= end_height || every_10s) + { + float blocks_per_s = static_cast(work_blocks) / std::max(1e-9, interval_duration.count()); + float bytes_per_s = static_cast(work_bytes) / std::max(1e-9, interval_duration.count()); + + MGINFO(fmt::format("... scanning height {}/{} ({:.2f}s) (get blks: {:.2f}s; mnl: {:.2f}s; bns: {:.2f}s; {:.1f} blks/s; {}/s)", + height, end_height, interval_duration.count(), + get_block_data_interval_duration.count(), + mnl_interval_duration.count(), + bns_interval_duration.count(), + blocks_per_s, + tools::get_human_readable_bytes(bytes_per_s))); + +#ifdef ENABLE_SYSTEMD + sd_notify(0, + ("EXTEND_TIMEOUT_USEC=120000000\nSTATUS=Rescanning blockchain; height " + + std::to_string(height)) + .c_str()); +#endif + + mnl_duration += mnl_interval_duration; + bns_duration += bns_interval_duration; + get_block_data_duration += get_block_data_interval_duration; + total_bytes += work_bytes; + + work_start = now; + get_block_data_interval_duration = mnl_interval_duration = bns_interval_duration = dseconds{0}; + work_blocks = work_bytes = 0; + } + + // Feed subsystem processing + work_blocks += chunk.blocks.size(); + work_bytes += chunk.size; + + for (size_t i = 0; i < chunk.blocks.size(); i++) + { + const auto &blk = chunk.blocks[i]; + uint64_t block_height = get_block_height(blk); + const auto &txs = chunk.txs[i]; + if (block_height >= mnl_height) { auto mnl_start = clock::now(); @@ -371,18 +630,21 @@ bool Blockchain::load_missing_blocks_into_beldex_subsystems() checkpoint_t *checkpoint_ptr = nullptr; checkpoint_t checkpoint; if (blk.major_version >= hf::hf15_flash && get_checkpoint(block_height, checkpoint)) - checkpoint_ptr = &checkpoint; + checkpoint_ptr = &checkpoint; - try { + try + { m_master_node_list.block_add(blk, txs, checkpoint_ptr); - } catch (const std::exception& e) { - MFATAL("Unable to process block {} for updating master node list: " << e.what()); + } + catch (const std::exception &e) + { + MFATAL("Unable to process block " << block_height << " for updating master node list: " << e.what()); return false; } - mnl_iteration_duration += clock::now() - mnl_start; + mnl_interval_duration += clock::now() - mnl_start; } - if (m_bns_db.db && (block_height >= bns_height)) + if (m_bns_db.db) { auto bns_start = clock::now(); if (!m_bns_db.add_block(blk, txs)) @@ -390,19 +652,46 @@ bool Blockchain::load_missing_blocks_into_beldex_subsystems() MERROR("Unable to process block for updating BNS DB: " << cryptonote::get_block_hash(blk)); return false; } - bns_iteration_duration += clock::now() - bns_start; + bns_interval_duration += clock::now() - bns_start; } } - } - if (total_blocks > 1) - { - MGINFO(fmt::format("Done recalculating beldex subsystems in {:.2f}s ({:.2f}s mnl; {:.2f}s bns)", - dseconds{clock::now() - scan_start}.count(), mnl_duration.count(), bns_duration.count())); - } + } // end main while + + auto end = clock::now(); if (total_blocks > 0) + { + if (m_master_node_list.height() != end_height - 1 || + (m_bns_db.db && m_bns_db.height() != end_height - 1)) + { + LOG_ERROR("Mismatched subsystem height after subsystem refresh: blockchain (" << end_height - 1 + << "), MNL (" << m_master_node_list.height() + << "), BNS (" << m_bns_db.height() << ")"); + return false; + } + + total_bytes += work_bytes; + dseconds duration{end - scan_start}; + float blocks_per_s = static_cast(total_blocks) / std::max(1e-9, duration.count()); + float bytes_per_s = static_cast(total_bytes) / std::max(1e-9, duration.count()); + MGINFO(fmt::format("Done recalculating beldex subsystems in {:.2f}s (get blks: {:.2f}s; mnl: {:.2f}s; bns: {:.2f}s; {:.1f} blks/s; {}/s)", + duration.count(), + get_block_data_duration.count(), + mnl_duration.count(), + bns_duration.count(), + blocks_per_s, + tools::get_human_readable_bytes(bytes_per_s))); + m_master_node_list.store(); + } + + if (use_threaded_load) + { + failure_rejoiner.cancel(); + assert(load_context.finished); + load_context.thread.join(); + } return true; } @@ -412,7 +701,9 @@ static bool exec_detach_hooks( uint64_t detach_height, std::vector hooks, // have to change bool by_pop_blocks, - bool load_missing_blocks = true) { + bool load_missing_blocks = true, + const std::atomic* abort = nullptr, + bool use_threaded_load = false) { detached_info hook_data{detach_height, by_pop_blocks}; for (const auto& hook : hooks) @@ -420,7 +711,7 @@ static bool exec_detach_hooks( bool result = true; if (load_missing_blocks) - result = blockchain.load_missing_blocks_into_beldex_subsystems(); + result = blockchain.load_missing_blocks_into_beldex_subsystems(abort, use_threaded_load); return result; } @@ -592,7 +883,9 @@ bool Blockchain::init(BlockchainDB* db, sqlite3 *bns_db, const network_type nett m_db->height(), m_blockchain_detached_hooks, /*by_pop_blocks*/ false, - /*load_missing_blocks_into_beldex_subsystems*/ true)) + /*load_missing_blocks_into_beldex_subsystems*/ true, + /* abort = */ nullptr, + /*use_threaded_load*/ true)) { MERROR("Failed to load blocks into beldex subsystems"); return false; @@ -2175,6 +2468,46 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return true; } + +bool Blockchain::_get_blocks_only( + uint64_t start_offset, + size_t count, + std::vector &blocks, + size_t *size_loaded) const +{ + const uint64_t height = m_db->height(); + if (size_loaded) + *size_loaded = 0; + + if (start_offset >= height) + return false; + + const size_t num_blocks = std::min(height - start_offset, count); + blocks.reserve(blocks.size() + num_blocks); + + for (size_t i = 0; i < num_blocks; i++) + { + try + { + size_t block_size = 0; + + // Load mainchain block by height + block blk = m_db->get_block_from_height(start_offset + i, &block_size); + + blocks.emplace_back(std::move(blk)); + + if (size_loaded) + *size_loaded += block_size; + } + catch (const std::exception &e) + { + LOG_ERROR("Invalid block at height " << start_offset + i << ". " << e.what()); + return false; + } + } + + return true; +} //------------------------------------------------------------------ bool Blockchain::get_blocks_only(uint64_t start_offset, size_t count, std::vector& blocks, std::vector* txs) const { @@ -2726,6 +3059,40 @@ bool Blockchain::get_split_transactions_blobs(const std::vector& t } return true; } +bool Blockchain::_get_transactions( + const std::vector& txs_ids, + std::vector& txs, + std::unordered_set* missed_txs, + size_t* total_size) const { + LOG_PRINT_L3("Blockchain::" << __func__); + + txs.reserve(txs_ids.size()); + + std::string blob; + if (total_size) + *total_size = 0; + + for (const auto& tx_hash : txs_ids) { + blob.clear(); + try { + if (m_db->get_tx_blob(tx_hash, blob)) { + if (total_size) + *total_size += blob.size(); + txs.emplace_back(); + if (!parse_and_validate_tx_from_blob(blob, txs.back())) { + LOG_ERROR("Invalid transaction"); + return false; + } + txs.back().set_hash(tx_hash); + txs.back().set_blob_size(blob.size()); + } else if (missed_txs) + missed_txs->insert(tx_hash); + } catch (const std::exception& e) { + return false; + } + } + return true; +} //------------------------------------------------------------------ bool Blockchain::get_transactions(const std::vector& txs_ids, std::vector& txs, std::unordered_set* missed_txs) const { @@ -2747,6 +3114,8 @@ bool Blockchain::get_transactions(const std::vector& txs_ids, std: LOG_ERROR("Invalid transaction"); return false; } + txs.back().set_hash(tx_hash); + txs.back().set_blob_size(tx.size()); } else if (missed_txs) missed_txs->insert(tx_hash); @@ -5766,4 +6135,4 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ m_btc_expected_reward = expected_reward; m_btc_pool_cookie = pool_cookie; m_btc_valid = true; -} +} \ No newline at end of file diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 3597991e9a6..e3fd23319af 100755 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -166,6 +166,11 @@ namespace cryptonote */ bool deinit(); + bool _get_blocks_only( + uint64_t start_offset, + size_t count, + std::vector& blocks, + size_t* blocks_size) const; bool get_blocks_only(uint64_t start_offset, size_t count, std::vector& blocks, std::vector *txs = nullptr) const; /** * @brief get blocks and transactions from blocks based on start height and count @@ -741,6 +746,11 @@ namespace cryptonote */ bool get_transactions_blobs(const std::vector& txs_ids, std::vector& txs, std::unordered_set* missed_txs = nullptr, bool pruned = false) const; bool get_split_transactions_blobs(const std::vector& txs_ids, std::vector>& txs, std::unordered_set* missed_txs = nullptr) const; + bool _get_transactions( + const std::vector& txs_ids, + std::vector& txs, + std::unordered_set* missed_txs, + size_t* total_size) const; bool get_transactions(const std::vector& txs_ids, std::vector& txs, std::unordered_set* missed_txs = nullptr) const; /** @@ -1028,7 +1038,7 @@ namespace cryptonote */ void flush_invalid_blocks(); - bool load_missing_blocks_into_beldex_subsystems(); + bool load_missing_blocks_into_beldex_subsystems(const std::atomic* abort = nullptr, bool use_threaded_load = false); #ifndef IN_UNIT_TESTS private: From 5bcf945da1d2e4cb5cae043bb985c780e2db9389 Mon Sep 17 00:00:00 2001 From: abdevil Date: Mon, 6 Apr 2026 17:59:47 +0530 Subject: [PATCH 2/2] tune scanning chunk size and queue limit --- src/cryptonote_core/blockchain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 1839ecb4886..1b67661e0ef 100755 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -310,8 +310,8 @@ struct block_data }; struct block_load_context { - static constexpr uint64_t CHUNK_SIZE = 300; // tuneable chunk size - static constexpr size_t MAX_QUEUE_SIZE = 12; // tuneable queue depth + static constexpr uint64_t CHUNK_SIZE = 50; // tuneable chunk size + static constexpr size_t MAX_QUEUE_SIZE = 5; // tuneable queue depth std::mutex block_mut; std::condition_variable block_cv; std::queue next_blocks;