From 926549e740e6150adaa3952f8f07c49945e9f013 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Wed, 12 Mar 2025 19:51:51 +0400 Subject: [PATCH] implemented warp-mode for zano daemon --- src/common/db_abstract_accessor.h | 2 +- src/common/util.cpp | 57 +++++++++++++++++++ src/common/util.h | 3 + src/currency_core/blockchain_storage.cpp | 51 +++++++++-------- src/currency_core/blockchain_storage.h | 7 ++- src/currency_core/blockchain_storage_basic.h | 20 +++++++ .../currency_format_utils_blocks.cpp | 1 + .../currency_format_utils_blocks.h | 1 - src/daemon/daemon.cpp | 44 +++++++++++++- 9 files changed, 156 insertions(+), 30 deletions(-) diff --git a/src/common/db_abstract_accessor.h b/src/common/db_abstract_accessor.h index c8b6a756..b26c4961 100644 --- a/src/common/db_abstract_accessor.h +++ b/src/common/db_abstract_accessor.h @@ -776,7 +776,7 @@ namespace tools return m_cache.get_max_elements(); } - void set_cache_size(uint64_t max_cache_size) + void set_cache_size(uint64_t max_cache_size) const { m_cache.set_max_elements(max_cache_size); } diff --git a/src/common/util.cpp b/src/common/util.cpp index f87f0582..2a8dd71d 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -784,4 +784,61 @@ std::string get_nix_version_display_string() return true; } + uint64_t get_total_system_memory() + { +#if defined(_WIN32) || defined(_WIN64) + // Windows + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + if (GlobalMemoryStatusEx(&status)) { + return static_cast(status.ullTotalPhys); + } + return 0; + +#elif defined(__APPLE__) || defined(__MACH__) || defined(__linux__) || defined(__unix__) + // POSIX (Linux, macOS, etc.) + // On most Unix-like systems, sysconf(_SC_PHYS_PAGES) and sysconf(_SC_PAGE_SIZE) + // will give total number of pages and page size in bytes. + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGE_SIZE); + if (pages == -1 || page_size == -1) { + return 0; + } + return static_cast(pages) * static_cast(page_size); +#else + // Fallback for other OS + return 0; +#endif + } + + std::pair pretty_print_big_nums_to_pair(std::uint64_t num) + { + // 1024-based suffixes + static const char* suffixes[] = { "", "K", "M", "G", "T", "P", "E" }; + + double count = static_cast(num); + int i = 0; + // Loop until we find the largest suffix that fits + while (count >= 1024.0 && i < (static_cast(sizeof(suffixes) / sizeof(suffixes[0])) - 1)) + { + count /= 1024.0; + i++; + } + + // Format with 2 decimal places (you can adjust as you like) + std::ostringstream os; + std::pair res; + os << std::fixed << std::setprecision(2) << count; + res.first = os.str(); + res.second = suffixes[i]; + return res; + } + + std::string pretty_print_big_nums(std::uint64_t num) + { + auto pr = pretty_print_big_nums_to_pair(num); + return pr.first + " " + pr.second; + } + + } // namespace tools diff --git a/src/common/util.h b/src/common/util.h index 7761775d..cfa5b19c 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -44,6 +44,9 @@ namespace tools bool create_directories_if_necessary(const std::string& path); std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name); + uint64_t get_total_system_memory(); + std::string pretty_print_big_nums(std::uint64_t num); + std::pair pretty_print_big_nums_to_pair(std::uint64_t num); inline crypto::hash get_proof_of_trust_hash(const nodetool::proof_of_trust& pot) { diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 602719a3..675682cf 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -73,11 +73,8 @@ using namespace currency; DISABLE_VS_WARNINGS(4267) -namespace -{ - const command_line::arg_descriptor arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file"); - const command_line::arg_descriptor arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers"); -} +const command_line::arg_descriptor arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file"); +const command_line::arg_descriptor arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers"); //------------------------------------------------------------------ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m_rw_lock), @@ -205,6 +202,21 @@ bool blockchain_storage::validate_instance(const std::string& path) } } //------------------------------------------------------------------ +void blockchain_storage::set_db_l2_cache_size(uint64_t ceched_elements) const +{ + LOG_PRINT_GREEN("Using db items cache size(L2): " << tools::pretty_print_big_nums(ceched_elements) << " items", LOG_LEVEL_0); + m_db_blocks_index.set_cache_size(ceched_elements); + m_db_blocks.set_cache_size(ceched_elements); + m_db_blocks_index.set_cache_size(ceched_elements); + m_db_transactions.set_cache_size(ceched_elements); + m_db_spent_keys.set_cache_size(ceched_elements); + //m_db_outputs.set_cache_size(ceched_elements); + m_db_multisig_outs.set_cache_size(ceched_elements); + m_db_solo_options.set_cache_size(ceched_elements); + m_db_aliases.set_cache_size(ceched_elements); + m_db_assets.set_cache_size(ceched_elements); + m_db_addr_to_alias.set_cache_size(ceched_elements); +} bool blockchain_storage::init(const std::string& config_folder, const boost::program_options::variables_map& vm) { // CRITICAL_REGION_LOCAL(m_read_lock); @@ -285,18 +297,7 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro if (command_line::has_arg(vm, arg_db_cache_l2)) { uint64_t cache_size = command_line::get_arg(vm, arg_db_cache_l2); - LOG_PRINT_GREEN("Using db items cache size(L2): " << cache_size, LOG_LEVEL_0); - m_db_blocks_index.set_cache_size(cache_size); - m_db_blocks.set_cache_size(cache_size); - m_db_blocks_index.set_cache_size(cache_size); - m_db_transactions.set_cache_size(cache_size); - m_db_spent_keys.set_cache_size(cache_size); - //m_db_outputs.set_cache_size(cache_size); - m_db_multisig_outs.set_cache_size(cache_size); - m_db_solo_options.set_cache_size(cache_size); - m_db_aliases.set_cache_size(cache_size); - m_db_assets.set_cache_size(cache_size); - m_db_addr_to_alias.set_cache_size(cache_size); + set_db_l2_cache_size(cache_size); } LOG_PRINT_L0("Opened DB ver " << m_db_storage_major_compatibility_version << "." << m_db_storage_minor_compatibility_version); @@ -2423,7 +2424,7 @@ bool blockchain_storage::get_main_block_rpc_details(uint64_t i, block_rpc_extend bei.total_txs_size = 0; if (true/*!ignore_transactions*/) { - crypto::hash coinbase_id = get_transaction_hash(core_bei_ptr->bl.miner_tx); + crypto::hash coinbase_id = get_coinbase_hash_cached(*core_bei_ptr); //load transactions details bei.transactions_details.push_back(tx_rpc_extended_info()); get_tx_rpc_details(coinbase_id, bei.transactions_details.back(), get_block_datetime(core_bei_ptr->bl), true); @@ -2508,7 +2509,7 @@ bool blockchain_storage::get_alt_block_rpc_details(const block_extended_info& be bei.is_orphan = true; - crypto::hash coinbase_id = get_transaction_hash(bei_core.bl.miner_tx); + crypto::hash coinbase_id = get_coinbase_hash_cached(bei_core); //load transactions details bei.transactions_details.push_back(tx_rpc_extended_info()); fill_tx_rpc_details(bei.transactions_details.back(), bei_core.bl.miner_tx, nullptr, coinbase_id, get_block_datetime(bei_core.bl)); @@ -3300,7 +3301,7 @@ void blockchain_storage::print_blockchain_with_tx(uint64_t start_index, uint64_t << ", id: " << get_block_hash(m_db_blocks[i]->bl) << ", difficulty: " << block_difficulty(i) << ", nonce " << m_db_blocks[i]->bl.nonce << ", tx_count " << m_db_blocks[i]->bl.tx_hashes.size() << ENDL; - ss << "[miner id]: " << get_transaction_hash(m_db_blocks[i]->bl.miner_tx) << ENDL << currency::obj_to_json_str(m_db_blocks[i]->bl.miner_tx) << ENDL; + ss << "[miner id]: " << get_coinbase_hash_cached(*m_db_blocks[i]) << ENDL << currency::obj_to_json_str(m_db_blocks[i]->bl.miner_tx) << ENDL; for (size_t j = 0; j != m_db_blocks[i]->bl.tx_hashes.size(); j++) { @@ -3696,7 +3697,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list mis; get_transactions_direct(m_db_blocks[i]->bl.tx_hashes, blocks.back().second, mis); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, block " << get_block_hash(m_db_blocks[i]->bl) << " [" << i << "] contains missing transactions: " << mis); - blocks.back().third = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx)); + blocks.back().third = m_db_transactions.find(get_coinbase_hash_cached(*m_db_blocks[i])); } return true; } @@ -5350,7 +5351,7 @@ void blockchain_storage::do_full_db_warm_up() const LOG_ERROR("some tx's not found"); return; } - auto coinbase_tx_ptr = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx)); + auto coinbase_tx_ptr = m_db_transactions.find(get_coinbase_hash_cached(*m_db_blocks[i])); if (!coinbase_tx_ptr) { LOG_ERROR("Coinbase not found"); @@ -5374,7 +5375,7 @@ void blockchain_storage::do_full_db_warm_up() const PRINT_CONTAINER(m_db_aliases); PRINT_CONTAINER(m_db_assets); PRINT_CONTAINER(m_db_addr_to_alias); - LOG_PRINT_CYAN("CACHE STATE: " << ENDL << strm.str(), LOG_LEVEL_0); + LOG_PRINT_CYAN("CACHE STATE: " << ENDL << strm.str(), LOG_LEVEL_2); } } } @@ -5910,7 +5911,7 @@ std::shared_ptr blockchain_storage::find_key_imag { if (boost::get(in).k_image == ki) { - id_result = get_transaction_hash(block_entry->bl.miner_tx); + id_result = get_coinbase_hash_cached(*block_entry); return get_tx_chain_entry(id_result); } } @@ -8400,7 +8401,7 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(on_board_tx.first, txs_by_id_and_height_altchain::value_type::second_type(on_board_tx.second, ch->second.height))); } //TODO: consider performance optimization (get_transaction_hash might slow down deep reorganizations ) - alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(get_transaction_hash(ch->second.bl.miner_tx), txs_by_id_and_height_altchain::value_type::second_type(ch->second.bl.miner_tx, ch->second.height))); + alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(get_coinbase_hash_cached(ch->second), txs_by_id_and_height_altchain::value_type::second_type(ch->second.bl.miner_tx, ch->second.height))); } } else diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index fdf8e713..31211aed 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -8,13 +8,13 @@ #pragma once #include #include +#include #include #include #include #include - #include #include @@ -23,6 +23,7 @@ #include "serialization/string.h" #include "serialization/multiprecision.h" +#include "common/command_line.h" #include "tx_pool.h" #include "blockchain_storage_basic.h" #include "common/util.h" @@ -49,6 +50,8 @@ MARK_AS_POD_C11(macro_alias_1); #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "core" +extern const command_line::arg_descriptor arg_db_cache_l2; + namespace currency { @@ -507,7 +510,7 @@ namespace currency bool print_tx_outputs_lookup(const crypto::hash& tx_id) const; uint64_t get_last_x_block_height(bool pos)const; bool is_tx_spendtime_unlocked(uint64_t unlock_time)const; - + void set_db_l2_cache_size(uint64_t ceched_elements) const; //experimental void do_full_db_warm_up() const; diff --git a/src/currency_core/blockchain_storage_basic.h b/src/currency_core/blockchain_storage_basic.h index c15cdbe9..102f8fb2 100644 --- a/src/currency_core/blockchain_storage_basic.h +++ b/src/currency_core/blockchain_storage_basic.h @@ -73,6 +73,10 @@ namespace currency FIELD(this_block_tx_fee_median) FIELD(effective_tx_fee_median) END_SERIALIZE() + + // This is an optional data fields, It is not included in serialization and therefore is never stored in the database. + // It might be calculated “on the fly” to speed up access operations. + mutable std::shared_ptr m_cache_coinbase_id; }; struct gindex_increment @@ -185,4 +189,20 @@ namespace currency END_KV_SERIALIZE_MAP() }; + + + + + inline crypto::hash get_coinbase_hash_cached(const block_extended_info& bei) + { + if (bei.m_cache_coinbase_id) + { + return *bei.m_cache_coinbase_id; + } + else + { + bei.m_cache_coinbase_id = std::make_shared(get_transaction_hash(bei.bl.miner_tx)); + return *bei.m_cache_coinbase_id; + } + } } // namespace currency diff --git a/src/currency_core/currency_format_utils_blocks.cpp b/src/currency_core/currency_format_utils_blocks.cpp index 30d7f261..85bcddfd 100644 --- a/src/currency_core/currency_format_utils_blocks.cpp +++ b/src/currency_core/currency_format_utils_blocks.cpp @@ -65,4 +65,5 @@ namespace currency txs_ids.push_back(th); return get_tx_tree_hash(txs_ids); } + } diff --git a/src/currency_core/currency_format_utils_blocks.h b/src/currency_core/currency_format_utils_blocks.h index e1a3eb41..130b51dc 100644 --- a/src/currency_core/currency_format_utils_blocks.h +++ b/src/currency_core/currency_format_utils_blocks.h @@ -21,5 +21,4 @@ namespace currency void get_tx_tree_hash(const std::vector& tx_hashes, crypto::hash& h); crypto::hash get_tx_tree_hash(const std::vector& tx_hashes); crypto::hash get_tx_tree_hash(const block& b); - } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 7ebb96c0..f2a48056 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -37,10 +37,14 @@ using namespace epee; #endif +const uint64_t min_ram_for_full_warp_mode = 32ULL * 1024ULL * 1024ULL * 1024ULL; +const uint64_t recommended_ram_for_full_warp_mode = 64ULL * 1024ULL * 1024ULL * 1024ULL; + //TODO: need refactoring here. (template classes can't be used in BOOST_CLASS_VERSION) BOOST_CLASS_VERSION(nodetool::node_server >, CURRENT_P2P_STORAGE_ARCHIVE_VER); const command_line::arg_descriptor arg_rpc_server_threads("rpc-server-threads", "Specify number of RPC server threads. Default: 10", RPC_SERVER_DEFAULT_THREADS_NUM); +const command_line::arg_descriptor arg_do_warp_mode("do-warp-mode", "This option pre-loads and unserialize all data into RAM and provide significant speed increase in RPC-handling, requires 32GB psychical RAM at least(64GB recommended). Might be helpful for production servers(like remote nodes or public nodes for mobile apps)."); namespace po = boost::program_options; @@ -166,8 +170,9 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload); command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link); command_line::add_arg(desc_cmd_sett, command_line::arg_disable_ntp); + command_line::add_arg(desc_cmd_sett, arg_rpc_server_threads); - + command_line::add_arg(desc_cmd_sett, arg_do_warp_mode); arg_market_disable.default_value = true; arg_market_disable.use_default = true; @@ -372,6 +377,43 @@ int main(int argc, char* argv[]) res = ccore.set_checkpoints(std::move(checkpoints)); CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core"); + + //do full warp mode if needed + if (command_line::has_arg(vm, arg_do_warp_mode)) + { + LOG_PRINT_MAGENTA("Initializing full warp-mode", LOG_LEVEL_0); + //let's check if cache size were specifically set + if (!command_line::has_arg(vm, arg_db_cache_l2)) + { + //overriding caching settings + uint64_t cache_size = ccore.get_blockchain_storage().get_total_transactions() * 10; + ccore.get_blockchain_storage().set_db_l2_cache_size(cache_size); + + LOG_PRINT_MAGENTA("[Warp]: Setting up db cache to " << tools::pretty_print_big_nums(cache_size) << " items.....", LOG_LEVEL_0); + } + uint64_t phisical_ram_detected = tools::get_total_system_memory(); + if (phisical_ram_detected < min_ram_for_full_warp_mode) + { + LOG_PRINT_RED_L0("[Warp]: Detected only " << tools::pretty_print_big_nums(phisical_ram_detected) << "B of RAM, it's not recommended to run daemon in full warm up mode under " << tools::pretty_print_big_nums(min_ram_for_full_warp_mode) << "B, stopping..."); + return 1; + } + else + { + if(phisical_ram_detected < recommended_ram_for_full_warp_mode) + { + LOG_PRINT_MAGENTA("[Warp]: Detected only " << tools::pretty_print_big_nums(phisical_ram_detected) << "B RAM, might be not optimal, recommended above " << tools::pretty_print_big_nums(recommended_ram_for_full_warp_mode) << "B", LOG_LEVEL_0); + } + else + { + LOG_PRINT_GREEN("[Warp]: Detected " << tools::pretty_print_big_nums(phisical_ram_detected) << "B RAM", LOG_LEVEL_0); + } + } + + LOG_PRINT_MAGENTA("[Warp]: Launching warm up....", LOG_LEVEL_0); + ccore.get_blockchain_storage().do_full_db_warm_up(); + LOG_PRINT_MAGENTA("[Warp]: Warm up finished!", LOG_LEVEL_0); + } + // start components if (!command_line::has_arg(vm, command_line::arg_console)) {