From ec54dc72c9465dafd35385209ad77ac552f10bcf Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Wed, 12 Mar 2025 14:17:20 +0400 Subject: [PATCH] added full warm up feature --- contrib/epee/include/cache_helper.h | 7 +++- src/common/command_line.cpp | 1 + src/common/db_abstract_accessor.h | 15 ++++++++ src/currency_core/blockchain_storage.cpp | 48 ++++++++++++++++++++++++ src/currency_core/blockchain_storage.h | 3 ++ src/daemon/daemon.cpp | 28 +++++++++++++- src/daemon/daemon_commands_handler.h | 9 +++++ src/p2p/net_node.h | 5 ++- src/p2p/net_node.inl | 17 +++++++++ src/rpc/core_rpc_server.cpp | 28 +++++++++----- src/rpc/core_rpc_server.h | 2 +- 11 files changed, 148 insertions(+), 15 deletions(-) diff --git a/contrib/epee/include/cache_helper.h b/contrib/epee/include/cache_helper.h index 0da21621..dd3bb883 100644 --- a/contrib/epee/include/cache_helper.h +++ b/contrib/epee/include/cache_helper.h @@ -70,12 +70,17 @@ namespace epee cache_base() : mac_allowed_elements(max_elements) {} - size_t size() + size_t size() const { return data.size(); } + uint64_t get_max_elements() const + { + return mac_allowed_elements; + } + void set_max_elements(uint64_t e) { mac_allowed_elements = e; diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index ae22e874..ed435352 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -40,6 +40,7 @@ namespace command_line const arg_descriptor arg_process_predownload_from_path("predownload-from-local-path", "Instead of downloading file use downloaded local file"); const arg_descriptor arg_validate_predownload ( "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database"); const arg_descriptor arg_predownload_link ( "predownload-link", "Override url for blockchain database pre-downloading"); + const arg_descriptor arg_deeplink ( "deeplink-params", "Deeplink parameter, in that case app just forward params to running app"); diff --git a/src/common/db_abstract_accessor.h b/src/common/db_abstract_accessor.h index 0a6ee08b..c8b6a756 100644 --- a/src/common/db_abstract_accessor.h +++ b/src/common/db_abstract_accessor.h @@ -766,6 +766,16 @@ namespace tools m_cache.clear(); } + uint64_t get_cacheed_items_count() const + { + return m_cache.size(); + } + + uint64_t get_cache_size() const + { + return m_cache.get_max_elements(); + } + void set_cache_size(uint64_t max_cache_size) { m_cache.set_max_elements(max_cache_size); @@ -806,6 +816,11 @@ namespace tools } return res; } + + std::shared_ptr find(const t_key& k) const + { + return get(k); + } size_t clear() { diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index c13f8cbd..602719a3 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -5331,6 +5331,54 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) const return currency::is_tx_spendtime_unlocked(unlock_time, get_current_blockchain_size(), m_core_runtime_config.get_core_time()); } //------------------------------------------------------------------ +void blockchain_storage::do_full_db_warm_up() const +{ + + uint64_t ticks_last_print = 0; + uint64_t current_sz = get_current_blockchain_size(); + LOG_PRINT_CYAN("Warming up starting, total blocks..." << current_sz, LOG_LEVEL_0); + for (uint64_t i = current_sz - 1; i != 0; --i) + { + CRITICAL_REGION_LOCAL(m_read_lock); + auto blk_ptr = m_db_blocks[i]; + std::list mis; + std::list > txs; + get_transactions_direct(blk_ptr->bl.tx_hashes, txs, mis); + CHECK_AND_ASSERT_MES(!mis.size(), void(), "internal error, block " << get_block_hash(m_db_blocks[i]->bl) << " [" << i << "] contains missing transactions: " << mis); + if (txs.size() != blk_ptr->bl.tx_hashes.size()) + { + LOG_ERROR("some tx's not found"); + return; + } + auto coinbase_tx_ptr = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx)); + if (!coinbase_tx_ptr) + { + LOG_ERROR("Coinbase not found"); + return; + } + if (epee::misc_utils::get_tick_count() - ticks_last_print > 1000) + { + ticks_last_print = epee::misc_utils::get_tick_count(); + LOG_PRINT_CYAN("Warming up: " << ( ( (current_sz - i) * 100)/ current_sz) << "%, " << (current_sz - i) << " of " << current_sz, LOG_LEVEL_0); + +#define PRINT_CONTAINER(cont_name) strm << #cont_name"[" << cont_name.get_cache_size() << "]:" << (cont_name.get_cacheed_items_count() * 100) / cont_name.get_cache_size() << " %, items: " << cont_name.get_cacheed_items_count() << ENDL + + std::stringstream strm; + PRINT_CONTAINER(m_db_blocks_index); + PRINT_CONTAINER(m_db_blocks); + PRINT_CONTAINER(m_db_blocks_index); + PRINT_CONTAINER(m_db_transactions); + PRINT_CONTAINER(m_db_spent_keys); + PRINT_CONTAINER(m_db_multisig_outs); + PRINT_CONTAINER(m_db_solo_options); + PRINT_CONTAINER(m_db_aliases); + PRINT_CONTAINER(m_db_assets); + PRINT_CONTAINER(m_db_addr_to_alias); + LOG_PRINT_CYAN("CACHE STATE: " << ENDL << strm.str(), LOG_LEVEL_0); + } + } +} +//------------------------------------------------------------------ bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase) const { CRITICAL_REGION_LOCAL(m_read_lock); diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index c98e86eb..fdf8e713 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -508,6 +508,9 @@ namespace currency uint64_t get_last_x_block_height(bool pos)const; bool is_tx_spendtime_unlocked(uint64_t unlock_time)const; + //experimental + void do_full_db_warm_up() const; + private: //-------------- DB containers -------------- diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 4e5d0de0..7ebb96c0 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -40,6 +40,7 @@ using namespace epee; //TODO: need refactoring here. (template classes can't be used in BOOST_CLASS_VERSION) BOOST_CLASS_VERSION(nodetool::node_server >, CURRENT_P2P_STORAGE_ARCHIVE_VER); +const command_line::arg_descriptor arg_rpc_server_threads("rpc-server-threads", "Specify number of RPC server threads. Default: 10", RPC_SERVER_DEFAULT_THREADS_NUM); namespace po = boost::program_options; @@ -165,7 +166,8 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload); command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link); command_line::add_arg(desc_cmd_sett, command_line::arg_disable_ntp); - + command_line::add_arg(desc_cmd_sett, arg_rpc_server_threads); + arg_market_disable.default_value = true; arg_market_disable.use_default = true; @@ -184,6 +186,8 @@ int main(int argc, char* argv[]) po::options_description desc_options("Allowed options"); desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; bool exit_requested = false; bool r = command_line::handle_error_helper(desc_options, [&]() @@ -374,8 +378,28 @@ int main(int argc, char* argv[]) dch.start_handling(); } + uint32_t rpc_threads_count = RPC_SERVER_DEFAULT_THREADS_NUM; + if (command_line::has_arg(vm, arg_rpc_server_threads)) + { + rpc_threads_count = command_line::get_arg(vm, arg_rpc_server_threads); + + if (rpc_threads_count < 2) + { + LOG_ERROR("RPC server threads number number is too low: " << rpc_threads_count << "(why would you do that?!)"); + return false; + } + LOG_PRINT_L0("RPC server threads number is overridden with " << rpc_threads_count); + if (rpc_threads_count < RPC_SERVER_DEFAULT_THREADS_NUM) + { + LOG_PRINT_RED_L0("Warning: RPC server threads number is overridden with low number: " << rpc_threads_count << "(why would you do that?!)"); + } + + + } + + LOG_PRINT_L0("Starting core rpc server..."); - res = rpc_server.run(2, false); + res = rpc_server.run(rpc_threads_count, false); CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core rpc server."); LOG_PRINT_L0("Core rpc server started ok"); diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index 2920c118..cab2be2b 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -79,6 +79,7 @@ public: m_cmd_binder.set_handler("print_tx_outputs_usage", boost::bind(&daemon_commands_handler::print_tx_outputs_usage, this, ph::_1), "Analyse if tx outputs for involved in subsequent transactions"); m_cmd_binder.set_handler("print_difficulties_of_last_n_blocks", boost::bind(&daemon_commands_handler::print_difficulties_of_last_n_blocks, this, ph::_1), "Print difficulties of last n blocks"); m_cmd_binder.set_handler("debug_remote_node_mode", boost::bind(&daemon_commands_handler::debug_remote_node_mode, this, ph::_1), " - If node got connected put node into 'debug mode' i.e. no sync process of other communication except ping responses, maintenance secrete key will be requested"); + m_cmd_binder.set_handler("full_db_cache_warmup", boost::bind(&daemon_commands_handler::full_db_cache_warmup, this, ph::_1), "(Experimental) Perform full DB loading to RAM cache(make sense only with big numbers passed to --db-cache-l2 option)"); #ifdef _DEBUG m_cmd_binder.set_handler("debug_set_time_adj", boost::bind(&daemon_commands_handler::debug_set_time_adj, this, ph::_1), "DEBUG: set core time adjustment"); #endif @@ -883,6 +884,14 @@ private: return true; } //-------------------------------------------------------------------------------- + bool full_db_cache_warmup(const std::vector& args) + { + + m_srv.get_payload_object().get_core().get_blockchain_storage().do_full_db_warm_up(); + + return true; + } + //-------------------------------------------------------------------------------- bool print_pool(const std::vector& args) { LOG_PRINT_L0("Pool state: " << ENDL << m_srv.get_payload_object().get_core().print_pool(false)); diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 38b5a5ab..a238f63f 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -36,6 +36,8 @@ ENABLE_CHANNEL_BY_DEFAULT(LOG_DEFAULT_CHANNEL); #define CURRENT_P2P_STORAGE_ARCHIVE_VER (CURRENCY_FORMATION_VERSION+13) +#define P2P_SERVER_DEFAULT_THREADS_NUM 10 + PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4355) @@ -249,8 +251,9 @@ namespace nodetool bool m_hide_my_port; bool m_offline_mode; bool m_debug_requests_enabled; - bool m_ip_auto_blocking_enabled; + bool m_ip_auto_blocking_enabled = false; uint64_t m_startup_time; + uint32_t m_threads_count = P2P_SERVER_DEFAULT_THREADS_NUM; //critical_section m_connections_lock; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index fb1c4109..1bb27fbb 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -38,6 +38,7 @@ namespace nodetool const command_line::arg_descriptor arg_p2p_offline_mode ( "offline-mode", "Don't connect to any node and reject any connections"); const command_line::arg_descriptor arg_p2p_disable_debug_reqs ( "disable-debug-p2p-requests", "Disable p2p debug requests"); const command_line::arg_descriptor arg_p2p_ip_auto_blocking ( "p2p-ip-auto-blocking", "Enable (1) or disable (0) peers auto-blocking by IP <0|1>. Default: 0", 1); + const command_line::arg_descriptor arg_p2p_server_threads ( "p2p-server-threads", "Specify number of p2p server threads. Default: 10", P2P_SERVER_DEFAULT_THREADS_NUM); } //----------------------------------------------------------------------------------- @@ -56,6 +57,7 @@ namespace nodetool command_line::add_arg(desc, arg_p2p_disable_debug_reqs); command_line::add_arg(desc, arg_p2p_use_only_priority_nodes); command_line::add_arg(desc, arg_p2p_ip_auto_blocking); + command_line::add_arg(desc, arg_p2p_server_threads); } //----------------------------------------------------------------------------------- template @@ -195,6 +197,21 @@ namespace nodetool m_offline_mode = command_line::get_arg(vm, arg_p2p_offline_mode); m_debug_requests_enabled = !command_line::get_arg(vm, arg_p2p_disable_debug_reqs); m_ip_auto_blocking_enabled = (command_line::get_arg(vm, arg_p2p_ip_auto_blocking) != 0); + + if (command_line::has_arg(vm, arg_p2p_server_threads)) + { + m_threads_count = command_line::get_arg(vm, arg_p2p_server_threads); + if (m_threads_count < 2) + { + LOG_ERROR("P2P server threads number number is too low: " << m_threads_count << "(why would you do that?!)"); + return false; + } + LOG_PRINT_L0("P2P server threads number is overridden with " << m_threads_count); + if (m_threads_count < P2P_SERVER_DEFAULT_THREADS_NUM) + { + LOG_PRINT_RED_L0("Warning: P2P server threads number is overridden with low number: " << m_threads_count << "(why would you do that?!)"); + } + } LOG_PRINT_L0("p2p peers auto-blocking is " << (m_ip_auto_blocking_enabled ? "enabled" : "disabled")); diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 6306d3ac..283a800c 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -102,12 +102,19 @@ namespace currency res.current_blocks_median = m_core.get_blockchain_storage().get_current_comulative_blocksize_limit() / 2; res.alias_count = m_core.get_blockchain_storage().get_aliases_count(); res.current_max_allowed_block_size = m_core.get_blockchain_storage().get_current_comulative_blocksize_limit(); - if (!res.outgoing_connections_count) - res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_connecting; - else if (m_p2p.get_payload_object().is_synchronized()) + if(m_ignore_offline_status) + { res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_online; + } else - res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_synchronizing; + { + if (!res.outgoing_connections_count) + res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_connecting; + else if (m_p2p.get_payload_object().is_synchronized()) + res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_online; + else + res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_synchronizing; + } res.synchronization_start_height = m_p2p.get_payload_object().get_core_inital_height(); res.max_net_seen_height = m_p2p.get_payload_object().get_max_seen_height(); m_p2p.get_maintainers_info(res.mi); @@ -311,7 +318,7 @@ namespace currency bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx) { CHECK_CORE_READY(); - + LOG_PRINT_L2("[on_get_blocks]: Prevalidating...."); if (req.block_ids.back() != m_core.get_blockchain_storage().get_block_id_by_height(0)) { //genesis mismatch, return specific @@ -325,7 +332,7 @@ namespace currency res.status = API_RETURN_CODE_BAD_ARG; return true; } - + LOG_PRINT_L2("[on_get_blocks]: find_blockchain_supplement ...."); blockchain_storage::blocks_direct_container bs; if (!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height)) { @@ -333,23 +340,24 @@ namespace currency return false; } + LOG_PRINT_L2("[on_get_blocks]: Enumerating over blocks ...."); for (auto& b : bs) { - res.blocks.resize(res.blocks.size()+1); + res.blocks.resize(res.blocks.size() + 1); res.blocks.back().block = block_to_blob(b.first->bl); CHECK_AND_ASSERT_MES(b.third.get(), false, "Internal error on handling COMMAND_RPC_GET_BLOCKS_FAST: b.third is empty, ie coinbase info is not prepared"); res.blocks.back().coinbase_global_outs = b.third->m_global_output_indexes; res.blocks.back().tx_global_outs.resize(b.second.size()); size_t i = 0; - - BOOST_FOREACH(auto& t, b.second) + + BOOST_FOREACH(auto & t, b.second) { res.blocks.back().txs.push_back(tx_to_blob(t->tx)); res.blocks.back().tx_global_outs[i].v = t->m_global_output_indexes; i++; } } - + LOG_PRINT_L2("[on_get_blocks]: Finished"); res.status = API_RETURN_CODE_OK; return true; } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 49fd54c6..048b5236 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -16,7 +16,7 @@ #include "currency_protocol/currency_protocol_handler.h" #include "currency_core/bc_offers_service.h" - +#define RPC_SERVER_DEFAULT_THREADS_NUM 10 #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "rpc"