forked from lthn/blockchain
implemented warp-mode for zano daemon
This commit is contained in:
cryptozoidberg
parent
a588173605
commit
926549e740
9 changed files with 156 additions and 30 deletions
|
|
@ -776,7 +776,7 @@ namespace tools
|
|||
return m_cache.get_max_elements();
|
||||
}
|
||||
|
||||
void set_cache_size(uint64_t max_cache_size)
|
||||
void set_cache_size(uint64_t max_cache_size) const
|
||||
{
|
||||
m_cache.set_max_elements(max_cache_size);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -784,4 +784,61 @@ std::string get_nix_version_display_string()
|
|||
return true;
|
||||
}
|
||||
|
||||
uint64_t get_total_system_memory()
|
||||
{
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
// Windows
|
||||
MEMORYSTATUSEX status;
|
||||
status.dwLength = sizeof(status);
|
||||
if (GlobalMemoryStatusEx(&status)) {
|
||||
return static_cast<std::uint64_t>(status.ullTotalPhys);
|
||||
}
|
||||
return 0;
|
||||
|
||||
#elif defined(__APPLE__) || defined(__MACH__) || defined(__linux__) || defined(__unix__)
|
||||
// POSIX (Linux, macOS, etc.)
|
||||
// On most Unix-like systems, sysconf(_SC_PHYS_PAGES) and sysconf(_SC_PAGE_SIZE)
|
||||
// will give total number of pages and page size in bytes.
|
||||
long pages = sysconf(_SC_PHYS_PAGES);
|
||||
long page_size = sysconf(_SC_PAGE_SIZE);
|
||||
if (pages == -1 || page_size == -1) {
|
||||
return 0;
|
||||
}
|
||||
return static_cast<std::uint64_t>(pages) * static_cast<std::uint64_t>(page_size);
|
||||
#else
|
||||
// Fallback for other OS
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
std::pair<std::string, std::string> pretty_print_big_nums_to_pair(std::uint64_t num)
|
||||
{
|
||||
// 1024-based suffixes
|
||||
static const char* suffixes[] = { "", "K", "M", "G", "T", "P", "E" };
|
||||
|
||||
double count = static_cast<double>(num);
|
||||
int i = 0;
|
||||
// Loop until we find the largest suffix that fits
|
||||
while (count >= 1024.0 && i < (static_cast<int>(sizeof(suffixes) / sizeof(suffixes[0])) - 1))
|
||||
{
|
||||
count /= 1024.0;
|
||||
i++;
|
||||
}
|
||||
|
||||
// Format with 2 decimal places (you can adjust as you like)
|
||||
std::ostringstream os;
|
||||
std::pair<std::string, std::string> res;
|
||||
os << std::fixed << std::setprecision(2) << count;
|
||||
res.first = os.str();
|
||||
res.second = suffixes[i];
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string pretty_print_big_nums(std::uint64_t num)
|
||||
{
|
||||
auto pr = pretty_print_big_nums_to_pair(num);
|
||||
return pr.first + " " + pr.second;
|
||||
}
|
||||
|
||||
|
||||
} // namespace tools
|
||||
|
|
|
|||
|
|
@ -44,6 +44,9 @@ namespace tools
|
|||
|
||||
bool create_directories_if_necessary(const std::string& path);
|
||||
std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name);
|
||||
uint64_t get_total_system_memory();
|
||||
std::string pretty_print_big_nums(std::uint64_t num);
|
||||
std::pair<std::string, std::string> pretty_print_big_nums_to_pair(std::uint64_t num);
|
||||
|
||||
inline crypto::hash get_proof_of_trust_hash(const nodetool::proof_of_trust& pot)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -73,11 +73,8 @@ using namespace currency;
|
|||
|
||||
DISABLE_VS_WARNINGS(4267)
|
||||
|
||||
namespace
|
||||
{
|
||||
const command_line::arg_descriptor<uint32_t> arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file");
|
||||
const command_line::arg_descriptor<uint32_t> arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers");
|
||||
}
|
||||
const command_line::arg_descriptor<uint32_t> arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file");
|
||||
const command_line::arg_descriptor<uint32_t> arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers");
|
||||
|
||||
//------------------------------------------------------------------
|
||||
blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m_rw_lock),
|
||||
|
|
@ -205,6 +202,21 @@ bool blockchain_storage::validate_instance(const std::string& path)
|
|||
}
|
||||
}
|
||||
//------------------------------------------------------------------
|
||||
void blockchain_storage::set_db_l2_cache_size(uint64_t ceched_elements) const
|
||||
{
|
||||
LOG_PRINT_GREEN("Using db items cache size(L2): " << tools::pretty_print_big_nums(ceched_elements) << " items", LOG_LEVEL_0);
|
||||
m_db_blocks_index.set_cache_size(ceched_elements);
|
||||
m_db_blocks.set_cache_size(ceched_elements);
|
||||
m_db_blocks_index.set_cache_size(ceched_elements);
|
||||
m_db_transactions.set_cache_size(ceched_elements);
|
||||
m_db_spent_keys.set_cache_size(ceched_elements);
|
||||
//m_db_outputs.set_cache_size(ceched_elements);
|
||||
m_db_multisig_outs.set_cache_size(ceched_elements);
|
||||
m_db_solo_options.set_cache_size(ceched_elements);
|
||||
m_db_aliases.set_cache_size(ceched_elements);
|
||||
m_db_assets.set_cache_size(ceched_elements);
|
||||
m_db_addr_to_alias.set_cache_size(ceched_elements);
|
||||
}
|
||||
bool blockchain_storage::init(const std::string& config_folder, const boost::program_options::variables_map& vm)
|
||||
{
|
||||
// CRITICAL_REGION_LOCAL(m_read_lock);
|
||||
|
|
@ -285,18 +297,7 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro
|
|||
if (command_line::has_arg(vm, arg_db_cache_l2))
|
||||
{
|
||||
uint64_t cache_size = command_line::get_arg(vm, arg_db_cache_l2);
|
||||
LOG_PRINT_GREEN("Using db items cache size(L2): " << cache_size, LOG_LEVEL_0);
|
||||
m_db_blocks_index.set_cache_size(cache_size);
|
||||
m_db_blocks.set_cache_size(cache_size);
|
||||
m_db_blocks_index.set_cache_size(cache_size);
|
||||
m_db_transactions.set_cache_size(cache_size);
|
||||
m_db_spent_keys.set_cache_size(cache_size);
|
||||
//m_db_outputs.set_cache_size(cache_size);
|
||||
m_db_multisig_outs.set_cache_size(cache_size);
|
||||
m_db_solo_options.set_cache_size(cache_size);
|
||||
m_db_aliases.set_cache_size(cache_size);
|
||||
m_db_assets.set_cache_size(cache_size);
|
||||
m_db_addr_to_alias.set_cache_size(cache_size);
|
||||
set_db_l2_cache_size(cache_size);
|
||||
}
|
||||
|
||||
LOG_PRINT_L0("Opened DB ver " << m_db_storage_major_compatibility_version << "." << m_db_storage_minor_compatibility_version);
|
||||
|
|
@ -2423,7 +2424,7 @@ bool blockchain_storage::get_main_block_rpc_details(uint64_t i, block_rpc_extend
|
|||
bei.total_txs_size = 0;
|
||||
if (true/*!ignore_transactions*/)
|
||||
{
|
||||
crypto::hash coinbase_id = get_transaction_hash(core_bei_ptr->bl.miner_tx);
|
||||
crypto::hash coinbase_id = get_coinbase_hash_cached(*core_bei_ptr);
|
||||
//load transactions details
|
||||
bei.transactions_details.push_back(tx_rpc_extended_info());
|
||||
get_tx_rpc_details(coinbase_id, bei.transactions_details.back(), get_block_datetime(core_bei_ptr->bl), true);
|
||||
|
|
@ -2508,7 +2509,7 @@ bool blockchain_storage::get_alt_block_rpc_details(const block_extended_info& be
|
|||
|
||||
bei.is_orphan = true;
|
||||
|
||||
crypto::hash coinbase_id = get_transaction_hash(bei_core.bl.miner_tx);
|
||||
crypto::hash coinbase_id = get_coinbase_hash_cached(bei_core);
|
||||
//load transactions details
|
||||
bei.transactions_details.push_back(tx_rpc_extended_info());
|
||||
fill_tx_rpc_details(bei.transactions_details.back(), bei_core.bl.miner_tx, nullptr, coinbase_id, get_block_datetime(bei_core.bl));
|
||||
|
|
@ -3300,7 +3301,7 @@ void blockchain_storage::print_blockchain_with_tx(uint64_t start_index, uint64_t
|
|||
<< ", id: " << get_block_hash(m_db_blocks[i]->bl)
|
||||
<< ", difficulty: " << block_difficulty(i) << ", nonce " << m_db_blocks[i]->bl.nonce << ", tx_count " << m_db_blocks[i]->bl.tx_hashes.size() << ENDL;
|
||||
|
||||
ss << "[miner id]: " << get_transaction_hash(m_db_blocks[i]->bl.miner_tx) << ENDL << currency::obj_to_json_str(m_db_blocks[i]->bl.miner_tx) << ENDL;
|
||||
ss << "[miner id]: " << get_coinbase_hash_cached(*m_db_blocks[i]) << ENDL << currency::obj_to_json_str(m_db_blocks[i]->bl.miner_tx) << ENDL;
|
||||
|
||||
for (size_t j = 0; j != m_db_blocks[i]->bl.tx_hashes.size(); j++)
|
||||
{
|
||||
|
|
@ -3696,7 +3697,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash
|
|||
std::list<crypto::hash> mis;
|
||||
get_transactions_direct(m_db_blocks[i]->bl.tx_hashes, blocks.back().second, mis);
|
||||
CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, block " << get_block_hash(m_db_blocks[i]->bl) << " [" << i << "] contains missing transactions: " << mis);
|
||||
blocks.back().third = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx));
|
||||
blocks.back().third = m_db_transactions.find(get_coinbase_hash_cached(*m_db_blocks[i]));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -5350,7 +5351,7 @@ void blockchain_storage::do_full_db_warm_up() const
|
|||
LOG_ERROR("some tx's not found");
|
||||
return;
|
||||
}
|
||||
auto coinbase_tx_ptr = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx));
|
||||
auto coinbase_tx_ptr = m_db_transactions.find(get_coinbase_hash_cached(*m_db_blocks[i]));
|
||||
if (!coinbase_tx_ptr)
|
||||
{
|
||||
LOG_ERROR("Coinbase not found");
|
||||
|
|
@ -5374,7 +5375,7 @@ void blockchain_storage::do_full_db_warm_up() const
|
|||
PRINT_CONTAINER(m_db_aliases);
|
||||
PRINT_CONTAINER(m_db_assets);
|
||||
PRINT_CONTAINER(m_db_addr_to_alias);
|
||||
LOG_PRINT_CYAN("CACHE STATE: " << ENDL << strm.str(), LOG_LEVEL_0);
|
||||
LOG_PRINT_CYAN("CACHE STATE: " << ENDL << strm.str(), LOG_LEVEL_2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5910,7 +5911,7 @@ std::shared_ptr<const transaction_chain_entry> blockchain_storage::find_key_imag
|
|||
{
|
||||
if (boost::get<txin_to_key>(in).k_image == ki)
|
||||
{
|
||||
id_result = get_transaction_hash(block_entry->bl.miner_tx);
|
||||
id_result = get_coinbase_hash_cached(*block_entry);
|
||||
return get_tx_chain_entry(id_result);
|
||||
}
|
||||
}
|
||||
|
|
@ -8400,7 +8401,7 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha
|
|||
alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(on_board_tx.first, txs_by_id_and_height_altchain::value_type::second_type(on_board_tx.second, ch->second.height)));
|
||||
}
|
||||
//TODO: consider performance optimization (get_transaction_hash might slow down deep reorganizations )
|
||||
alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(get_transaction_hash(ch->second.bl.miner_tx), txs_by_id_and_height_altchain::value_type::second_type(ch->second.bl.miner_tx, ch->second.height)));
|
||||
alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(get_coinbase_hash_cached(ch->second), txs_by_id_and_height_altchain::value_type::second_type(ch->second.bl.miner_tx, ch->second.height)));
|
||||
}
|
||||
}
|
||||
else
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@
|
|||
#pragma once
|
||||
#include <boost/program_options/options_description.hpp>
|
||||
#include <boost/program_options/variables_map.hpp>
|
||||
#include <boost/program_options.hpp>
|
||||
|
||||
#include <boost/interprocess/sync/named_mutex.hpp>
|
||||
|
||||
#include <boost/serialization/serialization.hpp>
|
||||
#include <boost/serialization/version.hpp>
|
||||
#include <boost/serialization/list.hpp>
|
||||
|
||||
#include <boost/foreach.hpp>
|
||||
#include <atomic>
|
||||
|
||||
|
|
@ -23,6 +23,7 @@
|
|||
#include "serialization/string.h"
|
||||
#include "serialization/multiprecision.h"
|
||||
|
||||
#include "common/command_line.h"
|
||||
#include "tx_pool.h"
|
||||
#include "blockchain_storage_basic.h"
|
||||
#include "common/util.h"
|
||||
|
|
@ -49,6 +50,8 @@ MARK_AS_POD_C11(macro_alias_1);
|
|||
#undef LOG_DEFAULT_CHANNEL
|
||||
#define LOG_DEFAULT_CHANNEL "core"
|
||||
|
||||
extern const command_line::arg_descriptor<uint32_t> arg_db_cache_l2;
|
||||
|
||||
namespace currency
|
||||
{
|
||||
|
||||
|
|
@ -507,7 +510,7 @@ namespace currency
|
|||
bool print_tx_outputs_lookup(const crypto::hash& tx_id) const;
|
||||
uint64_t get_last_x_block_height(bool pos)const;
|
||||
bool is_tx_spendtime_unlocked(uint64_t unlock_time)const;
|
||||
|
||||
void set_db_l2_cache_size(uint64_t ceched_elements) const;
|
||||
//experimental
|
||||
void do_full_db_warm_up() const;
|
||||
|
||||
|
|
|
|||
|
|
@ -73,6 +73,10 @@ namespace currency
|
|||
FIELD(this_block_tx_fee_median)
|
||||
FIELD(effective_tx_fee_median)
|
||||
END_SERIALIZE()
|
||||
|
||||
// This is an optional data fields, It is not included in serialization and therefore is never stored in the database.
|
||||
// It might be calculated “on the fly” to speed up access operations.
|
||||
mutable std::shared_ptr<crypto::hash> m_cache_coinbase_id;
|
||||
};
|
||||
|
||||
struct gindex_increment
|
||||
|
|
@ -185,4 +189,20 @@ namespace currency
|
|||
END_KV_SERIALIZE_MAP()
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
inline crypto::hash get_coinbase_hash_cached(const block_extended_info& bei)
|
||||
{
|
||||
if (bei.m_cache_coinbase_id)
|
||||
{
|
||||
return *bei.m_cache_coinbase_id;
|
||||
}
|
||||
else
|
||||
{
|
||||
bei.m_cache_coinbase_id = std::make_shared<crypto::hash>(get_transaction_hash(bei.bl.miner_tx));
|
||||
return *bei.m_cache_coinbase_id;
|
||||
}
|
||||
}
|
||||
} // namespace currency
|
||||
|
|
|
|||
|
|
@ -65,4 +65,5 @@ namespace currency
|
|||
txs_ids.push_back(th);
|
||||
return get_tx_tree_hash(txs_ids);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,5 +21,4 @@ namespace currency
|
|||
void get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes, crypto::hash& h);
|
||||
crypto::hash get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes);
|
||||
crypto::hash get_tx_tree_hash(const block& b);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,10 +37,14 @@ using namespace epee;
|
|||
#endif
|
||||
|
||||
|
||||
const uint64_t min_ram_for_full_warp_mode = 32ULL * 1024ULL * 1024ULL * 1024ULL;
|
||||
const uint64_t recommended_ram_for_full_warp_mode = 64ULL * 1024ULL * 1024ULL * 1024ULL;
|
||||
|
||||
//TODO: need refactoring here. (template classes can't be used in BOOST_CLASS_VERSION)
|
||||
BOOST_CLASS_VERSION(nodetool::node_server<currency::t_currency_protocol_handler<currency::core> >, CURRENT_P2P_STORAGE_ARCHIVE_VER);
|
||||
|
||||
const command_line::arg_descriptor<uint32_t> arg_rpc_server_threads("rpc-server-threads", "Specify number of RPC server threads. Default: 10", RPC_SERVER_DEFAULT_THREADS_NUM);
|
||||
const command_line::arg_descriptor<bool> arg_do_warp_mode("do-warp-mode", "This option pre-loads and unserialize all data into RAM and provide significant speed increase in RPC-handling, requires 32GB psychical RAM at least(64GB recommended). Might be helpful for production servers(like remote nodes or public nodes for mobile apps).");
|
||||
|
||||
namespace po = boost::program_options;
|
||||
|
||||
|
|
@ -166,8 +170,9 @@ int main(int argc, char* argv[])
|
|||
command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload);
|
||||
command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link);
|
||||
command_line::add_arg(desc_cmd_sett, command_line::arg_disable_ntp);
|
||||
|
||||
command_line::add_arg(desc_cmd_sett, arg_rpc_server_threads);
|
||||
|
||||
command_line::add_arg(desc_cmd_sett, arg_do_warp_mode);
|
||||
|
||||
arg_market_disable.default_value = true;
|
||||
arg_market_disable.use_default = true;
|
||||
|
|
@ -372,6 +377,43 @@ int main(int argc, char* argv[])
|
|||
res = ccore.set_checkpoints(std::move(checkpoints));
|
||||
CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core");
|
||||
|
||||
|
||||
//do full warp mode if needed
|
||||
if (command_line::has_arg(vm, arg_do_warp_mode))
|
||||
{
|
||||
LOG_PRINT_MAGENTA("Initializing full warp-mode", LOG_LEVEL_0);
|
||||
//let's check if cache size were specifically set
|
||||
if (!command_line::has_arg(vm, arg_db_cache_l2))
|
||||
{
|
||||
//overriding caching settings
|
||||
uint64_t cache_size = ccore.get_blockchain_storage().get_total_transactions() * 10;
|
||||
ccore.get_blockchain_storage().set_db_l2_cache_size(cache_size);
|
||||
|
||||
LOG_PRINT_MAGENTA("[Warp]: Setting up db cache to " << tools::pretty_print_big_nums(cache_size) << " items.....", LOG_LEVEL_0);
|
||||
}
|
||||
uint64_t phisical_ram_detected = tools::get_total_system_memory();
|
||||
if (phisical_ram_detected < min_ram_for_full_warp_mode)
|
||||
{
|
||||
LOG_PRINT_RED_L0("[Warp]: Detected only " << tools::pretty_print_big_nums(phisical_ram_detected) << "B of RAM, it's not recommended to run daemon in full warm up mode under " << tools::pretty_print_big_nums(min_ram_for_full_warp_mode) << "B, stopping...");
|
||||
return 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
if(phisical_ram_detected < recommended_ram_for_full_warp_mode)
|
||||
{
|
||||
LOG_PRINT_MAGENTA("[Warp]: Detected only " << tools::pretty_print_big_nums(phisical_ram_detected) << "B RAM, might be not optimal, recommended above " << tools::pretty_print_big_nums(recommended_ram_for_full_warp_mode) << "B", LOG_LEVEL_0);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_PRINT_GREEN("[Warp]: Detected " << tools::pretty_print_big_nums(phisical_ram_detected) << "B RAM", LOG_LEVEL_0);
|
||||
}
|
||||
}
|
||||
|
||||
LOG_PRINT_MAGENTA("[Warp]: Launching warm up....", LOG_LEVEL_0);
|
||||
ccore.get_blockchain_storage().do_full_db_warm_up();
|
||||
LOG_PRINT_MAGENTA("[Warp]: Warm up finished!", LOG_LEVEL_0);
|
||||
}
|
||||
|
||||
// start components
|
||||
if (!command_line::has_arg(vm, command_line::arg_console))
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue