Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity

non-pruning mode implemented using checkpoints

Browse source
This commit is contained in:
sowle 2025-04-30 03:55:13 +03:00
parent cc7188f450
commit e5323d0c5a
No known key found for this signature in database
GPG key ID: C07A24B2D89D49FC
10 changed files with 97 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/common/command_line.cpp
View file

@ -40,7 +40,7 @@ namespace command_line
const arg_descriptor<std::string> arg_process_predownload_from_path("predownload-from-local-path", "Instead of downloading file use downloaded local file");
const arg_descriptor<bool> arg_validate_predownload ( "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database");
const arg_descriptor<std::string> arg_predownload_link ( "predownload-link", "Override url for blockchain database pre-downloading");
const arg_descriptor<bool> arg_non_pruning_mode ( "non-pruning-mode", "Enables a special operational mode with full retention of all tx signatures. Will terminate if the DB was previously in (normal) pruning mode. Use only if you know what you do.");
const arg_descriptor<std::string> arg_deeplink ( "deeplink-params", "Deeplink parameter, in that case app just forward params to running app");

1
src/common/command_line.h
View file

@ -232,6 +232,7 @@ namespace command_line
extern const arg_descriptor<std::string> arg_process_predownload_from_path;
extern const arg_descriptor<bool> arg_validate_predownload;
extern const arg_descriptor<std::string> arg_predownload_link;
extern const arg_descriptor<bool> arg_non_pruning_mode;
extern const arg_descriptor<std::string> arg_deeplink;
extern const arg_descriptor<std::string> arg_generate_rpc_autodoc;

7
src/common/pre_download.h
View file

@ -23,9 +23,13 @@ namespace tools
#ifndef TESTNET
static constexpr pre_download_entry c_pre_download_mdbx = { "https://f005.backblazeb2.com/file/zano-predownload/zano_mdbx_95_3083770.pak", "e7cb7b5e1560c3a7615604880feda9df37636be83264a5afff01f44b5f824cc8", 8357805798, 12884705280 };
static constexpr pre_download_entry c_pre_download_lmdb = { "https://f005.backblazeb2.com/file/zano-predownload/zano_lmdb_95_3083770.pak", "685db01e1a4c827d20e777563009f771be593fe80cc32b8a4dfe2711e6a2b2f8", 10070627937, 12842385408 };
static constexpr pre_download_entry c_pre_download_mdbx_non_pruned = { "", "", 0, 0 };
static constexpr pre_download_entry c_pre_download_lmdb_non_pruned = { "", "", 0, 0 };
#else
static constexpr pre_download_entry c_pre_download_mdbx = { "", "", 0, 0 };
static constexpr pre_download_entry c_pre_download_lmdb = { "", "", 0, 0 };
static constexpr pre_download_entry c_pre_download_mdbx_non_pruned = { "", "", 0, 0 };
static constexpr pre_download_entry c_pre_download_lmdb_non_pruned = { "", "", 0, 0 };
#endif
static constexpr uint64_t pre_download_min_size_difference = 512 * 1024 * 1024; // minimum difference in size between local DB and the downloadable one to start downloading
@ -41,7 +45,8 @@ namespace tools
std::string working_folder = dbbs.get_db_folder_path();
std::string db_main_file_path = working_folder + "/" + dbbs.get_db_main_file_name();
pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? c_pre_download_lmdb : c_pre_download_mdbx;
bool non_pruning_mode_enabled = tools::is_non_pruning_mode_enabled(vm);
pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? (non_pruning_mode_enabled ? c_pre_download_lmdb_non_pruned : c_pre_download_lmdb) : (non_pruning_mode_enabled ? c_pre_download_mdbx_non_pruned : c_pre_download_mdbx);
// override pre-download link if necessary
std::string url = pre_download.url;

12
src/common/util.cpp
View file

@ -26,6 +26,7 @@ using namespace epee;
#include <boost/asio.hpp>
#include "string_coding.h"
#include "command_line.h"
namespace tools
{
@ -729,6 +730,17 @@ std::string get_nix_version_display_string()
return true;
}
bool is_non_pruning_mode_enabled(const boost::program_options::variables_map& vm, bool *p_enabled_via_env /* = nullptr */)
{
const char* npm_env = std::getenv("ZANO_NON_PRUNING_MODE");
std::string npm_env_str = boost::algorithm::to_lower_copy(std::string(npm_env ? npm_env : ""));
bool npm_env_enabled = (npm_env_str == "1" || npm_env_str == "on" || npm_env_str == "true");
bool result = (command_line::has_arg(vm, command_line::arg_non_pruning_mode) && command_line::get_arg(vm, command_line::arg_non_pruning_mode)) || npm_env_enabled;
if (result && p_enabled_via_env != nullptr)
*p_enabled_via_env = npm_env_enabled;
return result;
}
//this code was taken from https://stackoverflow.com/a/8594696/5566653
//credits goes to @nijansen: https://stackoverflow.com/users/1056003/nijansen
bool copy_dir( boost::filesystem::path const & source, boost::filesystem::path const & destination)

2
src/common/util.h
View file

@ -41,6 +41,7 @@ namespace tools
bool parse_client_version(const std::string& str, int& major, int& minor, int& revision, int& build_number, std::string& commit_id, bool& dirty);
bool parse_client_version_build_number(const std::string& str, int& build_number);
bool check_remote_client_version(const std::string& client_ver);
bool is_non_pruning_mode_enabled(const boost::program_options::variables_map& vm, bool *p_enabled_via_env = nullptr);
bool create_directories_if_necessary(const std::string& path);
std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name);
@ -57,6 +58,7 @@ namespace tools
return crypto::cn_fast_hash(s.data(), s.size());
}
// the following is unsafe, consider removing -- sowle
inline
crypto::public_key get_public_key_from_string(const std::string& str_key)
{

65
src/currency_core/blockchain_storage.cpp
View file

@ -74,8 +74,8 @@ using namespace currency;
DISABLE_VS_WARNINGS(4267)
const command_line::arg_descriptor<uint32_t> arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file");
const command_line::arg_descriptor<uint32_t> arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers");
const command_line::arg_descriptor<uint32_t> arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file");
const command_line::arg_descriptor<uint32_t> arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers");
//------------------------------------------------------------------
blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m_rw_lock),
@ -111,7 +111,8 @@ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m
m_deinit_is_done(false),
m_cached_next_pow_difficulty(0),
m_cached_next_pos_difficulty(0),
m_blockchain_launch_timestamp(0)
m_blockchain_launch_timestamp(0),
m_non_pruning_mode_enabled(false)
{
@ -155,6 +156,7 @@ void blockchain_storage::init_options(boost::program_options::options_descriptio
{
command_line::add_arg(desc, arg_db_cache_l1);
command_line::add_arg(desc, arg_db_cache_l2);
command_line::add_arg(desc, command_line::arg_non_pruning_mode);
}
//------------------------------------------------------------------
uint64_t blockchain_storage::get_block_h_older_then(uint64_t timestamp) const
@ -278,6 +280,13 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro
return false;
}
bool npm_env_enabled = false;
m_non_pruning_mode_enabled = tools::is_non_pruning_mode_enabled(vm, &npm_env_enabled);
if (m_non_pruning_mode_enabled)
{
LOG_PRINT_CYAN("*** Non-pruning mode ENABLED" << (npm_env_enabled ? " (via an environment variable)" : "") << ". This build will allways retain all transaction data regardless of checkpoints. The DB is expected to have all data and will be verified soon.", LOG_LEVEL_0);
}
uint64_t cache_size_l1 = CACHE_SIZE;
if (command_line::has_arg(vm, arg_db_cache_l1))
{
@ -594,6 +603,52 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro
<< " major failure: " << (m_db_major_failure ? "true" : "false"),
LOG_LEVEL_0);
if (m_non_pruning_mode_enabled)
{
size_t txs = 0;
size_t pruned_txs = 0;
size_t signatures = 0;
size_t attachments = 0;
uint64_t last_block_height = m_db_blocks.size() > 0 ? m_db_blocks.size() - 1 : 0;
LOG_PRINT_CYAN("The blockchain will be scanned now; it takes a while, please wait...", LOG_LEVEL_0);
for (uint64_t height = 0; height <= last_block_height; height++)
{
auto vptr = m_db_blocks[height];
CHECK_AND_ASSERT_MES(vptr.get(), false, "Failed to get block on height");
for (const auto& h : vptr->bl.tx_hashes)
{
auto it = m_db_transactions.find(h);
CHECK_AND_ASSERT_MES(it != m_db_transactions.end(), false, "failed to find transaction " << h << " in blockchain index, in block on height = " << height);
CHECK_AND_ASSERT_MES(it->m_keeper_block_height == height, false,
"failed to validate extra check, it->second.m_keeper_block_height = " << it->m_keeper_block_height <<
"is mot equal to height = " << height << " in blockchain index, for block on height = " << height);
if (it->tx.signatures.size() == 0)
{
pruned_txs += 1;
CHECK_AND_ASSERT_THROW_MES(false, "found pruned tx " << h << ", non pruning mode couldn't be activated on pruned DB, terminating...");
}
txs += 1;
signatures += it->tx.signatures.size();
attachments += it->tx.attachment.size();
}
}
LOG_PRINT_CYAN(ENDL << "blockchain (non)pruning status:" << ENDL <<
" last block height: " << last_block_height << ENDL <<
" total txs: " << txs << ENDL <<
" pruned txs: " << pruned_txs << ENDL <<
" total signatures: " << signatures << ENDL <<
" total attachments: " << attachments << ENDL <<
ENDL << "Blockchain DB was successfully scanned for pruned txs.", LOG_LEVEL_0);
}
return true;
}
@ -699,6 +754,9 @@ bool blockchain_storage::pop_block_from_blockchain(transactions_map& onboard_tra
//------------------------------------------------------------------
bool blockchain_storage::set_checkpoints(checkpoints&& chk_pts)
{
if (m_non_pruning_mode_enabled)
return true;
m_checkpoints = chk_pts;
try
{
@ -760,6 +818,7 @@ bool blockchain_storage::prune_ring_signatures_and_attachments(uint64_t height,
bool blockchain_storage::prune_ring_signatures_and_attachments_if_need()
{
CRITICAL_REGION_LOCAL(m_read_lock);
CHECK_AND_ASSERT_MES(!m_non_pruning_mode_enabled, false, "cannot prune while non-pruning mode is enabled");
uint64_t top_block_height = get_top_block_height();
uint64_t pruning_end_height = m_checkpoints.get_checkpoint_before_height(top_block_height);

2
src/currency_core/blockchain_storage.h
View file

@ -341,6 +341,7 @@ namespace currency
bool is_multisig_output_spent(const crypto::hash& multisig_id) const;
boost::multiprecision::uint128_t total_coins()const;
bool is_pos_allowed()const;
bool is_non_pruning_mode_enabled() const { return m_non_pruning_mode_enabled; }
uint64_t get_tx_fee_median()const;
uint64_t get_tx_fee_window_value_median() const;
uint64_t get_tx_expiration_median() const;
@ -593,6 +594,7 @@ namespace currency
std::atomic<bool> m_is_in_checkpoint_zone;
std::atomic<bool> m_is_blockchain_storing;
bool m_non_pruning_mode_enabled;
std::string m_config_folder;
//events

5
src/currency_protocol/currency_protocol_handler.inl
View file

@ -194,11 +194,12 @@ namespace currency
{
LOG_PRINT_RED("Remote node has longer checkpoints zone (" << hshd.last_checkpoint_height << ") " <<
"than local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << "). " <<
"It means that current software is outdated, please updated it! " <<
(m_core.get_blockchain_storage().is_non_pruning_mode_enabled() ? "It is expected since this node is in non-pruning mode. " : "It means that current software is outdated, please updated it! ") <<
"Current height lays under checkpoints zone on remote host, so it's impossible to validate remote transactions locally, disconnecting.", LOG_LEVEL_0);
return false;
}
else if (m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() < hshd.last_checkpoint_height)
if (!m_core.get_blockchain_storage().is_non_pruning_mode_enabled() && m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() < hshd.last_checkpoint_height)
{
LOG_PRINT_MAGENTA("Remote node has longer checkpoints zone (" << hshd.last_checkpoint_height << ") " <<
"than local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << "). " <<

13
src/daemon/daemon.cpp
View file

@ -193,6 +193,7 @@ int main(int argc, char* argv[])
std::string data_dir;
po::variables_map vm;
bool exit_requested = false;
bool r = command_line::handle_error_helper(desc_options, [&]()
@ -213,7 +214,7 @@ int main(int argc, char* argv[])
return true;
}
std::string data_dir = command_line::get_arg(vm, command_line::arg_data_dir);
data_dir = command_line::get_arg(vm, command_line::arg_data_dir);
std::string config = command_line::get_arg(vm, command_line::arg_config_file);
boost::filesystem::path data_dir_path(epee::string_encoding::utf8_to_wstring(data_dir));
@ -240,17 +241,11 @@ int main(int argc, char* argv[])
return EXIT_SUCCESS;
//set up logging options
std::string log_dir;
std::string log_dir = data_dir;
std::string log_file_name = log_space::log_singletone::get_default_log_file();
//check if there was specific option
if (command_line::has_arg(vm, command_line::arg_log_dir))
{
log_dir = command_line::get_arg(vm, command_line::arg_log_dir);
}
else
{
log_dir = command_line::get_arg(vm, command_line::arg_data_dir);
}
log_space::log_singletone::add_logger(LOGGER_FILE, log_file_name.c_str(), log_dir.c_str());
LOG_PRINT_L0(CURRENCY_NAME << " v" << PROJECT_VERSION_LONG);
@ -264,7 +259,7 @@ int main(int argc, char* argv[])
// stratum server is enabled if any of its options present
bool stratum_enabled = currency::stratum_server::should_start(vm);
LOG_PRINT("Module folder: " << argv[0], LOG_LEVEL_0);
LOG_PRINT("Module folder: " << argv[0] << ", data folder: " << data_dir, LOG_LEVEL_0);
//create objects and link them
bc_services::bc_offers_service offers_service(nullptr);

8
src/p2p/net_node.inl
View file

@ -570,14 +570,14 @@ namespace nodetool
{
if(!m_payload_handler.process_payload_sync_data(rsp.payload_data, context, true))
{
LOG_ERROR_CCONTEXT("COMMAND_HANDSHAKE invoked, but process_payload_sync_data returned false, dropping connection.");
LOG_PRINT_L1("COMMAND_HANDSHAKE invoked, but process_payload_sync_data returned false, dropping connection.");
hsh_result = false;
return;
}
if (is_peer_id_used(rsp.node_data.peer_id))
{
LOG_PRINT_L0("It seems that peer " << std::hex << rsp.node_data.peer_id << " has already been connected, dropping connection");
LOG_PRINT_L1("It seems that peer " << std::hex << rsp.node_data.peer_id << " has already been connected, dropping connection");
hsh_result = false;
return;
}
@ -587,7 +587,7 @@ namespace nodetool
if(rsp.node_data.peer_id == m_config.m_peer_id)
{
LOG_PRINT_L0("Connection to self detected, dropping connection");
LOG_PRINT_L1("Connection to self detected, dropping connection");
hsh_result = false;
return;
}
@ -606,7 +606,7 @@ namespace nodetool
if(!hsh_result)
{
LOG_PRINT_CC_L0(context_, "COMMAND_HANDSHAKE Failed, closing connection");
LOG_PRINT_CC_L1(context_, "COMMAND_HANDSHAKE Failed, closing connection");
m_net_server.get_config_object().close(context_.m_connection_id);
}