diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index ed435352..ff83104d 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -40,7 +40,7 @@ namespace command_line const arg_descriptor arg_process_predownload_from_path("predownload-from-local-path", "Instead of downloading file use downloaded local file"); const arg_descriptor arg_validate_predownload ( "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database"); const arg_descriptor arg_predownload_link ( "predownload-link", "Override url for blockchain database pre-downloading"); - + const arg_descriptor arg_non_pruning_mode ( "non-pruning-mode", "Enables a special operational mode with full retention of all tx signatures. Will terminate if the DB was previously in (normal) pruning mode. Use only if you know what you do."); const arg_descriptor arg_deeplink ( "deeplink-params", "Deeplink parameter, in that case app just forward params to running app"); diff --git a/src/common/command_line.h b/src/common/command_line.h index 0b326917..6797cc18 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -232,6 +232,7 @@ namespace command_line extern const arg_descriptor arg_process_predownload_from_path; extern const arg_descriptor arg_validate_predownload; extern const arg_descriptor arg_predownload_link; + extern const arg_descriptor arg_non_pruning_mode; extern const arg_descriptor arg_deeplink; extern const arg_descriptor arg_generate_rpc_autodoc; diff --git a/src/common/pre_download.h b/src/common/pre_download.h index 102c65bd..f2408ed9 100644 --- a/src/common/pre_download.h +++ b/src/common/pre_download.h @@ -23,9 +23,13 @@ namespace tools #ifndef TESTNET static constexpr pre_download_entry c_pre_download_mdbx = { "https://f005.backblazeb2.com/file/zano-predownload/zano_mdbx_95_3083770.pak", "e7cb7b5e1560c3a7615604880feda9df37636be83264a5afff01f44b5f824cc8", 8357805798, 12884705280 }; static constexpr pre_download_entry c_pre_download_lmdb = { "https://f005.backblazeb2.com/file/zano-predownload/zano_lmdb_95_3083770.pak", "685db01e1a4c827d20e777563009f771be593fe80cc32b8a4dfe2711e6a2b2f8", 10070627937, 12842385408 }; + static constexpr pre_download_entry c_pre_download_mdbx_non_pruned = { "", "", 0, 0 }; + static constexpr pre_download_entry c_pre_download_lmdb_non_pruned = { "", "", 0, 0 }; #else static constexpr pre_download_entry c_pre_download_mdbx = { "", "", 0, 0 }; static constexpr pre_download_entry c_pre_download_lmdb = { "", "", 0, 0 }; + static constexpr pre_download_entry c_pre_download_mdbx_non_pruned = { "", "", 0, 0 }; + static constexpr pre_download_entry c_pre_download_lmdb_non_pruned = { "", "", 0, 0 }; #endif static constexpr uint64_t pre_download_min_size_difference = 512 * 1024 * 1024; // minimum difference in size between local DB and the downloadable one to start downloading @@ -41,7 +45,8 @@ namespace tools std::string working_folder = dbbs.get_db_folder_path(); std::string db_main_file_path = working_folder + "/" + dbbs.get_db_main_file_name(); - pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? c_pre_download_lmdb : c_pre_download_mdbx; + bool non_pruning_mode_enabled = tools::is_non_pruning_mode_enabled(vm); + pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? (non_pruning_mode_enabled ? c_pre_download_lmdb_non_pruned : c_pre_download_lmdb) : (non_pruning_mode_enabled ? c_pre_download_mdbx_non_pruned : c_pre_download_mdbx); // override pre-download link if necessary std::string url = pre_download.url; diff --git a/src/common/util.cpp b/src/common/util.cpp index b2827d59..58bcd243 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -26,6 +26,7 @@ using namespace epee; #include #include "string_coding.h" +#include "command_line.h" namespace tools { @@ -729,6 +730,17 @@ std::string get_nix_version_display_string() return true; } + bool is_non_pruning_mode_enabled(const boost::program_options::variables_map& vm, bool *p_enabled_via_env /* = nullptr */) + { + const char* npm_env = std::getenv("ZANO_NON_PRUNING_MODE"); + std::string npm_env_str = boost::algorithm::to_lower_copy(std::string(npm_env ? npm_env : "")); + bool npm_env_enabled = (npm_env_str == "1" || npm_env_str == "on" || npm_env_str == "true"); + bool result = (command_line::has_arg(vm, command_line::arg_non_pruning_mode) && command_line::get_arg(vm, command_line::arg_non_pruning_mode)) || npm_env_enabled; + if (result && p_enabled_via_env != nullptr) + *p_enabled_via_env = npm_env_enabled; + return result; + } + //this code was taken from https://stackoverflow.com/a/8594696/5566653 //credits goes to @nijansen: https://stackoverflow.com/users/1056003/nijansen bool copy_dir( boost::filesystem::path const & source, boost::filesystem::path const & destination) diff --git a/src/common/util.h b/src/common/util.h index ebf261aa..7ffbbf56 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -41,6 +41,7 @@ namespace tools bool parse_client_version(const std::string& str, int& major, int& minor, int& revision, int& build_number, std::string& commit_id, bool& dirty); bool parse_client_version_build_number(const std::string& str, int& build_number); bool check_remote_client_version(const std::string& client_ver); + bool is_non_pruning_mode_enabled(const boost::program_options::variables_map& vm, bool *p_enabled_via_env = nullptr); bool create_directories_if_necessary(const std::string& path); std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name); @@ -57,6 +58,7 @@ namespace tools return crypto::cn_fast_hash(s.data(), s.size()); } + // the following is unsafe, consider removing -- sowle inline crypto::public_key get_public_key_from_string(const std::string& str_key) { diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index c9f3ec0b..2e867f1f 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -74,8 +74,8 @@ using namespace currency; DISABLE_VS_WARNINGS(4267) -const command_line::arg_descriptor arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file"); -const command_line::arg_descriptor arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers"); +const command_line::arg_descriptor arg_db_cache_l1 ( "db-cache-l1", "Specify size of memory mapped db cache file"); +const command_line::arg_descriptor arg_db_cache_l2 ( "db-cache-l2", "Specify cached elements in db helpers"); //------------------------------------------------------------------ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m_rw_lock), @@ -111,7 +111,8 @@ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m m_deinit_is_done(false), m_cached_next_pow_difficulty(0), m_cached_next_pos_difficulty(0), - m_blockchain_launch_timestamp(0) + m_blockchain_launch_timestamp(0), + m_non_pruning_mode_enabled(false) { @@ -155,6 +156,7 @@ void blockchain_storage::init_options(boost::program_options::options_descriptio { command_line::add_arg(desc, arg_db_cache_l1); command_line::add_arg(desc, arg_db_cache_l2); + command_line::add_arg(desc, command_line::arg_non_pruning_mode); } //------------------------------------------------------------------ uint64_t blockchain_storage::get_block_h_older_then(uint64_t timestamp) const @@ -278,6 +280,13 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro return false; } + bool npm_env_enabled = false; + m_non_pruning_mode_enabled = tools::is_non_pruning_mode_enabled(vm, &npm_env_enabled); + if (m_non_pruning_mode_enabled) + { + LOG_PRINT_CYAN("*** Non-pruning mode ENABLED" << (npm_env_enabled ? " (via an environment variable)" : "") << ". This build will allways retain all transaction data regardless of checkpoints. The DB is expected to have all data and will be verified soon.", LOG_LEVEL_0); + } + uint64_t cache_size_l1 = CACHE_SIZE; if (command_line::has_arg(vm, arg_db_cache_l1)) { @@ -594,6 +603,52 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro << " major failure: " << (m_db_major_failure ? "true" : "false"), LOG_LEVEL_0); + + if (m_non_pruning_mode_enabled) + { + size_t txs = 0; + size_t pruned_txs = 0; + size_t signatures = 0; + size_t attachments = 0; + + uint64_t last_block_height = m_db_blocks.size() > 0 ? m_db_blocks.size() - 1 : 0; + + LOG_PRINT_CYAN("The blockchain will be scanned now; it takes a while, please wait...", LOG_LEVEL_0); + + for (uint64_t height = 0; height <= last_block_height; height++) + { + auto vptr = m_db_blocks[height]; + CHECK_AND_ASSERT_MES(vptr.get(), false, "Failed to get block on height"); + + for (const auto& h : vptr->bl.tx_hashes) + { + auto it = m_db_transactions.find(h); + CHECK_AND_ASSERT_MES(it != m_db_transactions.end(), false, "failed to find transaction " << h << " in blockchain index, in block on height = " << height); + CHECK_AND_ASSERT_MES(it->m_keeper_block_height == height, false, + "failed to validate extra check, it->second.m_keeper_block_height = " << it->m_keeper_block_height << + "is mot equal to height = " << height << " in blockchain index, for block on height = " << height); + + if (it->tx.signatures.size() == 0) + { + pruned_txs += 1; + CHECK_AND_ASSERT_THROW_MES(false, "found pruned tx " << h << ", non pruning mode couldn't be activated on pruned DB, terminating..."); + } + + txs += 1; + signatures += it->tx.signatures.size(); + attachments += it->tx.attachment.size(); + } + } + + LOG_PRINT_CYAN(ENDL << "blockchain (non)pruning status:" << ENDL << + " last block height: " << last_block_height << ENDL << + " total txs: " << txs << ENDL << + " pruned txs: " << pruned_txs << ENDL << + " total signatures: " << signatures << ENDL << + " total attachments: " << attachments << ENDL << + ENDL << "Blockchain DB was successfully scanned for pruned txs.", LOG_LEVEL_0); + } + return true; } @@ -699,6 +754,9 @@ bool blockchain_storage::pop_block_from_blockchain(transactions_map& onboard_tra //------------------------------------------------------------------ bool blockchain_storage::set_checkpoints(checkpoints&& chk_pts) { + if (m_non_pruning_mode_enabled) + return true; + m_checkpoints = chk_pts; try { @@ -760,6 +818,7 @@ bool blockchain_storage::prune_ring_signatures_and_attachments(uint64_t height, bool blockchain_storage::prune_ring_signatures_and_attachments_if_need() { CRITICAL_REGION_LOCAL(m_read_lock); + CHECK_AND_ASSERT_MES(!m_non_pruning_mode_enabled, false, "cannot prune while non-pruning mode is enabled"); uint64_t top_block_height = get_top_block_height(); uint64_t pruning_end_height = m_checkpoints.get_checkpoint_before_height(top_block_height); diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index efa31508..cca1aecd 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -341,6 +341,7 @@ namespace currency bool is_multisig_output_spent(const crypto::hash& multisig_id) const; boost::multiprecision::uint128_t total_coins()const; bool is_pos_allowed()const; + bool is_non_pruning_mode_enabled() const { return m_non_pruning_mode_enabled; } uint64_t get_tx_fee_median()const; uint64_t get_tx_fee_window_value_median() const; uint64_t get_tx_expiration_median() const; @@ -593,6 +594,7 @@ namespace currency std::atomic m_is_in_checkpoint_zone; std::atomic m_is_blockchain_storing; + bool m_non_pruning_mode_enabled; std::string m_config_folder; //events diff --git a/src/currency_protocol/currency_protocol_handler.inl b/src/currency_protocol/currency_protocol_handler.inl index 8f5bbae1..959e963c 100644 --- a/src/currency_protocol/currency_protocol_handler.inl +++ b/src/currency_protocol/currency_protocol_handler.inl @@ -194,11 +194,12 @@ namespace currency { LOG_PRINT_RED("Remote node has longer checkpoints zone (" << hshd.last_checkpoint_height << ") " << "than local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << "). " << - "It means that current software is outdated, please updated it! " << + (m_core.get_blockchain_storage().is_non_pruning_mode_enabled() ? "It is expected since this node is in non-pruning mode. " : "It means that current software is outdated, please updated it! ") << "Current height lays under checkpoints zone on remote host, so it's impossible to validate remote transactions locally, disconnecting.", LOG_LEVEL_0); return false; } - else if (m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() < hshd.last_checkpoint_height) + + if (!m_core.get_blockchain_storage().is_non_pruning_mode_enabled() && m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() < hshd.last_checkpoint_height) { LOG_PRINT_MAGENTA("Remote node has longer checkpoints zone (" << hshd.last_checkpoint_height << ") " << "than local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << "). " << diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 176b7eb1..6150edc8 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -193,6 +193,7 @@ int main(int argc, char* argv[]) + std::string data_dir; po::variables_map vm; bool exit_requested = false; bool r = command_line::handle_error_helper(desc_options, [&]() @@ -213,7 +214,7 @@ int main(int argc, char* argv[]) return true; } - std::string data_dir = command_line::get_arg(vm, command_line::arg_data_dir); + data_dir = command_line::get_arg(vm, command_line::arg_data_dir); std::string config = command_line::get_arg(vm, command_line::arg_config_file); boost::filesystem::path data_dir_path(epee::string_encoding::utf8_to_wstring(data_dir)); @@ -240,17 +241,11 @@ int main(int argc, char* argv[]) return EXIT_SUCCESS; //set up logging options - std::string log_dir; + std::string log_dir = data_dir; std::string log_file_name = log_space::log_singletone::get_default_log_file(); //check if there was specific option if (command_line::has_arg(vm, command_line::arg_log_dir)) - { log_dir = command_line::get_arg(vm, command_line::arg_log_dir); - } - else - { - log_dir = command_line::get_arg(vm, command_line::arg_data_dir); - } log_space::log_singletone::add_logger(LOGGER_FILE, log_file_name.c_str(), log_dir.c_str()); LOG_PRINT_L0(CURRENCY_NAME << " v" << PROJECT_VERSION_LONG); @@ -264,7 +259,7 @@ int main(int argc, char* argv[]) // stratum server is enabled if any of its options present bool stratum_enabled = currency::stratum_server::should_start(vm); - LOG_PRINT("Module folder: " << argv[0], LOG_LEVEL_0); + LOG_PRINT("Module folder: " << argv[0] << ", data folder: " << data_dir, LOG_LEVEL_0); //create objects and link them bc_services::bc_offers_service offers_service(nullptr); diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 1bb27fbb..43024815 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -570,14 +570,14 @@ namespace nodetool { if(!m_payload_handler.process_payload_sync_data(rsp.payload_data, context, true)) { - LOG_ERROR_CCONTEXT("COMMAND_HANDSHAKE invoked, but process_payload_sync_data returned false, dropping connection."); + LOG_PRINT_L1("COMMAND_HANDSHAKE invoked, but process_payload_sync_data returned false, dropping connection."); hsh_result = false; return; } if (is_peer_id_used(rsp.node_data.peer_id)) { - LOG_PRINT_L0("It seems that peer " << std::hex << rsp.node_data.peer_id << " has already been connected, dropping connection"); + LOG_PRINT_L1("It seems that peer " << std::hex << rsp.node_data.peer_id << " has already been connected, dropping connection"); hsh_result = false; return; } @@ -587,7 +587,7 @@ namespace nodetool if(rsp.node_data.peer_id == m_config.m_peer_id) { - LOG_PRINT_L0("Connection to self detected, dropping connection"); + LOG_PRINT_L1("Connection to self detected, dropping connection"); hsh_result = false; return; } @@ -606,7 +606,7 @@ namespace nodetool if(!hsh_result) { - LOG_PRINT_CC_L0(context_, "COMMAND_HANDSHAKE Failed, closing connection"); + LOG_PRINT_CC_L1(context_, "COMMAND_HANDSHAKE Failed, closing connection"); m_net_server.get_config_object().close(context_.m_connection_id); }