diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index e3407412..4926e345 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -31,4 +31,10 @@ namespace command_line const arg_descriptor arg_disable_stop_on_low_free_space = { "disable-stop-on-low-free-space", "Do not stop the daemon if free space at data dir is critically low", false, true }; const arg_descriptor arg_enable_offers_service = { "enable-offers-service", "Enables marketplace feature", false, false}; const arg_descriptor arg_db_engine = { "db-engine", "Specify database engine for storage. May be \"lmdb\"(default) or \"mdbx\"", ARG_DB_ENGINE_LMDB, false }; + + const arg_descriptor arg_no_predownload = { "no-predownload", "Do not pre-download blockchain database", }; + const arg_descriptor arg_explicit_predownload = { "explicit-predownload", "Pre-download blockchain database regardless of it's status", }; + const arg_descriptor arg_validate_predownload = { "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database", }; + const arg_descriptor arg_predownload_link = { "predownload-link", "Override url for blockchain database pre-downloading", "", true }; + } diff --git a/src/common/command_line.h b/src/common/command_line.h index eb849101..bbd871d6 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -192,4 +192,8 @@ namespace command_line extern const arg_descriptor arg_disable_stop_on_low_free_space; extern const arg_descriptor arg_enable_offers_service; extern const arg_descriptor arg_db_engine; + extern const arg_descriptor arg_no_predownload; + extern const arg_descriptor arg_explicit_predownload; + extern const arg_descriptor arg_validate_predownload; + extern const arg_descriptor arg_predownload_link; } diff --git a/src/common/db_backend_selector.cpp b/src/common/db_backend_selector.cpp index c284a32b..9c7917ca 100644 --- a/src/common/db_backend_selector.cpp +++ b/src/common/db_backend_selector.cpp @@ -8,9 +8,8 @@ #include "db_backend_lmdb.h" #include "db_backend_mdbx.h" - -#define LMDB_MAIN_FILE_NAME "data.mdb" -#define MDBX_MAIN_FILE_NAME "mdbx.dat" +#define LMDB_MAIN_FILE_NAME "data.mdb" +#define MDBX_MAIN_FILE_NAME "mdbx.dat" namespace tools { @@ -65,10 +64,20 @@ namespace db std::string db_backend_selector::get_db_folder_path() const { //CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited"); - return m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; } + std::string db_backend_selector::get_temp_db_folder_path() const + { + //CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited"); + return get_temp_config_folder() + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; + } + + std::string db_backend_selector::get_pool_db_folder_path() const + { + return m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX; + } + std::string db_backend_selector::get_db_main_file_name() const { switch (m_engine_type) @@ -111,6 +120,12 @@ namespace db } } + std::string db_backend_selector::get_temp_config_folder() const + { + return m_config_folder + "_TEMP"; + } + + } // namespace db diff --git a/src/common/db_backend_selector.h b/src/common/db_backend_selector.h index a758406e..76668582 100644 --- a/src/common/db_backend_selector.h +++ b/src/common/db_backend_selector.h @@ -26,7 +26,11 @@ namespace tools db_engine_type get_engine_type() const { return m_engine_type; } std::string get_engine_name() const; std::string get_config_folder() const { return m_config_folder; } - + std::string get_temp_config_folder() const; + std::string get_temp_db_folder_path() const; + + std::string get_pool_db_folder_path() const; + std::shared_ptr create_backend(); private: diff --git a/src/common/pre_download.h b/src/common/pre_download.h new file mode 100644 index 00000000..210d4041 --- /dev/null +++ b/src/common/pre_download.h @@ -0,0 +1,218 @@ +// Copyright (c) 2020 Zano Project +// Copyright (c) 2012-2018 The Boolberry developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#pragma once + +#include +#include "net/http_client.h" +#include "db_backend_selector.h" +#include "crypto/crypto.h" + +namespace tools +{ + struct pre_download_entry + { + const char* url; + const char* hash; + uint64_t packed_size; + uint64_t unpacked_size; + }; + +#ifndef TESTNET + static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_lmdb_94_425000.pak", "ac0928aabf1aa350732f20476d7e798310a9f5f63b4174440588d46c344f3d55", 684683919, 1021865984 }; + static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_mdbx_94_425000.pak", "bcb01a3628c1fd16d60153aae0e9cd88432f2bad490872050e7d38fe2efe6217", 573016874, 1073725440 }; +#else + static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_testnet_lmdb_94_99000.pak", "2b2021d08d3dee50bcd8625146ba69378007d89e9a2bd636a28c539e9741a175", 83786986, 131379200 }; + static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_testnet_mdbx_94_99000.pak", "017598ebbbedd45c65870b290387ab1ca5bdd813f0384739422ed4bf16f21ef8", 166013858, 268431360 }; +#endif + + template + bool process_predownload(const boost::program_options::variables_map& vm, callback_t cb_should_stop, db::db_backend_selector& dbbs) + { + std::string config_folder = dbbs.get_config_folder(); + std::string working_folder = dbbs.get_db_folder_path(); + std::string db_main_file_path = working_folder + "/" + dbbs.get_db_main_file_name(); + + pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? c_pre_download_lmdb : c_pre_download_mdbx; + + // override pre-download link if necessary + std::string url = pre_download.url; + if (command_line::has_arg(vm, command_line::arg_predownload_link)) + url = command_line::get_arg(vm, command_line::arg_predownload_link); + + boost::system::error_code ec; + uint64_t sz = boost::filesystem::file_size(db_main_file_path, ec); + if (!(ec || (pre_download.unpacked_size > sz && pre_download.unpacked_size - sz > 500000000) || command_line::has_arg(vm, command_line::arg_explicit_predownload)) ) + { + LOG_PRINT_MAGENTA("Pre-downloading not needed (db file size = " << sz << ")", LOG_LEVEL_0); + return true; + } + + // okay, let's download + + std::string downloading_file_path = db_main_file_path + ".download"; + + LOG_PRINT_MAGENTA("Trying to download blockchain database file from " << url << " ...", LOG_LEVEL_0); + epee::net_utils::http::interruptible_http_client cl; + + crypto::stream_cn_hash hash_stream; + auto last_update = std::chrono::system_clock::now(); + + auto cb = [&hash_stream, &last_update, &cb_should_stop](const std::string& buff, uint64_t total_bytes, uint64_t received_bytes) + { + if (cb_should_stop(total_bytes, received_bytes)) + { + LOG_PRINT_MAGENTA(ENDL << "Interrupting download", LOG_LEVEL_0); + return false; + } + + hash_stream.update(buff.data(), buff.size()); + + auto dif = std::chrono::system_clock::now() - last_update; + if (dif >= std::chrono::milliseconds(300)) + { + std::cout << "Received " << received_bytes / 1048576 << " of " << total_bytes / 1048576 << " MiB ( " << std::fixed << std::setprecision(1) << 100.0 * received_bytes / total_bytes << " %)\r"; + last_update = std::chrono::system_clock::now(); + } + + return true; + }; + + tools::create_directories_if_necessary(working_folder); + bool r = cl.download_and_unzip(cb, downloading_file_path, url, 1000 /* timout */); + if (!r) + { + LOG_PRINT_RED("Download failed", LOG_LEVEL_0); + return false; + } + + crypto::hash data_hash = hash_stream.calculate_hash(); + if (epee::string_tools::pod_to_hex(data_hash) != pre_download.hash) + { + LOG_ERROR("hash missmatch in downloaded file, got: " << epee::string_tools::pod_to_hex(data_hash) << ", expected: " << pre_download.hash); + return false; + } + + LOG_PRINT_GREEN("Download succeeded, hash " << pre_download.hash << " is correct" , LOG_LEVEL_0); + + if (!command_line::has_arg(vm, command_line::arg_validate_predownload)) + { + boost::filesystem::remove(db_main_file_path, ec); + if (ec) + { + LOG_ERROR("Failed to remove " << db_main_file_path); + return false; + } + LOG_PRINT_L1("Removed " << db_main_file_path); + + boost::filesystem::rename(downloading_file_path, db_main_file_path, ec); + if (ec) + { + LOG_ERROR("Failed to rename " << downloading_file_path << " -> " << db_main_file_path); + return false; + } + LOG_PRINT_L1("Renamed " << downloading_file_path << " -> " << db_main_file_path); + + LOG_PRINT_GREEN("Blockchain successfully replaced with the new pre-downloaded data file", LOG_LEVEL_0); + return true; + } + + // + // paranoid mode + // move downloaded blockchain into a temporary folder + // + std::string path_to_temp_datafolder = dbbs.get_temp_config_folder(); + std::string path_to_temp_blockchain = dbbs.get_temp_db_folder_path(); + std::string path_to_temp_blockchain_file = path_to_temp_blockchain + "/" + dbbs.get_db_main_file_name(); + + tools::create_directories_if_necessary(path_to_temp_blockchain); + boost::filesystem::rename(downloading_file_path, path_to_temp_blockchain_file, ec); + if (ec) + { + LOG_ERROR("Rename failed: " << downloading_file_path << " -> " << path_to_temp_blockchain_file); + return false; + } + + // remove old blockchain database from disk + boost::filesystem::remove_all(working_folder, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << working_folder); + return false; + } + + std::string pool_db_path = dbbs.get_pool_db_folder_path(); + boost::filesystem::remove_all(pool_db_path, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << pool_db_path); + return false; + } + + // source core + currency::core source_core(nullptr); + boost::program_options::variables_map source_core_vm; + source_core_vm.insert(std::make_pair("data-dir", boost::program_options::variable_value(path_to_temp_datafolder, false))); + source_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false))); + //source_core_vm.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false))); + + db::db_backend_selector source_core_dbbs; + r = source_core_dbbs.init(source_core_vm); + CHECK_AND_ASSERT_MES(r, false, "failed to init source_core_dbbs"); + + r = source_core.init(source_core_vm, source_core_dbbs); + CHECK_AND_ASSERT_MES(r, false, "Failed to init source core"); + + // target core + currency::core target_core(nullptr); + boost::program_options::variables_map target_core_vm(vm); + target_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false))); + //vm_with_fast_sync.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false))); + + db::db_backend_selector target_core_dbbs; + r = target_core_dbbs.init(target_core_vm); + CHECK_AND_ASSERT_MES(r, false, "failed to init target_core_dbbs"); + + r = target_core.init(target_core_vm, target_core_dbbs); + CHECK_AND_ASSERT_MES(r, false, "Failed to init target core"); + + CHECK_AND_ASSERT_MES(target_core.get_top_block_height() == 0, false, "Target blockchain initialized not empty"); + uint64_t total_blocks = source_core.get_current_blockchain_size(); + + LOG_PRINT_GREEN("Manually processing blocks from 1 to " << total_blocks << "...", LOG_LEVEL_0); + + for (uint64_t i = 1; i != total_blocks; i++) + { + std::list blocks; + std::list txs; + bool r = source_core.get_blocks(i, 1, blocks, txs); + CHECK_AND_ASSERT_MES(r && blocks.size()==1, false, "Filed to get block " << i << " from core"); + currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash); + for (auto& tx : txs) + { + r = target_core.handle_incoming_tx(tx, tvc, true /* kept_by_block */); + CHECK_AND_ASSERT_MES(r && tvc.m_added_to_pool == true, false, "Failed to add a tx from block " << i << " from core"); + } + currency::block_verification_context bvc = AUTO_VAL_INIT(bvc); + r = target_core.handle_incoming_block(*blocks.begin(), bvc); + CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain == true, false, "Failed to add block " << i << " to core"); + if (!(i % 100)) + std::cout << "Block " << i << "(" << (i * 100) / total_blocks << "%) \r"; + + if (cb_should_stop(total_blocks, i)) + { + LOG_PRINT_MAGENTA(ENDL << "Interrupting updating db...", LOG_LEVEL_0); + return false; + } + } + + LOG_PRINT_GREEN("Processing finished, " << total_blocks << " successfully added.", LOG_LEVEL_0); + target_core.deinit(); + source_core.deinit(); + + return true; + } +} + diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index ccd4644b..5293dc2d 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -236,7 +236,7 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0); boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path)); } - ; + const std::string db_folder_path = dbbs.get_db_folder_path(); LOG_PRINT_L0("Loading blockchain from " << db_folder_path); diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index cab606f6..bf5c7b12 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -184,6 +184,34 @@ namespace currency return true; } //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ /* = null_hash */) + { + TIME_MEASURE_START_MS(wait_lock_time); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); + TIME_MEASURE_FINISH_MS(wait_lock_time); + + crypto::hash tx_hash = tx_hash_; + if (tx_hash == null_hash) + tx_hash = get_transaction_hash(tx); + + TIME_MEASURE_START_MS(add_new_tx_time); + bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block); + TIME_MEASURE_FINISH_MS(add_new_tx_time); + + if(tvc.m_verification_failed) + {LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);} + else if(tvc.m_verification_impossible) + {LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);} + + if (tvc.m_added_to_pool) + { + LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool"); + } + LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX1]: timing " << wait_lock_time + << "/" << add_new_tx_time); + return r; + } + //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block) { CHECK_AND_ASSERT_MES(!kept_by_block, false, "Transaction associated with block came throw handle_incoming_tx!(not allowed anymore)"); @@ -212,7 +240,6 @@ namespace currency } TIME_MEASURE_FINISH_MS(parse_tx_time); - TIME_MEASURE_START_MS(check_tx_semantic_time); if(!validate_tx_semantic(tx, tx_blob.size())) { @@ -222,23 +249,10 @@ namespace currency } TIME_MEASURE_FINISH_MS(check_tx_semantic_time); - TIME_MEASURE_START_MS(add_new_tx_time); - bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block); - TIME_MEASURE_FINISH_MS(add_new_tx_time); - - if(tvc.m_verification_failed) - {LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);} - else if(tvc.m_verification_impossible) - {LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);} - - if (tvc.m_added_to_pool) - { - LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool"); - } - LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX]: timing " << wait_lock_time + bool r = handle_incoming_tx(tx, tvc, kept_by_block, tx_hash); + LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX2]: timing " << wait_lock_time << "/" << parse_tx_time - << "/" << check_tx_semantic_time - << "/" << add_new_tx_time); + << "/" << check_tx_semantic_time); return r; } //----------------------------------------------------------------------------------------------- diff --git a/src/currency_core/currency_core.h b/src/currency_core/currency_core.h index ec1ae672..f6690bec 100644 --- a/src/currency_core/currency_core.h +++ b/src/currency_core/currency_core.h @@ -42,6 +42,7 @@ namespace currency core(i_currency_protocol* pprotocol); bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, currency_connection_context& context)const ; bool on_idle(); + bool handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ = null_hash); bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block); bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true); bool handle_incoming_block(const block& b, block_verification_context& bvc, bool update_miner_blocktemplate = true); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 4f7ec0d2..4012e3b7 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -27,6 +27,7 @@ using namespace epee; #include "version.h" #include "currency_core/core_tools.h" #include "common/callstack_helper.h" +#include "common/pre_download.h" #include @@ -148,6 +149,11 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, command_line::arg_disable_stop_on_low_free_space); command_line::add_arg(desc_cmd_sett, command_line::arg_enable_offers_service); + command_line::add_arg(desc_cmd_sett, command_line::arg_no_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_explicit_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link); + arg_market_disable.default_value = true; arg_market_disable.not_use_default = false; @@ -276,6 +282,17 @@ int main(int argc, char* argv[]) res = dbbs.init(vm); CHECK_AND_ASSERT_MES(res, EXIT_FAILURE, "db_backend_selector failed to initialize"); + //do pre_download if needed + if (!command_line::has_arg(vm, command_line::arg_no_predownload) || command_line::has_arg(vm, command_line::arg_explicit_predownload)) + { + tools::process_predownload(vm, [&](uint64_t total_bytes, uint64_t received_bytes){ + return static_cast::connection_context> *>(&p2psrv)->is_stop_signal_sent(); + }, dbbs); + if (static_cast::connection_context>*>(&p2psrv)->is_stop_signal_sent()) + return 1; + } + + //initialize objects LOG_PRINT_L0("Initializing p2p server..."); res = p2psrv.init(vm);