Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity

blockchain database predownloading: first version

Browse source
This commit is contained in:
sowle 2020-03-12 18:07:31 +03:00
parent 05b5f2294b
commit 5567275518
No known key found for this signature in database
GPG key ID: C07A24B2D89D49FC
9 changed files with 302 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/common/command_line.cpp
View file

@ -31,4 +31,10 @@ namespace command_line
const arg_descriptor<bool> arg_disable_stop_on_low_free_space = { "disable-stop-on-low-free-space", "Do not stop the daemon if free space at data dir is critically low", false, true };
const arg_descriptor<bool> arg_enable_offers_service = { "enable-offers-service", "Enables marketplace feature", false, false};
const arg_descriptor<std::string> arg_db_engine = { "db-engine", "Specify database engine for storage. May be \"lmdb\"(default) or \"mdbx\"", ARG_DB_ENGINE_LMDB, false };
const arg_descriptor<bool> arg_no_predownload = { "no-predownload", "Do not pre-download blockchain database", };
const arg_descriptor<bool> arg_explicit_predownload = { "explicit-predownload", "Pre-download blockchain database regardless of it's status", };
const arg_descriptor<bool> arg_validate_predownload = { "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database", };
const arg_descriptor<std::string> arg_predownload_link = { "predownload-link", "Override url for blockchain database pre-downloading", "", true };
}

4
src/common/command_line.h
View file

@ -192,4 +192,8 @@ namespace command_line
extern const arg_descriptor<bool> arg_disable_stop_on_low_free_space;
extern const arg_descriptor<bool> arg_enable_offers_service;
extern const arg_descriptor<std::string> arg_db_engine;
extern const arg_descriptor<bool> arg_no_predownload;
extern const arg_descriptor<bool> arg_explicit_predownload;
extern const arg_descriptor<bool> arg_validate_predownload;
extern const arg_descriptor<std::string> arg_predownload_link;
}

23
src/common/db_backend_selector.cpp
View file

@ -8,9 +8,8 @@
#include "db_backend_lmdb.h"
#include "db_backend_mdbx.h"
#define LMDB_MAIN_FILE_NAME "data.mdb"
#define MDBX_MAIN_FILE_NAME "mdbx.dat"
#define LMDB_MAIN_FILE_NAME "data.mdb"
#define MDBX_MAIN_FILE_NAME "mdbx.dat"
namespace tools
{
@ -65,10 +64,20 @@ namespace db
std::string db_backend_selector::get_db_folder_path() const
{
//CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited");
return m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX;
}
std::string db_backend_selector::get_temp_db_folder_path() const
{
//CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited");
return get_temp_config_folder() + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX;
}
std::string db_backend_selector::get_pool_db_folder_path() const
{
return m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX;
}
std::string db_backend_selector::get_db_main_file_name() const
{
switch (m_engine_type)
@ -111,6 +120,12 @@ namespace db
}
}
std::string db_backend_selector::get_temp_config_folder() const
{
return m_config_folder + "_TEMP";
}
} // namespace db

6
src/common/db_backend_selector.h
View file

@ -26,7 +26,11 @@ namespace tools
db_engine_type get_engine_type() const { return m_engine_type; }
std::string get_engine_name() const;
std::string get_config_folder() const { return m_config_folder; }
std::string get_temp_config_folder() const;
std::string get_temp_db_folder_path() const;
std::string get_pool_db_folder_path() const;
std::shared_ptr<tools::db::i_db_backend> create_backend();
private:

218
src/common/pre_download.h Normal file
View file

@ -0,0 +1,218 @@
// Copyright (c) 2020 Zano Project
// Copyright (c) 2012-2018 The Boolberry developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <boost/program_options.hpp>
#include "net/http_client.h"
#include "db_backend_selector.h"
#include "crypto/crypto.h"
namespace tools
{
struct pre_download_entry
{
const char* url;
const char* hash;
uint64_t packed_size;
uint64_t unpacked_size;
};
#ifndef TESTNET
static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_lmdb_94_425000.pak", "ac0928aabf1aa350732f20476d7e798310a9f5f63b4174440588d46c344f3d55", 684683919, 1021865984 };
static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_mdbx_94_425000.pak", "bcb01a3628c1fd16d60153aae0e9cd88432f2bad490872050e7d38fe2efe6217", 573016874, 1073725440 };
#else
static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_testnet_lmdb_94_99000.pak", "2b2021d08d3dee50bcd8625146ba69378007d89e9a2bd636a28c539e9741a175", 83786986, 131379200 };
static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_testnet_mdbx_94_99000.pak", "017598ebbbedd45c65870b290387ab1ca5bdd813f0384739422ed4bf16f21ef8", 166013858, 268431360 };
#endif
template<class callback_t>
bool process_predownload(const boost::program_options::variables_map& vm, callback_t cb_should_stop, db::db_backend_selector& dbbs)
{
std::string config_folder = dbbs.get_config_folder();
std::string working_folder = dbbs.get_db_folder_path();
std::string db_main_file_path = working_folder + "/" + dbbs.get_db_main_file_name();
pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? c_pre_download_lmdb : c_pre_download_mdbx;
// override pre-download link if necessary
std::string url = pre_download.url;
if (command_line::has_arg(vm, command_line::arg_predownload_link))
url = command_line::get_arg(vm, command_line::arg_predownload_link);
boost::system::error_code ec;
uint64_t sz = boost::filesystem::file_size(db_main_file_path, ec);
if (!(ec || (pre_download.unpacked_size > sz && pre_download.unpacked_size - sz > 500000000) || command_line::has_arg(vm, command_line::arg_explicit_predownload)) )
{
LOG_PRINT_MAGENTA("Pre-downloading not needed (db file size = " << sz << ")", LOG_LEVEL_0);
return true;
}
// okay, let's download
std::string downloading_file_path = db_main_file_path + ".download";
LOG_PRINT_MAGENTA("Trying to download blockchain database file from " << url << " ...", LOG_LEVEL_0);
epee::net_utils::http::interruptible_http_client cl;
crypto::stream_cn_hash hash_stream;
auto last_update = std::chrono::system_clock::now();
auto cb = [&hash_stream, &last_update, &cb_should_stop](const std::string& buff, uint64_t total_bytes, uint64_t received_bytes)
{
if (cb_should_stop(total_bytes, received_bytes))
{
LOG_PRINT_MAGENTA(ENDL << "Interrupting download", LOG_LEVEL_0);
return false;
}
hash_stream.update(buff.data(), buff.size());
auto dif = std::chrono::system_clock::now() - last_update;
if (dif >= std::chrono::milliseconds(300))
{
std::cout << "Received " << received_bytes / 1048576 << " of " << total_bytes / 1048576 << " MiB ( " << std::fixed << std::setprecision(1) << 100.0 * received_bytes / total_bytes << " %)\r";
last_update = std::chrono::system_clock::now();
}
return true;
};
tools::create_directories_if_necessary(working_folder);
bool r = cl.download_and_unzip(cb, downloading_file_path, url, 1000 /* timout */);
if (!r)
{
LOG_PRINT_RED("Download failed", LOG_LEVEL_0);
return false;
}
crypto::hash data_hash = hash_stream.calculate_hash();
if (epee::string_tools::pod_to_hex(data_hash) != pre_download.hash)
{
LOG_ERROR("hash missmatch in downloaded file, got: " << epee::string_tools::pod_to_hex(data_hash) << ", expected: " << pre_download.hash);
return false;
}
LOG_PRINT_GREEN("Download succeeded, hash " << pre_download.hash << " is correct" , LOG_LEVEL_0);
if (!command_line::has_arg(vm, command_line::arg_validate_predownload))
{
boost::filesystem::remove(db_main_file_path, ec);
if (ec)
{
LOG_ERROR("Failed to remove " << db_main_file_path);
return false;
}
LOG_PRINT_L1("Removed " << db_main_file_path);
boost::filesystem::rename(downloading_file_path, db_main_file_path, ec);
if (ec)
{
LOG_ERROR("Failed to rename " << downloading_file_path << " -> " << db_main_file_path);
return false;
}
LOG_PRINT_L1("Renamed " << downloading_file_path << " -> " << db_main_file_path);
LOG_PRINT_GREEN("Blockchain successfully replaced with the new pre-downloaded data file", LOG_LEVEL_0);
return true;
}
//
// paranoid mode
// move downloaded blockchain into a temporary folder
//
std::string path_to_temp_datafolder = dbbs.get_temp_config_folder();
std::string path_to_temp_blockchain = dbbs.get_temp_db_folder_path();
std::string path_to_temp_blockchain_file = path_to_temp_blockchain + "/" + dbbs.get_db_main_file_name();
tools::create_directories_if_necessary(path_to_temp_blockchain);
boost::filesystem::rename(downloading_file_path, path_to_temp_blockchain_file, ec);
if (ec)
{
LOG_ERROR("Rename failed: " << downloading_file_path << " -> " << path_to_temp_blockchain_file);
return false;
}
// remove old blockchain database from disk
boost::filesystem::remove_all(working_folder, ec);
if (ec)
{
LOG_ERROR("Failed to remove all from " << working_folder);
return false;
}
std::string pool_db_path = dbbs.get_pool_db_folder_path();
boost::filesystem::remove_all(pool_db_path, ec);
if (ec)
{
LOG_ERROR("Failed to remove all from " << pool_db_path);
return false;
}
// source core
currency::core source_core(nullptr);
boost::program_options::variables_map source_core_vm;
source_core_vm.insert(std::make_pair("data-dir", boost::program_options::variable_value(path_to_temp_datafolder, false)));
source_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false)));
//source_core_vm.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false)));
db::db_backend_selector source_core_dbbs;
r = source_core_dbbs.init(source_core_vm);
CHECK_AND_ASSERT_MES(r, false, "failed to init source_core_dbbs");
r = source_core.init(source_core_vm, source_core_dbbs);
CHECK_AND_ASSERT_MES(r, false, "Failed to init source core");
// target core
currency::core target_core(nullptr);
boost::program_options::variables_map target_core_vm(vm);
target_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false)));
//vm_with_fast_sync.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false)));
db::db_backend_selector target_core_dbbs;
r = target_core_dbbs.init(target_core_vm);
CHECK_AND_ASSERT_MES(r, false, "failed to init target_core_dbbs");
r = target_core.init(target_core_vm, target_core_dbbs);
CHECK_AND_ASSERT_MES(r, false, "Failed to init target core");
CHECK_AND_ASSERT_MES(target_core.get_top_block_height() == 0, false, "Target blockchain initialized not empty");
uint64_t total_blocks = source_core.get_current_blockchain_size();
LOG_PRINT_GREEN("Manually processing blocks from 1 to " << total_blocks << "...", LOG_LEVEL_0);
for (uint64_t i = 1; i != total_blocks; i++)
{
std::list<currency::block> blocks;
std::list<currency::transaction> txs;
bool r = source_core.get_blocks(i, 1, blocks, txs);
CHECK_AND_ASSERT_MES(r && blocks.size()==1, false, "Filed to get block " << i << " from core");
currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash);
for (auto& tx : txs)
{
r = target_core.handle_incoming_tx(tx, tvc, true /* kept_by_block */);
CHECK_AND_ASSERT_MES(r && tvc.m_added_to_pool == true, false, "Failed to add a tx from block " << i << " from core");
}
currency::block_verification_context bvc = AUTO_VAL_INIT(bvc);
r = target_core.handle_incoming_block(*blocks.begin(), bvc);
CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain == true, false, "Failed to add block " << i << " to core");
if (!(i % 100))
std::cout << "Block " << i << "(" << (i * 100) / total_blocks << "%) \r";
if (cb_should_stop(total_blocks, i))
{
LOG_PRINT_MAGENTA(ENDL << "Interrupting updating db...", LOG_LEVEL_0);
return false;
}
}
LOG_PRINT_GREEN("Processing finished, " << total_blocks << " successfully added.", LOG_LEVEL_0);
target_core.deinit();
source_core.deinit();
return true;
}
}

2
src/currency_core/blockchain_storage.cpp
View file

@ -236,7 +236,7 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro
LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0);
boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path));
}
;
const std::string db_folder_path = dbbs.get_db_folder_path();
LOG_PRINT_L0("Loading blockchain from " << db_folder_path);

48
src/currency_core/currency_core.cpp
View file

@ -184,6 +184,34 @@ namespace currency
return true;
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ /* = null_hash */)
{
TIME_MEASURE_START_MS(wait_lock_time);
CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
TIME_MEASURE_FINISH_MS(wait_lock_time);
crypto::hash tx_hash = tx_hash_;
if (tx_hash == null_hash)
tx_hash = get_transaction_hash(tx);
TIME_MEASURE_START_MS(add_new_tx_time);
bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block);
TIME_MEASURE_FINISH_MS(add_new_tx_time);
if(tvc.m_verification_failed)
{LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);}
else if(tvc.m_verification_impossible)
{LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);}
if (tvc.m_added_to_pool)
{
LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool");
}
LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX1]: timing " << wait_lock_time
<< "/" << add_new_tx_time);
return r;
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block)
{
CHECK_AND_ASSERT_MES(!kept_by_block, false, "Transaction associated with block came throw handle_incoming_tx!(not allowed anymore)");
@ -212,7 +240,6 @@ namespace currency
}
TIME_MEASURE_FINISH_MS(parse_tx_time);
TIME_MEASURE_START_MS(check_tx_semantic_time);
if(!validate_tx_semantic(tx, tx_blob.size()))
{
@ -222,23 +249,10 @@ namespace currency
}
TIME_MEASURE_FINISH_MS(check_tx_semantic_time);
TIME_MEASURE_START_MS(add_new_tx_time);
bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block);
TIME_MEASURE_FINISH_MS(add_new_tx_time);
if(tvc.m_verification_failed)
{LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);}
else if(tvc.m_verification_impossible)
{LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);}
if (tvc.m_added_to_pool)
{
LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool");
}
LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX]: timing " << wait_lock_time
bool r = handle_incoming_tx(tx, tvc, kept_by_block, tx_hash);
LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX2]: timing " << wait_lock_time
<< "/" << parse_tx_time
<< "/" << check_tx_semantic_time
<< "/" << add_new_tx_time);
<< "/" << check_tx_semantic_time);
return r;
}
//-----------------------------------------------------------------------------------------------

1
src/currency_core/currency_core.h
View file

@ -42,6 +42,7 @@ namespace currency
core(i_currency_protocol* pprotocol);
bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, currency_connection_context& context)const ;
bool on_idle();
bool handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ = null_hash);
bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block);
bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true);
bool handle_incoming_block(const block& b, block_verification_context& bvc, bool update_miner_blocktemplate = true);

17
src/daemon/daemon.cpp
View file

@ -27,6 +27,7 @@ using namespace epee;
#include "version.h"
#include "currency_core/core_tools.h"
#include "common/callstack_helper.h"
#include "common/pre_download.h"
#include <cstdlib>
@ -148,6 +149,11 @@ int main(int argc, char* argv[])
command_line::add_arg(desc_cmd_sett, command_line::arg_disable_stop_on_low_free_space);
command_line::add_arg(desc_cmd_sett, command_line::arg_enable_offers_service);
command_line::add_arg(desc_cmd_sett, command_line::arg_no_predownload);
command_line::add_arg(desc_cmd_sett, command_line::arg_explicit_predownload);
command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload);
command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link);
arg_market_disable.default_value = true;
arg_market_disable.not_use_default = false;
@ -276,6 +282,17 @@ int main(int argc, char* argv[])
res = dbbs.init(vm);
CHECK_AND_ASSERT_MES(res, EXIT_FAILURE, "db_backend_selector failed to initialize");
//do pre_download if needed
if (!command_line::has_arg(vm, command_line::arg_no_predownload) || command_line::has_arg(vm, command_line::arg_explicit_predownload))
{
tools::process_predownload(vm, [&](uint64_t total_bytes, uint64_t received_bytes){
return static_cast<nodetool::i_p2p_endpoint<currency::t_currency_protocol_handler<currency::core>::connection_context> *>(&p2psrv)->is_stop_signal_sent();
}, dbbs);
if (static_cast<nodetool::i_p2p_endpoint<currency::t_currency_protocol_handler<currency::core>::connection_context>*>(&p2psrv)->is_stop_signal_sent())
return 1;
}
//initialize objects
LOG_PRINT_L0("Initializing p2p server...");
res = p2psrv.init(vm);