From 1caa86c499faca9452cd35b589e6403ed732767d Mon Sep 17 00:00:00 2001 From: sowle Date: Sat, 17 Aug 2019 07:10:37 +0300 Subject: [PATCH] DB: fixed a bug in tx pool major compartibility option storing, implemented more robust two-stage lmdb opening for blockchain and tx pool; added handling for 0.9.18->0.9.24 lmdb migration for case, when a user has >128GB of free space --- src/currency_core/blockchain_storage.cpp | 169 +++++++++++++---------- src/currency_core/blockchain_storage.h | 2 +- src/currency_core/tx_pool.cpp | 116 ++++++++++------ src/currency_core/tx_pool.h | 2 +- 4 files changed, 173 insertions(+), 116 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 76f8a9ff..61cf312b 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -76,7 +76,6 @@ DISABLE_VS_WARNINGS(4267) namespace { - const command_line::arg_descriptor arg_db_cache_l1 = { "db-cache-l1", "Specify size of memory mapped db cache file", 0, true }; const command_line::arg_descriptor arg_db_cache_l2 = { "db-cache-l2", "Specify cached elements in db helpers", 0, true }; } @@ -152,7 +151,6 @@ std::shared_ptr blockchain_storage::get_tx(const crypto::hash &id) //------------------------------------------------------------------ void blockchain_storage::init_options(boost::program_options::options_description& desc) { - command_line::add_arg(desc, arg_db_cache_l1); command_line::add_arg(desc, arg_db_cache_l2); } //------------------------------------------------------------------ @@ -215,77 +213,104 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro return false; } - uint64_t cache_size = CACHE_SIZE; - if (command_line::has_arg(vm, arg_db_cache_l1)) - { - cache_size = command_line::get_arg(vm, arg_db_cache_l1); - } - LOG_PRINT_GREEN("Using db file cache size(L1): " << cache_size, LOG_LEVEL_0); - - m_config_folder = config_folder; - LOG_PRINT_L0("Loading blockchain..."); - const std::string folder_name = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME; - tools::create_directories_if_necessary(folder_name); - bool res = m_db.open(folder_name, cache_size); - CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << folder_name); + const std::string db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME; + LOG_PRINT_L0("Loading blockchain from " << db_folder_path); - res = m_db_blocks.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_blocks_index.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS_INDEX); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_transactions.init(BLOCKCHAIN_STORAGE_CONTAINER_TRANSACTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_spent_keys.init(BLOCKCHAIN_STORAGE_CONTAINER_SPENT_KEYS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_outputs.init(BLOCKCHAIN_STORAGE_CONTAINER_OUTPUTS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_multisig_outs.init(BLOCKCHAIN_STORAGE_CONTAINER_MULTISIG_OUTS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_solo_options.init(BLOCKCHAIN_STORAGE_CONTAINER_SOLO_OPTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_aliases.init(BLOCKCHAIN_STORAGE_CONTAINER_ALIASES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_addr_to_alias.init(BLOCKCHAIN_STORAGE_CONTAINER_ADDR_TO_ALIAS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_per_block_gindex_incs.init(BLOCKCHAIN_STORAGE_CONTAINER_GINDEX_INCS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - - if (command_line::has_arg(vm, arg_db_cache_l2)) + bool db_opened_okay = false; + for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no) { - uint64_t cache_size = command_line::get_arg(vm, arg_db_cache_l2); - LOG_PRINT_GREEN("Using db items cache size(L2): " << cache_size, LOG_LEVEL_0); - m_db_blocks_index.set_cache_size(cache_size); - m_db_blocks.set_cache_size(cache_size); - m_db_blocks_index.set_cache_size(cache_size); - m_db_transactions.set_cache_size(cache_size); - m_db_spent_keys.set_cache_size(cache_size); - //m_db_outputs.set_cache_size(cache_size); - m_db_multisig_outs.set_cache_size(cache_size); - m_db_solo_options.set_cache_size(cache_size); - m_db_aliases.set_cache_size(cache_size); - m_db_addr_to_alias.set_cache_size(cache_size); + bool res = m_db.open(db_folder_path); + if (!res) + { + // if DB could not be opened -- try to remove the whole folder and re-open DB + LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0); + boost::filesystem::remove_all(db_folder_path); + res = m_db.open(db_folder_path); + CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt"); + } + + res = m_db_blocks.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_blocks_index.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS_INDEX); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_transactions.init(BLOCKCHAIN_STORAGE_CONTAINER_TRANSACTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_spent_keys.init(BLOCKCHAIN_STORAGE_CONTAINER_SPENT_KEYS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_outputs.init(BLOCKCHAIN_STORAGE_CONTAINER_OUTPUTS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_multisig_outs.init(BLOCKCHAIN_STORAGE_CONTAINER_MULTISIG_OUTS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_solo_options.init(BLOCKCHAIN_STORAGE_CONTAINER_SOLO_OPTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_aliases.init(BLOCKCHAIN_STORAGE_CONTAINER_ALIASES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_addr_to_alias.init(BLOCKCHAIN_STORAGE_CONTAINER_ADDR_TO_ALIAS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_per_block_gindex_incs.init(BLOCKCHAIN_STORAGE_CONTAINER_GINDEX_INCS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + + if (command_line::has_arg(vm, arg_db_cache_l2)) + { + uint64_t cache_size = command_line::get_arg(vm, arg_db_cache_l2); + LOG_PRINT_GREEN("Using db items cache size(L2): " << cache_size, LOG_LEVEL_0); + m_db_blocks_index.set_cache_size(cache_size); + m_db_blocks.set_cache_size(cache_size); + m_db_blocks_index.set_cache_size(cache_size); + m_db_transactions.set_cache_size(cache_size); + m_db_spent_keys.set_cache_size(cache_size); + //m_db_outputs.set_cache_size(cache_size); + m_db_multisig_outs.set_cache_size(cache_size); + m_db_solo_options.set_cache_size(cache_size); + m_db_aliases.set_cache_size(cache_size); + m_db_addr_to_alias.set_cache_size(cache_size); + } + + bool need_reinit = false; + if (m_db_blocks.size() != 0) + { + if (m_db_storage_major_compatibility_version != BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION) + { + need_reinit = true; + LOG_PRINT_MAGENTA("DB storage needs reinit because it has major compatibility ver " << m_db_storage_major_compatibility_version << ", expected : " << BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); + } + else if (m_db_storage_minor_compatibility_version != BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION) + { + // nothing + } + } + + if (need_reinit) + { + LOG_PRINT_L1("DB at " << db_folder_path << " is about to be deleted and re-created..."); + m_db_blocks.deinit(); + m_db_blocks_index.deinit(); + m_db_transactions.deinit(); + m_db_spent_keys.deinit(); + m_db_outputs.deinit(); + m_db_multisig_outs.deinit(); + m_db_solo_options.deinit(); + m_db_aliases.deinit(); + m_db_addr_to_alias.deinit(); + m_db_per_block_gindex_incs.deinit(); + m_db.close(); + size_t files_removed = boost::filesystem::remove_all(db_folder_path); + LOG_PRINT_L1(files_removed << " files at " << db_folder_path << " removed"); + + // try to re-create DB and re-init containers + continue; + } + + db_opened_okay = true; + break; } - bool need_reinit = false; - bool need_reinit_medians = false; + CHECK_AND_ASSERT_MES(db_opened_okay, false, "All attempts to open DB at " << db_folder_path << " failed"); + if (!m_db_blocks.size()) { - need_reinit = true; - } - else if (m_db_storage_major_compatibility_version != BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION) - { - need_reinit = true; - LOG_PRINT_MAGENTA("DB storage needs reinit because it has major compatibility ver " << m_db_storage_major_compatibility_version << ", expected : " << BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); - } - else if (m_db_storage_minor_compatibility_version != BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION) - { - if (m_db_storage_minor_compatibility_version < 1) - need_reinit_medians = true; - } - if (need_reinit) - { - clear(); + // empty DB: generate and add genesis block block bl = boost::value_initialized(); block_verification_context bvc = boost::value_initialized(); generate_genesis_block(bl); @@ -293,14 +318,8 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro CHECK_AND_ASSERT_MES(!bvc.m_verification_failed, false, "Failed to add genesis block to blockchain"); LOG_PRINT_MAGENTA("Storage initialized with genesis", LOG_LEVEL_0); } - if (need_reinit_medians) - { - bool r = rebuild_tx_fee_medians(); - CHECK_AND_ASSERT_MES(r, false, "failed to rebuild_tx_fee_medians()"); - } - - initialize_db_solo_options_values(); + store_db_solo_options_values(); m_services_mgr.init(config_folder, vm); @@ -374,7 +393,7 @@ void blockchain_storage::patch_out_if_needed(txout_to_key& out, const crypto::h } } //------------------------------------------------------------------ -void blockchain_storage::initialize_db_solo_options_values() +void blockchain_storage::store_db_solo_options_values() { m_db.begin_transaction(); m_db_storage_major_compatibility_version = BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION; @@ -505,7 +524,7 @@ bool blockchain_storage::clear() m_db_transactions.clear(); m_db_spent_keys.clear(); m_db_solo_options.clear(); - initialize_db_solo_options_values(); + store_db_solo_options_values(); m_db_outputs.clear(); m_db_multisig_outs.clear(); m_db_aliases.clear(); diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 4d6f7d4a..9a215268 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -535,7 +535,7 @@ namespace currency bool init_tx_fee_median(); bool update_tx_fee_median(); - void initialize_db_solo_options_values(); + void store_db_solo_options_values(); bool set_lost_tx_unmixable(); bool set_lost_tx_unmixable_for_height(uint64_t height); void patch_out_if_needed(txout_to_key& out, const crypto::hash& tx_id, uint64_t n)const ; diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index d750de58..99838ead 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -29,6 +29,7 @@ DISABLE_VS_WARNINGS(4244 4345 4503) //'boost::foreach_detail_::or_' : decorated #define TRANSACTION_POOL_CONTAINER_ALIAS_ADDRESSES "alias_addresses" #define TRANSACTION_POOL_CONTAINER_KEY_IMAGES "key_images" #define TRANSACTION_POOL_CONTAINER_SOLO_OPTIONS "solo" +#define TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 92 // DON'T CHANGE THIS, if you need to resync db! Change TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION instead! #define TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION + 1 #undef LOG_DEFAULT_CHANNEL @@ -48,7 +49,7 @@ namespace currency m_db_key_images_set(m_db), m_db_alias_names(m_db), m_db_alias_addresses(m_db), - m_db_storage_major_compatibility_version(TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION, m_db_solo_options) + m_db_storage_major_compatibility_version(TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION, m_db_solo_options) { } @@ -1108,7 +1109,7 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::initialize_db_solo_options_values() + void tx_memory_pool::store_db_solo_options_values() { m_db.begin_transaction(); m_db_storage_major_compatibility_version = TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION; @@ -1119,46 +1120,83 @@ namespace currency { m_config_folder = config_folder; - uint64_t cache_size = CACHE_SIZE; - LOG_PRINT_GREEN("Using pool db file cache size(L1): " << cache_size, LOG_LEVEL_0); - LOG_PRINT_L0("Loading blockchain..."); - const std::string folder_name = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME; - tools::create_directories_if_necessary(folder_name); - bool res = m_db.open(folder_name, cache_size); - CHECK_AND_ASSERT_MES(res, false, "Failed to initialize pool database in folder: " << folder_name); - - res = m_db_transactions.init(TRANSACTION_POOL_CONTAINER_TRANSACTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_black_tx_list.init(TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_alias_names.init(TRANSACTION_POOL_CONTAINER_ALIAS_NAMES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_alias_addresses.init(TRANSACTION_POOL_CONTAINER_ALIAS_ADDRESSES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_solo_options.init(TRANSACTION_POOL_CONTAINER_SOLO_OPTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - - - m_db_transactions.set_cache_size(1000); - m_db_alias_names.set_cache_size(10000); - m_db_alias_addresses.set_cache_size(10000); - m_db_black_tx_list.set_cache_size(1000); - - bool need_reinit = false; - if (m_db_storage_major_compatibility_version != TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION) - need_reinit = true; - - if (need_reinit) + const std::string db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME; + + bool db_opened_okay = false; + for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no) { - clear(); - LOG_PRINT_MAGENTA("Tx Pool reinitialized.", LOG_LEVEL_0); - } - initialize_db_solo_options_values(); + bool res = m_db.open(db_folder_path); + if (!res) + { + // if DB could not be opened -- try to remove the whole folder and re-open DB + LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0); + boost::filesystem::remove_all(db_folder_path); + res = m_db.open(db_folder_path); + CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt"); + } - LOG_PRINT_GREEN("TX_POOL Initialized ok. (" << m_db_transactions.size() << " transactions)", LOG_LEVEL_0); + res = m_db_transactions.init(TRANSACTION_POOL_CONTAINER_TRANSACTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_black_tx_list.init(TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_alias_names.init(TRANSACTION_POOL_CONTAINER_ALIAS_NAMES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_alias_addresses.init(TRANSACTION_POOL_CONTAINER_ALIAS_ADDRESSES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_solo_options.init(TRANSACTION_POOL_CONTAINER_SOLO_OPTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + + m_db_transactions.set_cache_size(1000); + m_db_alias_names.set_cache_size(10000); + m_db_alias_addresses.set_cache_size(10000); + m_db_black_tx_list.set_cache_size(1000); + + bool need_reinit = false; + if (m_db_storage_major_compatibility_version > 0 && m_db_storage_major_compatibility_version != TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION) + { + need_reinit = true; + LOG_PRINT_MAGENTA("Tx pool DB needs reinit because it has major compatibility ver is " << m_db_storage_major_compatibility_version << ", expected: " << TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); + } + + if (need_reinit) + { + LOG_PRINT_L1("DB at " << db_folder_path << " is about to be deleted and re-created..."); + m_db_transactions.deinit(); + m_db_key_images_set.deinit(); + m_db_black_tx_list.deinit(); + m_db_alias_names.deinit(); + m_db_alias_addresses.deinit(); + m_db_solo_options.deinit(); + m_db.close(); + size_t files_removed = boost::filesystem::remove_all(db_folder_path); + LOG_PRINT_L1(files_removed << " files at " << db_folder_path << " removed"); + + // try to re-create DB and re-init containers + continue; + } + + db_opened_okay = true; + break; + } + + CHECK_AND_ASSERT_MES(db_opened_okay, false, "All attempts to open DB at " << db_folder_path << " failed"); + + store_db_solo_options_values(); + + LOG_PRINT_GREEN("tx pool loaded ok from " << db_folder_path << ", loaded " << m_db_transactions.size() << " transactions", LOG_LEVEL_0); + if (epee::log_space::log_singletone::get_log_detalisation_level() >= LOG_LEVEL_2 && m_db_transactions.size() != 0) + { + std::stringstream ss; + m_db_transactions.enumerate_items([&](uint64_t i, const crypto::hash& h, const tx_details &tx_entry) + { + ss << h << " sz: " << std::setw(5) << tx_entry.blob_size << " rcv: " << misc_utils::get_time_interval_string(time(nullptr) - tx_entry.receive_time) << " ago" << ENDL; + return true; + }); + LOG_PRINT_L2(ss.str()); + } return true; } diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index fd014b1f..d546a4a8 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -146,7 +146,7 @@ namespace currency bool remove_alias_info(const transaction& tx); bool is_valid_contract_finalization_tx(const transaction &tx)const; - void initialize_db_solo_options_values(); + void store_db_solo_options_values(); bool is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const; bool validate_alias_info(const transaction& tx, bool is_in_block)const; bool get_key_images_from_tx_pool(std::unordered_set& key_images)const;