From 57e0aa063d103f734513c7e9bb57b7cd74e2d197 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 16 Nov 2019 20:26:40 +0100 Subject: [PATCH] pool code modificated to work with onboard transactions --- contrib/epee/include/misc_language.h | 16 ++- contrib/epee/include/syncobj.h | 3 + src/currency_core/blockchain_storage.cpp | 11 +- src/currency_core/blockchain_storage.h | 2 +- .../currency_format_utils_transactions.cpp | 15 +++ .../currency_format_utils_transactions.h | 1 + src/currency_core/tx_pool.cpp | 102 +++++++++--------- src/currency_core/tx_pool.h | 21 ++-- 8 files changed, 107 insertions(+), 64 deletions(-) diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index b0cfd15c..6c2a287d 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -393,7 +393,21 @@ namespace misc_utils auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); return res.first->second; - } + } + + template + typename t_container_type::iterator it_get_or_insert_value_initialized(t_container_type& container, const typename t_container_type::key_type& key) + { + auto it = container.find(key); + if (it != container.end()) + { + return it; + } + + auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); + return res.first; + } + } // namespace misc_utils } // namespace epee diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index dffbd614..b5907538 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -698,6 +698,9 @@ namespace epee #define CRITICAL_REGION_BEGIN1(x) CRITICAL_REGION_BEGIN_VAR(x, critical_region_var1) #define CRITICAL_REGION_END() } + +#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj##.##op;obj##_lock.unlock();} + #define SHARED_CRITICAL_REGION_LOCAL(x) boost::shared_lock< boost::shared_mutex > critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_LOCAL(x) boost::unique_lock< boost::shared_mutex > critical_region_var(x) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 571e7345..6743337b 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -4900,6 +4900,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t tx_processed_count = 0; uint64_t fee_summary = 0; uint64_t burned_coins = 0; + std::list block_summary_kimages; for(const crypto::hash& tx_id : bl.tx_hashes) { @@ -4983,6 +4984,8 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt ++tx_processed_count; if (fee) block_fees.push_back(fee); + + read_keyimages_from_tx(tx, block_summary_kimages); } TIME_MEASURE_FINISH_PD(all_txs_insert_time_5); @@ -5017,8 +5020,6 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if (is_pos_bl) bei.stake_hash = proof_hash; - - ////////////////////////////////////////////////////////////////////////// //old style cumulative difficulty collecting @@ -5156,17 +5157,17 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "/" << etc_stuff_6 << "))"); - on_block_added(bei, id); + on_block_added(bei, id, block_summary_kimages); bvc.m_added_to_main_chain = true; return true; } //------------------------------------------------------------------ -void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id) +void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk) { update_next_comulative_size_limit(); m_timestamps_median_cache.clear(); - m_tx_pool.on_blockchain_inc(bei.height, id); + m_tx_pool.on_blockchain_inc(bei.height, id, bsk); update_targetdata_cache_on_block_added(bei); diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 70187fda..bc595256 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -581,7 +581,7 @@ namespace currency bool check_block_timestamp(std::vector timestamps, const block& b)const; std::vector get_last_n_blocks_timestamps(size_t n)const; const std::vector& get_txin_etc_options(const txin_v& in)const; - void on_block_added(const block_extended_info& bei, const crypto::hash& id); + void on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk); void on_block_removed(const block_extended_info& bei); void update_targetdata_cache_on_block_added(const block_extended_info& bei); void update_targetdata_cache_on_block_removed(const block_extended_info& bei); diff --git a/src/currency_core/currency_format_utils_transactions.cpp b/src/currency_core/currency_format_utils_transactions.cpp index 20ecd4e0..da78894f 100644 --- a/src/currency_core/currency_format_utils_transactions.cpp +++ b/src/currency_core/currency_format_utils_transactions.cpp @@ -256,5 +256,20 @@ namespace currency { return t_serializable_object_to_blob(tx, b_blob); } + //--------------------------------------------------------------- + bool read_keyimages_from_tx(const transaction& tx, std::list& kil) + { + std::unordered_set ki; + BOOST_FOREACH(const auto& in, tx.vin) + { + if (in.type() == typeid(txin_to_key)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); + if (!ki.insert(tokey_in.k_image).second) + return false; + } + } + return true; + } } \ No newline at end of file diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index d903f38e..603329e2 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -108,4 +108,5 @@ namespace currency size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size); blobdata tx_to_blob(const transaction& b); bool tx_to_blob(const transaction& b, blobdata& b_blob); + bool read_keyimages_from_tx(const transaction& tx, std::list& kil); } diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index aef91d5a..8a3045b1 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -50,7 +50,7 @@ namespace currency m_db_transactions(m_db), m_db_black_tx_list(m_db), m_db_solo_options(m_db), - m_db_key_images_set(m_db), +// m_db_key_images_set(m_db), m_db_alias_names(m_db), m_db_alias_addresses(m_db), m_db_storage_major_compatibility_version(TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION, m_db_solo_options) @@ -255,7 +255,7 @@ namespace currency td.receive_time = get_core_time(); m_db_transactions.set(id, td); - on_tx_add(tx, kept_by_block); + on_tx_add(id, tx, kept_by_block); TIME_MEASURE_FINISH_PD(update_db_time); return true; @@ -385,7 +385,7 @@ namespace currency blob_size = txe_tr->blob_size; fee = txe_tr->fee; m_db_transactions.erase(id); - on_tx_remove(tx, txe_tr->kept_by_block); + on_tx_remove(id, tx, txe_tr->kept_by_block); set_taken(id); return true; } @@ -472,7 +472,7 @@ namespace currency for (auto& e : to_delete) { m_db_transactions.erase(e.hash); - on_tx_remove(e.tx, e.kept_by_block); + on_tx_remove(e.hash, e.tx, e.kept_by_block); } @@ -613,8 +613,10 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) + bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk) { + + return true; } //--------------------------------------------------------------------------------- @@ -694,36 +696,33 @@ namespace currency return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::insert_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::insert_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (ki_entry_ptr.get()) - count = *ki_entry_ptr; - uint64_t count_before = count; - ++count; - m_db_key_images_set.set(tokey_in.k_image, count); - LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + const txin_to_key& tokey_in = boost::get(in); + auto& id_set = m_key_images[tokey_in.k_image]; + size_t sz_before = id_set.size(); + id_set.insert(tx_id); + LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << sz_before << " -> " << id_set.size()); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_add(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block) { - insert_key_images(tx, kept_by_block); + insert_key_images(tx_id, tx, kept_by_block); insert_alias_info(tx); return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_remove(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_remove(const crypto::hash &id, const transaction& tx, bool kept_by_block) { - remove_key_images(tx, kept_by_block); + remove_key_images(id, tx, kept_by_block); remove_alias_info(tx); return true; } @@ -757,34 +756,33 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::remove_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (!ki_entry_ptr.get() || *ki_entry_ptr == 0) - { - LOG_ERROR("INTERNAL_ERROR: for tx " << get_transaction_hash(tx) << " key image " << tokey_in.k_image << " not found"); - continue; - } - count = *ki_entry_ptr; - uint64_t count_before = count; - --count; - if (count) - m_db_key_images_set.set(tokey_in.k_image, count); - else - m_db_key_images_set.erase(tokey_in.k_image); - LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + { + const txin_to_key& tokey_in = boost::get(in); + + auto it_map = epee::misc_utils::it_get_or_insert_value_initialized(m_key_images, tokey_in.k_image); + auto& id_set = it_map->second; + size_t count_before = id_set.size(); + auto it_set = id_set.find(tx_id); + if(it_set != id_set.end()) + id_set.erase(it_set); + + size_t count_after = id_set.size(); + if (id_set.size() == 0) + m_key_images.erase(it_map); + + LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << count_before << " -> " << count_after); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_key_images_from_tx_pool(std::unordered_set& key_images) const + bool tx_memory_pool::get_key_images_from_tx_pool(key_image_cache& key_images) const { m_db_transactions.enumerate_items([&](uint64_t i, const crypto::hash& h, const tx_details &tx_entry) @@ -793,7 +791,7 @@ namespace currency { if (in.type() == typeid(txin_to_key)) { - key_images.insert(boost::get(in).k_image); + key_images[boost::get(in).k_image].insert(h); } } return true; @@ -804,9 +802,9 @@ namespace currency //--------------------------------------------------------------------------------- bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im)const { - - auto ptr = m_db_key_images_set.find(key_im); - if (ptr) + CRITICAL_REGION_LOCAL(m_key_images_lock); + auto it = m_key_images.find(key_im); + if (it != m_key_images.end()) return true; return false; } @@ -826,9 +824,9 @@ namespace currency m_db.begin_transaction(); m_db_transactions.clear(); - m_db_key_images_set.clear(); m_db.commit_transaction(); // should m_db_black_tx_list be cleared here? + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- void tx_memory_pool::clear() @@ -836,8 +834,8 @@ namespace currency m_db.begin_transaction(); m_db_transactions.clear(); m_db_black_tx_list.clear(); - m_db_key_images_set.clear(); m_db.commit_transaction(); + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- bool tx_memory_pool::is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const @@ -1193,8 +1191,8 @@ namespace currency res = m_db_transactions.init(TRANSACTION_POOL_CONTAINER_TRANSACTIONS); CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); +// res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); +// CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); res = m_db_black_tx_list.init(TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST); CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); res = m_db_alias_names.init(TRANSACTION_POOL_CONTAINER_ALIAS_NAMES); @@ -1220,7 +1218,7 @@ namespace currency { LOG_PRINT_L1("DB at " << db_folder_path << " is about to be deleted and re-created..."); m_db_transactions.deinit(); - m_db_key_images_set.deinit(); +// m_db_key_images_set.deinit(); m_db_black_tx_list.deinit(); m_db_alias_names.deinit(); m_db_alias_addresses.deinit(); @@ -1252,9 +1250,17 @@ namespace currency }); LOG_PRINT_L2(ss.str()); } + + load_keyimages_cache(); + return true; } - + //--------------------------------------------------------------------------------- + bool tx_memory_pool::load_keyimages_cache() + { + CRITICAL_REGION_LOCAL(m_key_images_lock); + return get_key_images_from_tx_pool(m_key_images); + } //--------------------------------------------------------------------------------- bool tx_memory_pool::deinit() { diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index d8642e8e..b3031566 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -81,6 +81,7 @@ namespace currency epee::math_helper::average db_commit_time; }; + typedef std::unordered_map> key_image_cache; tx_memory_pool(blockchain_storage& bchs, i_currency_protocol* pprotocol); bool add_tx(const transaction &tx, const crypto::hash &id, uint64_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool from_core = false); @@ -99,7 +100,7 @@ namespace currency bool check_tx_multisig_ins_and_outs(const transaction& tx, bool check_against_pool_txs)const; - bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id); + bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk); bool on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id); bool on_finalize_db_transaction(); bool add_transaction_to_black_list(const transaction& tx); @@ -139,10 +140,10 @@ namespace currency bool remove_stuck_transactions(); // made public to be called from coretests private: - bool on_tx_add(const transaction& tx, bool kept_by_block); - bool on_tx_remove(const transaction& tx, bool kept_by_block); - bool insert_key_images(const transaction& tx, bool kept_by_block); - bool remove_key_images(const transaction& tx, bool kept_by_block); + bool on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block); + bool on_tx_remove(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); + bool insert_key_images(const crypto::hash& tx_id, const transaction& tx, bool kept_by_block); + bool remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); bool insert_alias_info(const transaction& tx); bool remove_alias_info(const transaction& tx); @@ -150,12 +151,11 @@ namespace currency void store_db_solo_options_values(); bool is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const; bool validate_alias_info(const transaction& tx, bool is_in_block)const; - bool get_key_images_from_tx_pool(std::unordered_set& key_images)const; - //bool push_alias_info(const transaction& tx); - //bool pop_alias_info(const transaction& tx); + bool get_key_images_from_tx_pool(key_image_cache& key_images) const; bool check_is_taken(const crypto::hash& id) const; void set_taken(const crypto::hash& id); void reset_all_taken(); + bool load_keyimages_cache(); typedef tools::db::cached_key_value_accessor transactions_container; typedef tools::db::cached_key_value_accessor hash_container; @@ -172,7 +172,7 @@ namespace currency transactions_container m_db_transactions; hash_container m_db_black_tx_list; - key_images_container m_db_key_images_set; + //key_images_container m_db_key_images_set; aliases_container m_db_alias_names; address_to_aliases_container m_db_alias_addresses; solo_options_container m_db_solo_options; @@ -189,6 +189,9 @@ namespace currency //in memory containers mutable epee::critical_section m_taken_txs_lock; std::unordered_set m_taken_txs; + + mutable epee::critical_section m_key_images_lock; + key_image_cache m_key_images; mutable epee::critical_section m_remove_stuck_txs_lock;