diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index b0cfd15c..6c2a287d 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -393,7 +393,21 @@ namespace misc_utils auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); return res.first->second; - } + } + + template + typename t_container_type::iterator it_get_or_insert_value_initialized(t_container_type& container, const typename t_container_type::key_type& key) + { + auto it = container.find(key); + if (it != container.end()) + { + return it; + } + + auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); + return res.first; + } + } // namespace misc_utils } // namespace epee diff --git a/contrib/epee/include/serialization/keyvalue_helpers.h b/contrib/epee/include/serialization/keyvalue_helpers.h index 61882816..9d32d024 100644 --- a/contrib/epee/include/serialization/keyvalue_helpers.h +++ b/contrib/epee/include/serialization/keyvalue_helpers.h @@ -51,6 +51,7 @@ namespace epee } }; + //basic helpers for pod-to-hex serialization template std::string transform_t_pod_to_str(const t_pod_type & a) diff --git a/contrib/epee/include/serialization/keyvalue_hexemizer.h b/contrib/epee/include/serialization/keyvalue_hexemizer.h new file mode 100644 index 00000000..47e7e341 --- /dev/null +++ b/contrib/epee/include/serialization/keyvalue_hexemizer.h @@ -0,0 +1,41 @@ +// Copyright (c) 2006-2019, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#pragma once +#include "keyvalue_serialization.h" +namespace epee +{ + + struct hexemizer + { + std::string blob; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(blob) + END_KV_SERIALIZE_MAP() + }; + + +} \ No newline at end of file diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index dffbd614..457c5f37 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -698,6 +698,9 @@ namespace epee #define CRITICAL_REGION_BEGIN1(x) CRITICAL_REGION_BEGIN_VAR(x, critical_region_var1) #define CRITICAL_REGION_END() } + +#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj . op;obj##_lock.unlock();} + #define SHARED_CRITICAL_REGION_LOCAL(x) boost::shared_lock< boost::shared_mutex > critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_LOCAL(x) boost::unique_lock< boost::shared_mutex > critical_region_var(x) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ba358dd6..630a41c0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,6 @@ if(POLICY CMP0043) cmake_policy(SET CMP0043 OLD) endif() - ########### # using shared PCH -- this is unusual case for MSVC... so mystery, such hack, many wow. See also: https://stackoverflow.com/questions/645747/sharing-precompiled-headers-between-projects-in-visual-studio/4170902#4170902 # define USE_PCH to YES for using precomiled headers diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index eaa25474..7ecc4a3a 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -32,6 +32,7 @@ #include "storages/portable_storage_template_helper.h" #include "basic_pow_helpers.h" #include "version.h" +#include "tx_semantic_validation.h" #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "core" @@ -438,7 +439,7 @@ bool blockchain_storage::deinit() return true; } //------------------------------------------------------------------ -bool blockchain_storage::pop_block_from_blockchain() +bool blockchain_storage::pop_block_from_blockchain(transactions_map& onboard_transactions) { CRITICAL_REGION_LOCAL(m_read_lock); @@ -448,7 +449,7 @@ bool blockchain_storage::pop_block_from_blockchain() CHECK_AND_ASSERT_MES(bei_ptr.get(), false, "pop_block_from_blockchain: can't pop from blockchain"); uint64_t fee_total = 0; - bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total); + bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total, onboard_transactions); CHECK_AND_ASSERT_MES(r, false, "Failed to purge_block_data_from_blockchain for block " << get_block_hash(bei_ptr->bl) << " on height " << h); pop_block_from_per_block_increments(bei_ptr->height); @@ -648,7 +649,7 @@ bool blockchain_storage::purge_transaction_keyimages_from_blockchain(const trans return true; } //------------------------------------------------------------------ -bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee) +bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx_) { fee = 0; CRITICAL_REGION_LOCAL(m_read_lock); @@ -656,6 +657,7 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t auto tx_res_ptr = m_db_transactions.find(tx_id); CHECK_AND_ASSERT_MES(tx_res_ptr != m_db_transactions.end(), false, "transaction " << tx_id << " is not found in blockchain index!!"); const transaction& tx = tx_res_ptr->tx; + tx_ = tx; fee = get_tx_fee(tx_res_ptr->tx); purge_transaction_keyimages_from_blockchain(tx, true); @@ -686,10 +688,11 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t bool blockchain_storage::purge_block_data_from_blockchain(const block& b, size_t processed_tx_count) { uint64_t total_fee = 0; - return purge_block_data_from_blockchain(b, processed_tx_count, total_fee); + transactions_map onboard_transactions; + return purge_block_data_from_blockchain(b, processed_tx_count, total_fee, onboard_transactions); } //------------------------------------------------------------------ -bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total) +bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total, transactions_map& onboard_transactions) { CRITICAL_REGION_LOCAL(m_read_lock); fee_total = 0; @@ -698,11 +701,13 @@ bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_ CHECK_AND_ASSERT_MES(processed_tx_count <= bl.tx_hashes.size(), false, "wrong processed_tx_count in purge_block_data_from_blockchain"); for(size_t count = 0; count != processed_tx_count; count++) { - res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee) && res; + transaction tx = AUTO_VAL_INIT(tx); + res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee, tx) && res; fee_total += fee; + onboard_transactions[bl.tx_hashes[(processed_tx_count - 1) - count]] = tx; } - - res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee) && res; + transaction tx = AUTO_VAL_INIT(tx); + res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee, tx) && res; return res; } //------------------------------------------------------------------ @@ -865,20 +870,22 @@ bool blockchain_storage::get_block_by_height(uint64_t h, block &blk) const // invalid.push_back(v.first); // } //------------------------------------------------------------------ -bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) +bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) { CRITICAL_REGION_LOCAL(m_read_lock); //remove failed subchain for(size_t i = m_db_blocks.size()-1; i >=rollback_height; i--) { - bool r = pop_block_from_blockchain(); + transactions_map ot; + bool r = pop_block_from_blockchain(ot); CHECK_AND_ASSERT_MES(r, false, "PANIC!!! failed to remove block while chain switching during the rollback!"); } //return back original chain - BOOST_FOREACH(auto& bl, original_chain) + BOOST_FOREACH(auto& oce, original_chain) { block_verification_context bvc = boost::value_initialized(); - bool r = handle_block_to_main_chain(bl, bvc); + bvc.m_onboard_transactions.swap(oce.onboard_transactions); + bool r = handle_block_to_main_chain(oce.b, bvc); CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC!!! failed to add (again) block while chain switching during the rollback!"); } @@ -941,13 +948,15 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch ); //disconnecting old chain - std::list disconnected_chain; + std::list disconnected_chain; for(size_t i = m_db_blocks.size()-1; i >=split_height; i--) { - block b = m_db_blocks[i]->bl; - bool r = pop_block_from_blockchain(); - CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(b) << " @ " << get_block_height(b) << " on chain switching"); - disconnected_chain.push_front(b); + disconnected_chain.push_front(block_ws_txs()); + block_ws_txs& bwt = disconnected_chain.front(); + bwt.b = m_db_blocks[i]->bl; + bool r = pop_block_from_blockchain(bwt.onboard_transactions); + CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(bwt.b) << " @ " << get_block_height(bwt.b) << " on chain switching"); + CHECK_AND_ASSERT_MES(validate_blockchain_prev_links(), false, "EPIC FAIL!"); } @@ -956,6 +965,7 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch { auto ch_ent = *alt_ch_iter; block_verification_context bvc = boost::value_initialized(); + bvc.m_onboard_transactions = ch_ent->second.onboard_transactions; bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc); if(!r || !bvc.m_added_to_main_chain) { @@ -977,7 +987,8 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch for(auto& old_ch_ent : disconnected_chain) { block_verification_context bvc = boost::value_initialized(); - bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc); + bvc.m_onboard_transactions.swap(old_ch_ent.onboard_transactions); + bool r = handle_alternative_block(old_ch_ent.b, get_block_hash(old_ch_ent.b), bvc); if(!r) { LOG_ERROR("Failed to push ex-main chain blocks to alternative chain "); @@ -1347,11 +1358,11 @@ bool blockchain_storage::create_block_template(const create_block_template_param CRITICAL_REGION_END(); - size_t txs_size; - uint64_t fee; + size_t txs_size = 0; + uint64_t fee = 0; bool block_filled = false; if (pcustom_fill_block_template_func == nullptr) - block_filled = m_tx_pool.fill_block_template(b, pos, median_size, already_generated_coins, txs_size, fee, height); + block_filled = m_tx_pool.fill_block_template(b, pos, median_size, already_generated_coins, txs_size, fee, height, params.explicit_txs); else block_filled = (*pcustom_fill_block_template_func)(b, pos, median_size, already_generated_coins, txs_size, fee, height); @@ -1601,6 +1612,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: alt_block_extended_info abei = AUTO_VAL_INIT(abei); abei.bl = b; + abei.onboard_transactions.swap(bvc.m_onboard_transactions); abei.timestamp = m_core_runtime_config.get_core_time(); abei.height = alt_chain.size() ? it_prev->second.height + 1 : *ptr_main_prev + 1; CHECK_AND_ASSERT_MES_CUSTOM(coinbase_height == abei.height, false, bvc.m_verification_failed = true, "block coinbase height doesn't match with altchain height, declined"); @@ -1715,7 +1727,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: alt_chain.push_back(i_res.first); //check if difficulty bigger then in main chain - bvc.height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0; + bvc.m_height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0; crypto::hash proof = null_hash; std::stringstream ss_pow_pos_info; @@ -1752,7 +1764,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: bvc.m_verification_failed = true; return r; } - bvc.added_to_altchain = true; + bvc.m_added_to_altchain = true; //protect ourself from altchains container flood if (m_alternative_chains.size() > m_core_runtime_config.max_alt_blocks) @@ -4762,6 +4774,18 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise return 0; } //------------------------------------------------------------------ +bool get_tx_from_cache(const crypto::hash& tx_id, transactions_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t& fee) +{ + auto it = tx_cache.find(tx_id); + if (it == tx_cache.end()) + return false; + + tx = it->second; + blob_size = get_object_blobsize(tx); + fee = get_tx_fee(tx); + return true; +} +//------------------------------------------------------------------ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) { TIME_MEASURE_START_PD_MS(block_processing_time_0_ms); @@ -4876,13 +4900,17 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t tx_processed_count = 0; uint64_t fee_summary = 0; uint64_t burned_coins = 0; + std::list block_summary_kimages; for(const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; size_t blob_size = 0; uint64_t fee = 0; - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + + bool taken_from_cache = get_tx_from_cache(tx_id, bvc.m_onboard_transactions, tx, blob_size, fee); + bool taken_from_pool = m_tx_pool.take_tx(tx_id, tx, blob_size, fee); + if(!taken_from_cache && !taken_from_pool) { LOG_PRINT_L0("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); purge_block_data_from_blockchain(bl, tx_processed_count); @@ -4891,6 +4919,15 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } + if (!validate_tx_semantic(tx, blob_size)) + { + LOG_PRINT_L0("Block with id: " << id << " has at least one transaction with wrong semantic, tx_id: " << tx_id); + purge_block_data_from_blockchain(bl, tx_processed_count); + //add_block_as_invalid(bl, id); + bvc.m_verification_failed = true; + return false; + } + append_per_block_increments_for_tx(tx, gindices); //If we under checkpoints, ring signatures should be pruned @@ -4905,9 +4942,12 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt { LOG_PRINT_L0("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs."); currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); - m_tx_pool.add_transaction_to_black_list(tx); - CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + if (taken_from_pool) + { + bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); + m_tx_pool.add_transaction_to_black_list(tx); + CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + } purge_block_data_from_blockchain(bl, tx_processed_count); add_block_as_invalid(bl, id); LOG_PRINT_L0("Block with id " << id << " added as invalid because of wrong inputs in transactions"); @@ -4925,10 +4965,13 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if(!add_transaction_from_block(tx, tx_id, id, current_bc_size, actual_timestamp)) { LOG_PRINT_L0("Block with id: " << id << " failed to add transaction to blockchain storage"); - currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); - m_tx_pool.add_transaction_to_black_list(tx); - CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + if (taken_from_pool) + { + currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); + m_tx_pool.add_transaction_to_black_list(tx); + CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + } purge_block_data_from_blockchain(bl, tx_processed_count); bvc.m_verification_failed = true; return false; @@ -4941,6 +4984,8 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt ++tx_processed_count; if (fee) block_fees.push_back(fee); + + read_keyimages_from_tx(tx, block_summary_kimages); } TIME_MEASURE_FINISH_PD(all_txs_insert_time_5); @@ -4975,8 +5020,6 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if (is_pos_bl) bei.stake_hash = proof_hash; - - ////////////////////////////////////////////////////////////////////////// //old style cumulative difficulty collecting @@ -5114,17 +5157,17 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "/" << etc_stuff_6 << "))"); - on_block_added(bei, id); + on_block_added(bei, id, block_summary_kimages); bvc.m_added_to_main_chain = true; return true; } //------------------------------------------------------------------ -void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id) +void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk) { update_next_comulative_size_limit(); m_timestamps_median_cache.clear(); - m_tx_pool.on_blockchain_inc(bei.height, id); + m_tx_pool.on_blockchain_inc(bei.height, id, bsk); update_targetdata_cache_on_block_added(bei); @@ -5313,7 +5356,8 @@ bool blockchain_storage::truncate_blockchain(uint64_t to_height) uint64_t inital_height = get_current_blockchain_size(); while (get_current_blockchain_size() > to_height) { - pop_block_from_blockchain(); + transactions_map ot; + pop_block_from_blockchain(ot); } CRITICAL_REGION_LOCAL(m_alternative_chains_lock); m_alternative_chains.clear(); @@ -6033,8 +6077,12 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha for (auto tx_id : b.tx_hashes) { std::shared_ptr tx_ptr; - CHECK_AND_ASSERT_MES(get_transaction_from_pool_or_db(tx_id, tx_ptr, split_height), false, "failed to get alt block tx " << tx_id << " with split_height == " << split_height); - transaction& tx = *tx_ptr; + auto it = abei.onboard_transactions.find(tx_id); + if (it == abei.onboard_transactions.end()) + { + CHECK_AND_ASSERT_MES(get_transaction_from_pool_or_db(tx_id, tx_ptr, split_height), false, "failed to get alt block tx " << tx_id << " with split_height == " << split_height); + } + const transaction& tx = it == abei.onboard_transactions.end() ? *tx_ptr : it->second; CHECK_AND_ASSERT_MES(tx.signatures.size() == tx.vin.size(), false, "invalid tx: tx.signatures.size() == " << tx.signatures.size() << ", tx.vin.size() == " << tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) { diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 46af15bd..bc595256 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -156,6 +156,9 @@ namespace currency //date added to alt chain storage uint64_t timestamp; + + //transactions associated with the block + transactions_map onboard_transactions; }; typedef std::unordered_map alt_chain_container; //typedef std::list alt_chain_type; @@ -543,10 +546,10 @@ namespace currency bool switch_to_alternative_blockchain(alt_chain_type& alt_chain); void purge_alt_block_txs_hashs(const block& b); void add_alt_block_txs_hashs(const block& b); - bool pop_block_from_blockchain(); + bool pop_block_from_blockchain(transactions_map& onboard_transactions); bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count); - bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee); - bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee); + bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee, transactions_map& onboard_transactions); + bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx); bool purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check); wide_difficulty_type get_next_difficulty_for_alternative_chain(const alt_chain_type& alt_chain, block_extended_info& bei, bool pos) const; bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); @@ -565,7 +568,7 @@ namespace currency bool get_transaction_from_pool_or_db(const crypto::hash& tx_id, std::shared_ptr& tx_ptr, uint64_t min_allowed_block_height = 0) const; void get_last_n_x_blocks(uint64_t n, bool pos_blocks, std::list>& blocks) const; bool prevalidate_miner_transaction(const block& b, uint64_t height, bool pos)const; - bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); + bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, uint64_t timestamp); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); @@ -578,7 +581,7 @@ namespace currency bool check_block_timestamp(std::vector timestamps, const block& b)const; std::vector get_last_n_blocks_timestamps(size_t n)const; const std::vector& get_txin_etc_options(const txin_v& in)const; - void on_block_added(const block_extended_info& bei, const crypto::hash& id); + void on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk); void on_block_removed(const block_extended_info& bei); void update_targetdata_cache_on_block_added(const block_extended_info& bei); void update_targetdata_cache_on_block_removed(const block_extended_info& bei); diff --git a/src/currency_core/blockchain_storage_basic.h b/src/currency_core/blockchain_storage_basic.h index b232b32e..6d2a1300 100644 --- a/src/currency_core/blockchain_storage_basic.h +++ b/src/currency_core/blockchain_storage_basic.h @@ -6,6 +6,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include #include #include @@ -145,6 +146,12 @@ namespace currency uint64_t height; }; - + typedef std::unordered_map transactions_map; + + struct block_ws_txs + { + block b; + transactions_map onboard_transactions; + }; } \ No newline at end of file diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index 20e81fed..ceb88406 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -19,6 +19,7 @@ using namespace epee; #include "currency_format_utils.h" #include "misc_language.h" #include "string_coding.h" +#include "tx_semantic_validation.h" #define MINIMUM_REQUIRED_FREE_SPACE_BYTES (1024 * 1024 * 100) @@ -185,13 +186,15 @@ namespace currency //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block) { + CHECK_AND_ASSERT_MES(!kept_by_block, false, "Transaction associated with block came throw handle_incoming_tx!(not allowed anymore)"); + tvc = boost::value_initialized(); //want to process all transactions sequentially TIME_MEASURE_START_MS(wait_lock_time); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); TIME_MEASURE_FINISH_MS(wait_lock_time); - if(tx_blob.size() > get_max_tx_size()) + if(tx_blob.size() > CURRENCY_MAX_TRANSACTION_BLOB_SIZE) { LOG_PRINT_L0("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected"); tvc.m_verification_failed = true; @@ -210,19 +213,10 @@ namespace currency TIME_MEASURE_FINISH_MS(parse_tx_time); - TIME_MEASURE_START_MS(check_tx_syntax_time); - if(!check_tx_syntax(tx)) - { - LOG_PRINT_L0("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " syntax, rejected"); - tvc.m_verification_failed = true; - return false; - } - TIME_MEASURE_FINISH_MS(check_tx_syntax_time); - TIME_MEASURE_START_MS(check_tx_semantic_time); - if(!check_tx_semantic(tx, kept_by_block)) + if(!validate_tx_semantic(tx, tx_blob.size())) { - LOG_PRINT_L0("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + LOG_PRINT_L0("WRONG TRANSACTION SEMANTICS, Failed to check tx " << tx_hash << " semantic, rejected"); tvc.m_verification_failed = true; return false; } @@ -243,7 +237,6 @@ namespace currency } LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX]: timing " << wait_lock_time << "/" << parse_tx_time - << "/" << check_tx_syntax_time << "/" << check_tx_semantic_time << "/" << add_new_tx_time); return r; @@ -296,88 +289,9 @@ namespace currency return true; } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_semantic(const transaction& tx, bool kept_by_block) - { - if(!tx.vin.size()) - { - LOG_PRINT_RED_L0("tx with empty inputs, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_inputs_types_supported(tx)) - { - LOG_PRINT_RED_L0("unsupported input types for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_outs_valid(tx)) - { - LOG_PRINT_RED_L0("tx with invalid outputs, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_money_overflow(tx)) - { - LOG_PRINT_RED_L0("tx has money overflow, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - - uint64_t amount_in = 0; - get_inputs_money_amount(tx, amount_in); - uint64_t amount_out = get_outs_money_amount(tx); - - if(amount_in < amount_out) - { - LOG_PRINT_RED_L0("tx with wrong amounts: ins " << amount_in << ", outs " << amount_out << ", rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - - if(!kept_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - CURRENCY_COINBASE_BLOB_RESERVED_SIZE) - { - LOG_PRINT_RED_L0("tx has too big size " << get_object_blobsize(tx) << ", expected no bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CURRENCY_COINBASE_BLOB_RESERVED_SIZE); - return false; - } - - //check if tx use different key images - if(!check_tx_inputs_keyimages_diff(tx)) - { - LOG_PRINT_RED_L0("tx inputs have the same key images"); - return false; - } - - if(!check_tx_extra(tx)) - { - LOG_PRINT_RED_L0("tx has wrong extra, rejected"); - return false; - } - - return true; - } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_extra(const transaction& tx) - { - tx_extra_info ei = AUTO_VAL_INIT(ei); - bool r = parse_and_validate_tx_extra(tx, ei); - if(!r) - return false; - return true; - } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_inputs_keyimages_diff(const transaction& tx) - { - std::unordered_set ki; - BOOST_FOREACH(const auto& in, tx.vin) - { - if (in.type() == typeid(txin_to_key)) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); - if (!ki.insert(tokey_in.k_image).second) - return false; - } - } - return true; - } //----------------------------------------------------------------------------------------------- bool core::add_new_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block) { @@ -419,6 +333,11 @@ namespace currency return m_blockchain_storage.create_block_template(b, adr, stakeholder_address, diffic, height, ex_nonce, pos, pe); } //----------------------------------------------------------------------------------------------- + bool core::get_block_template(const create_block_template_params& params, create_block_template_response& resp) + { + return m_blockchain_storage.create_block_template(params, resp); + } + //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); @@ -589,7 +508,6 @@ namespace currency //----------------------------------------------------------------------------------------------- bool core::handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate) { - bvc = AUTO_VAL_INIT_T(block_verification_context); block b = AUTO_VAL_INIT(b); if (!parse_block(block_blob, b, bvc)) { @@ -627,11 +545,6 @@ namespace currency { return parse_and_validate_tx_from_blob(blob, tx, tx_hash); } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_syntax(const transaction& tx) - { - return true; - } //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions(std::list& txs) { diff --git a/src/currency_core/currency_core.h b/src/currency_core/currency_core.h index 7fa9313a..faeec262 100644 --- a/src/currency_core/currency_core.h +++ b/src/currency_core/currency_core.h @@ -53,7 +53,8 @@ namespace currency //-------------------- i_miner_handler ----------------------- virtual bool handle_block_found(const block& b, block_verification_context* p_verification_result = nullptr); - virtual bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()); + virtual bool get_block_template(const create_block_template_params& params, create_block_template_response& resp); + bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()); miner& get_miner(){ return m_miner; } static void init_options(boost::program_options::options_description& desc); @@ -118,12 +119,6 @@ namespace currency bool add_new_block(const block& b, block_verification_context& bvc); bool load_state_data(); bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob); - bool check_tx_extra(const transaction& tx); - - bool check_tx_syntax(const transaction& tx); - //check correct values, amounts and all lightweight checks not related with database - bool check_tx_semantic(const transaction& tx, bool kept_by_block); - //check if tx already in memory pool or in main blockchain bool is_key_image_spent(const crypto::key_image& key_im); @@ -132,7 +127,6 @@ namespace currency bool update_miner_block_template(); bool handle_command_line(const boost::program_options::variables_map& vm); bool on_update_blocktemplate_interval(); - bool check_tx_inputs_keyimages_diff(const transaction& tx); void notify_blockchain_update_listeners(); diff --git a/src/currency_core/currency_format_utils.cpp b/src/currency_core/currency_format_utils.cpp index 1ab42769..56705fc7 100644 --- a/src/currency_core/currency_format_utils.cpp +++ b/src/currency_core/currency_format_utils.cpp @@ -2434,11 +2434,6 @@ namespace currency return CURRENCY_MAX_BLOCK_SIZE; } //----------------------------------------------------------------------------------------------- - size_t get_max_tx_size() - { - return CURRENCY_MAX_TRANSACTION_BLOB_SIZE; - } - //----------------------------------------------------------------------------------------------- uint64_t get_base_block_reward(bool is_pos, const boost::multiprecision::uint128_t& already_generated_coins, uint64_t height) { if (!height) diff --git a/src/currency_core/currency_format_utils_transactions.cpp b/src/currency_core/currency_format_utils_transactions.cpp index ba08914b..da78894f 100644 --- a/src/currency_core/currency_format_utils_transactions.cpp +++ b/src/currency_core/currency_format_utils_transactions.cpp @@ -193,6 +193,16 @@ namespace currency return get_object_blobsize(t, tx_blob_size); } //--------------------------------------------------------------- + size_t get_objects_blobsize(const std::list& ls) + { + size_t total = 0; + for (const auto& tx : ls) + { + total += get_object_blobsize(tx); + } + return total; + } + //--------------------------------------------------------------- size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size) { size_t tx_blob_size = prefix_blob_size; @@ -246,5 +256,20 @@ namespace currency { return t_serializable_object_to_blob(tx, b_blob); } + //--------------------------------------------------------------- + bool read_keyimages_from_tx(const transaction& tx, std::list& kil) + { + std::unordered_set ki; + BOOST_FOREACH(const auto& in, tx.vin) + { + if (in.type() == typeid(txin_to_key)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); + if (!ki.insert(tokey_in.k_image).second) + return false; + } + } + return true; + } } \ No newline at end of file diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index 4b993394..3c36798b 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -92,7 +92,7 @@ namespace currency inline void set_tx_flags(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } inline void set_tx_expiration_time(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations); - + //----------------------------------------------------------------------------------------------- bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median); uint64_t get_burned_amount(const transaction& tx); @@ -104,7 +104,11 @@ namespace currency bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, uint64_t& blob_size); size_t get_object_blobsize(const transaction& t); + size_t get_objects_blobsize(const std::list& ls); size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size); blobdata tx_to_blob(const transaction& b); bool tx_to_blob(const transaction& b, blobdata& b_blob); + bool read_keyimages_from_tx(const transaction& tx, std::list& kil); + + } diff --git a/src/currency_core/miner.h b/src/currency_core/miner.h index c1bd83f0..69a4bdc7 100644 --- a/src/currency_core/miner.h +++ b/src/currency_core/miner.h @@ -24,6 +24,7 @@ namespace currency struct i_miner_handler { virtual bool handle_block_found(const block& b, block_verification_context* p_verification_result = nullptr) = 0; + virtual bool get_block_template(const create_block_template_params& params, create_block_template_response& resp) = 0; virtual bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()) = 0; protected: ~i_miner_handler(){}; diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index ae8737a8..09eb40b0 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -50,7 +50,7 @@ namespace currency m_db_transactions(m_db), m_db_black_tx_list(m_db), m_db_solo_options(m_db), - m_db_key_images_set(m_db), +// m_db_key_images_set(m_db), m_db_alias_names(m_db), m_db_alias_addresses(m_db), m_db_storage_major_compatibility_version(TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION, m_db_solo_options) @@ -255,7 +255,7 @@ namespace currency td.receive_time = get_core_time(); m_db_transactions.set(id, td); - on_tx_add(tx, kept_by_block); + on_tx_add(id, tx, kept_by_block); TIME_MEASURE_FINISH_PD(update_db_time); return true; @@ -385,7 +385,7 @@ namespace currency blob_size = txe_tr->blob_size; fee = txe_tr->fee; m_db_transactions.erase(id); - on_tx_remove(tx, txe_tr->kept_by_block); + on_tx_remove(id, tx, txe_tr->kept_by_block); set_taken(id); return true; } @@ -472,7 +472,7 @@ namespace currency for (auto& e : to_delete) { m_db_transactions.erase(e.hash); - on_tx_remove(e.tx, e.kept_by_block); + on_tx_remove(e.hash, e.tx, e.kept_by_block); } @@ -613,8 +613,10 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) + bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk) { + + return true; } //--------------------------------------------------------------------------------- @@ -694,36 +696,33 @@ namespace currency return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::insert_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::insert_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (ki_entry_ptr.get()) - count = *ki_entry_ptr; - uint64_t count_before = count; - ++count; - m_db_key_images_set.set(tokey_in.k_image, count); - LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + const txin_to_key& tokey_in = boost::get(in); + auto& id_set = m_key_images[tokey_in.k_image]; + size_t sz_before = id_set.size(); + id_set.insert(tx_id); + LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << sz_before << " -> " << id_set.size()); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_add(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block) { - insert_key_images(tx, kept_by_block); + insert_key_images(tx_id, tx, kept_by_block); insert_alias_info(tx); return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_remove(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_remove(const crypto::hash &id, const transaction& tx, bool kept_by_block) { - remove_key_images(tx, kept_by_block); + remove_key_images(id, tx, kept_by_block); remove_alias_info(tx); return true; } @@ -757,34 +756,33 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::remove_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (!ki_entry_ptr.get() || *ki_entry_ptr == 0) - { - LOG_ERROR("INTERNAL_ERROR: for tx " << get_transaction_hash(tx) << " key image " << tokey_in.k_image << " not found"); - continue; - } - count = *ki_entry_ptr; - uint64_t count_before = count; - --count; - if (count) - m_db_key_images_set.set(tokey_in.k_image, count); - else - m_db_key_images_set.erase(tokey_in.k_image); - LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + { + const txin_to_key& tokey_in = boost::get(in); + + auto it_map = epee::misc_utils::it_get_or_insert_value_initialized(m_key_images, tokey_in.k_image); + auto& id_set = it_map->second; + size_t count_before = id_set.size(); + auto it_set = id_set.find(tx_id); + if(it_set != id_set.end()) + id_set.erase(it_set); + + size_t count_after = id_set.size(); + if (id_set.size() == 0) + m_key_images.erase(it_map); + + LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << count_before << " -> " << count_after); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_key_images_from_tx_pool(std::unordered_set& key_images) const + bool tx_memory_pool::get_key_images_from_tx_pool(key_image_cache& key_images) const { m_db_transactions.enumerate_items([&](uint64_t i, const crypto::hash& h, const tx_details &tx_entry) @@ -793,7 +791,7 @@ namespace currency { if (in.type() == typeid(txin_to_key)) { - key_images.insert(boost::get(in).k_image); + key_images[boost::get(in).k_image].insert(h); } } return true; @@ -804,9 +802,9 @@ namespace currency //--------------------------------------------------------------------------------- bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im)const { - - auto ptr = m_db_key_images_set.find(key_im); - if (ptr) + CRITICAL_REGION_LOCAL(m_key_images_lock); + auto it = m_key_images.find(key_im); + if (it != m_key_images.end()) return true; return false; } @@ -826,9 +824,9 @@ namespace currency m_db.begin_transaction(); m_db_transactions.clear(); - m_db_key_images_set.clear(); m_db.commit_transaction(); // should m_db_black_tx_list be cleared here? + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- void tx_memory_pool::clear() @@ -836,8 +834,8 @@ namespace currency m_db.begin_transaction(); m_db_transactions.clear(); m_db_black_tx_list.clear(); - m_db_key_images_set.clear(); m_db.commit_transaction(); + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- bool tx_memory_pool::is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const @@ -987,7 +985,9 @@ namespace currency const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, - uint64_t height) + uint64_t height, + const std::list& explicit_txs + ) { LOCAL_READONLY_TRANSACTION(); //typedef transactions_container::value_type txv; @@ -998,8 +998,8 @@ namespace currency txs_v.reserve(m_db_transactions.size()); std::vector txs; - //std::transform(m_transactions.begin(), m_transactions.end(), txs.begin(), [](txv &a) -> txv * { return &a; }); - //keep getting it as a values cz db items cache will keep it as unserialied object stored by shared ptrs + + //keep getting it as a values cz db items cache will keep it as unserialised object stored by shared ptrs m_db_transactions.enumerate_keys([&](uint64_t i, crypto::hash& k){txs_v.resize(i + 1); txs_v[i].first = k; return true;}); txs.resize(txs_v.size(), nullptr); @@ -1024,7 +1024,9 @@ namespace currency return a_ > b_; }); - size_t current_size = 0; + + size_t explicit_total_size = get_objects_blobsize(explicit_txs); + size_t current_size = explicit_total_size; uint64_t current_fee = 0; uint64_t best_money; if (!get_block_reward(pos, median_size, CURRENCY_COINBASE_BLOB_RESERVED_SIZE, already_generated_coins, best_money, height)) { @@ -1133,6 +1135,12 @@ namespace currency } } } + // add explicit transactions + for (const auto& tx : explicit_txs) + { + fee += get_tx_fee(tx); + bl.tx_hashes.push_back(get_transaction_hash(tx)); + } return true; } //--------------------------------------------------------------------------------- @@ -1184,8 +1192,8 @@ namespace currency res = m_db_transactions.init(TRANSACTION_POOL_CONTAINER_TRANSACTIONS); CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); +// res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); +// CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); res = m_db_black_tx_list.init(TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST); CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); res = m_db_alias_names.init(TRANSACTION_POOL_CONTAINER_ALIAS_NAMES); @@ -1211,7 +1219,7 @@ namespace currency { LOG_PRINT_L1("DB at " << db_folder_path << " is about to be deleted and re-created..."); m_db_transactions.deinit(); - m_db_key_images_set.deinit(); +// m_db_key_images_set.deinit(); m_db_black_tx_list.deinit(); m_db_alias_names.deinit(); m_db_alias_addresses.deinit(); @@ -1243,9 +1251,17 @@ namespace currency }); LOG_PRINT_L2(ss.str()); } + + load_keyimages_cache(); + return true; } - + //--------------------------------------------------------------------------------- + bool tx_memory_pool::load_keyimages_cache() + { + CRITICAL_REGION_LOCAL(m_key_images_lock); + return get_key_images_from_tx_pool(m_key_images); + } //--------------------------------------------------------------------------------- bool tx_memory_pool::deinit() { diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index 6d9bad9d..818ae50f 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -81,6 +81,7 @@ namespace currency epee::math_helper::average db_commit_time; }; + typedef std::unordered_map> key_image_cache; tx_memory_pool(blockchain_storage& bchs, i_currency_protocol* pprotocol); bool add_tx(const transaction &tx, const crypto::hash &id, uint64_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool from_core = false); @@ -99,7 +100,7 @@ namespace currency bool check_tx_multisig_ins_and_outs(const transaction& tx, bool check_against_pool_txs)const; - bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id); + bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk); bool on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id); bool on_finalize_db_transaction(); bool add_transaction_to_black_list(const transaction& tx); @@ -117,7 +118,7 @@ namespace currency // load/store operations bool init(const std::string& config_folder, const boost::program_options::variables_map& vm); bool deinit(); - bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height); + bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height, const std::list& explicit_txs); bool get_transactions(std::list& txs) const; bool get_all_transactions_details(std::list& txs)const; bool get_all_transactions_brief_details(std::list& txs)const; @@ -139,10 +140,10 @@ namespace currency bool remove_stuck_transactions(); // made public to be called from coretests private: - bool on_tx_add(const transaction& tx, bool kept_by_block); - bool on_tx_remove(const transaction& tx, bool kept_by_block); - bool insert_key_images(const transaction& tx, bool kept_by_block); - bool remove_key_images(const transaction& tx, bool kept_by_block); + bool on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block); + bool on_tx_remove(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); + bool insert_key_images(const crypto::hash& tx_id, const transaction& tx, bool kept_by_block); + bool remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); bool insert_alias_info(const transaction& tx); bool remove_alias_info(const transaction& tx); @@ -150,16 +151,15 @@ namespace currency void store_db_solo_options_values(); bool is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const; bool validate_alias_info(const transaction& tx, bool is_in_block)const; - bool get_key_images_from_tx_pool(std::unordered_set& key_images)const; - //bool push_alias_info(const transaction& tx); - //bool pop_alias_info(const transaction& tx); + bool get_key_images_from_tx_pool(key_image_cache& key_images) const; bool check_is_taken(const crypto::hash& id) const; void set_taken(const crypto::hash& id); void reset_all_taken(); + bool load_keyimages_cache(); typedef tools::db::cached_key_value_accessor transactions_container; typedef tools::db::cached_key_value_accessor hash_container; - typedef tools::db::cached_key_value_accessor key_images_container; + //typedef tools::db::cached_key_value_accessor key_images_container; typedef tools::db::cached_key_value_accessor solo_options_container; typedef tools::db::cached_key_value_accessor aliases_container; typedef tools::db::cached_key_value_accessor address_to_aliases_container; @@ -172,7 +172,7 @@ namespace currency transactions_container m_db_transactions; hash_container m_db_black_tx_list; - key_images_container m_db_key_images_set; + //key_images_container m_db_key_images_set; aliases_container m_db_alias_names; address_to_aliases_container m_db_alias_addresses; solo_options_container m_db_solo_options; @@ -189,6 +189,9 @@ namespace currency //in memory containers mutable epee::critical_section m_taken_txs_lock; std::unordered_set m_taken_txs; + + mutable epee::critical_section m_key_images_lock; + key_image_cache m_key_images; mutable epee::critical_section m_remove_stuck_txs_lock; diff --git a/src/currency_core/tx_semantic_validation.cpp b/src/currency_core/tx_semantic_validation.cpp new file mode 100644 index 00000000..91b700b5 --- /dev/null +++ b/src/currency_core/tx_semantic_validation.cpp @@ -0,0 +1,94 @@ +// Copyright (c) 2018-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + + +#include "tx_semantic_validation.h" +#include "currency_format_utils.h" + +namespace currency +{ + //----------------------------------------------------------------------------------------------- + bool check_tx_extra(const transaction& tx) + { + tx_extra_info ei = AUTO_VAL_INIT(ei); + bool r = parse_and_validate_tx_extra(tx, ei); + if (!r) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool check_tx_inputs_keyimages_diff(const transaction& tx) + { + std::unordered_set ki; + BOOST_FOREACH(const auto& in, tx.vin) + { + if (in.type() == typeid(txin_to_key)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); + if (!ki.insert(tokey_in.k_image).second) + return false; + } + } + return true; + } + //----------------------------------------------------------------------------------------------- + bool validate_tx_semantic(const transaction& tx, size_t tx_block_size) + { + if (!tx.vin.size()) + { + LOG_PRINT_RED_L0("tx with empty inputs, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_inputs_types_supported(tx)) + { + LOG_PRINT_RED_L0("unsupported input types for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_outs_valid(tx)) + { + LOG_PRINT_RED_L0("tx with invalid outputs, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_money_overflow(tx)) + { + LOG_PRINT_RED_L0("tx has money overflow, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + uint64_t amount_in = 0; + get_inputs_money_amount(tx, amount_in); + uint64_t amount_out = get_outs_money_amount(tx); + + if (amount_in < amount_out) + { + LOG_PRINT_RED_L0("tx with wrong amounts: ins " << amount_in << ", outs " << amount_out << ", rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (tx_block_size >= CURRENCY_MAX_TRANSACTION_BLOB_SIZE) + { + LOG_PRINT_RED_L0("tx has too big size " << tx_block_size << ", expected no bigger than " << CURRENCY_BLOCK_GRANTED_FULL_REWARD_ZONE); + return false; + } + + //check if tx use different key images + if (!check_tx_inputs_keyimages_diff(tx)) + { + LOG_PRINT_RED_L0("tx inputs have the same key images"); + return false; + } + + if (!check_tx_extra(tx)) + { + LOG_PRINT_RED_L0("tx has wrong extra, rejected"); + return false; + } + + return true; + } +} \ No newline at end of file diff --git a/src/currency_core/tx_semantic_validation.h b/src/currency_core/tx_semantic_validation.h new file mode 100644 index 00000000..2e6f4790 --- /dev/null +++ b/src/currency_core/tx_semantic_validation.h @@ -0,0 +1,16 @@ +// Copyright (c) 2018-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + + +#include "include_base_utils.h" +#include "currency_format_utils_transactions.h" + + +namespace currency +{ + //check correct values, amounts and all lightweight checks not related with database + bool validate_tx_semantic(const transaction& tx, size_t tx_block_size); +} diff --git a/src/currency_core/verification_context.h b/src/currency_core/verification_context.h index a9f1f646..85533cc5 100644 --- a/src/currency_core/verification_context.h +++ b/src/currency_core/verification_context.h @@ -25,7 +25,11 @@ namespace currency bool m_verification_failed; //bad block, should drop connection bool m_marked_as_orphaned; bool m_already_exists; - bool added_to_altchain; - uint64_t height_difference; + bool m_added_to_altchain; + uint64_t m_height_difference; + //this is work like a first-level cache for transactions while block is getting handled. It lets transactions + //associated with the block to get handled directly to core without being handled by tx_pool(which makes full + //inputs validation, including signatures check) + transactions_map m_onboard_transactions; }; } diff --git a/src/currency_protocol/currency_protocol_handler.inl b/src/currency_protocol/currency_protocol_handler.inl index 2566bd24..db0dc3f6 100644 --- a/src/currency_protocol/currency_protocol_handler.inl +++ b/src/currency_protocol/currency_protocol_handler.inl @@ -305,14 +305,22 @@ namespace currency //now actually process block for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) { - currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, true); - if(tvc.m_verification_failed) + if (tx_blob_it->size() > CURRENCY_MAX_TRANSACTION_BLOB_SIZE) { - LOG_PRINT_L0("Block verification failed: transaction verification failed, dropping connection"); + LOG_ERROR("WRONG TRANSACTION BLOB, too big size " << tx_blob_it->size() << ", rejected"); m_p2p->drop_connection(context); return 1; } + + crypto::hash tx_hash = null_hash; + transaction tx; + if (!parse_and_validate_tx_from_blob(*tx_blob_it, tx, tx_hash)) + { + LOG_ERROR("WRONG TRANSACTION BLOB, Failed to parse, rejected"); + m_p2p->drop_connection(context); + return 1; + } + bvc.m_onboard_transactions[tx_hash] = tx; } m_core.pause_mine(); @@ -327,10 +335,10 @@ namespace currency LOG_PRINT_GREEN("[HANDLE]NOTIFY_NEW_BLOCK EXTRA " << block_id << " bvc.m_added_to_main_chain=" << bvc.m_added_to_main_chain //<< ", prevalidate_result=" << prevalidate_relayed - << ", bvc.added_to_altchain=" << bvc.added_to_altchain + << ", bvc.added_to_altchain=" << bvc.m_added_to_altchain << ", bvc.m_marked_as_orphaned=" << bvc.m_marked_as_orphaned, LOG_LEVEL_2); - if (bvc.m_added_to_main_chain || (bvc.added_to_altchain && bvc.height_difference < 2)) + if (bvc.m_added_to_main_chain || (bvc.m_added_to_altchain && bvc.m_height_difference < 2)) { if (true/*!prevalidate_relayed*/) { @@ -521,27 +529,36 @@ namespace currency { CHECK_STOP_FLAG__DROP_AND_RETURN_IF_SET(1, "Blocks processing interrupted, connection dropped"); + block_verification_context bvc = boost::value_initialized(); //process transactions TIME_MEASURE_START(transactions_process_time); for (const auto& tx_blob : block_entry.txs) { CHECK_STOP_FLAG__DROP_AND_RETURN_IF_SET(1, "Block txs processing interrupted, connection dropped"); - - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(tx_blob, tvc, true); - if(tvc.m_verification_failed) + crypto::hash tx_id = null_hash; + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_id)) { - LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " + LOG_ERROR_CCONTEXT("failed to parse tx: " << string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); m_p2p->drop_connection(context); return 1; } + bvc.m_onboard_transactions[tx_id] = tx; +// tx_verification_context tvc = AUTO_VAL_INIT(tvc); +// m_core.handle_incoming_tx(tx_blob, tvc, true); +// if(tvc.m_verification_failed) +// { +// LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " +// << string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); +// m_p2p->drop_connection(context); +// return 1; +// } } TIME_MEASURE_FINISH(transactions_process_time); //process block TIME_MEASURE_START(block_process_time); - block_verification_context bvc = boost::value_initialized(); m_core.handle_incoming_block(block_entry.block, bvc, false); if (count > 2 && bvc.m_already_exists) diff --git a/src/gui/qt-daemon/application/core_fast_rpc_proxy.h b/src/gui/qt-daemon/application/core_fast_rpc_proxy.h index ca189240..c26c1fb2 100644 --- a/src/gui/qt-daemon/application/core_fast_rpc_proxy.h +++ b/src/gui/qt-daemon/application/core_fast_rpc_proxy.h @@ -109,6 +109,11 @@ namespace tools return m_rpc.on_submitblock(req, rsp, m_err_stub, m_cntxt_stub); } //------------------------------------------------------------------------------------------------------------------------------ + bool call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) override + { + return m_rpc.on_submitblock2(req, rsp, m_err_stub, m_cntxt_stub); + } + //------------------------------------------------------------------------------------------------------------------------------ bool call_COMMAND_RPC_GET_POS_MINING_DETAILS(const currency::COMMAND_RPC_GET_POS_MINING_DETAILS::request& req, currency::COMMAND_RPC_GET_POS_MINING_DETAILS::response& rsp) override { return m_rpc.on_get_pos_mining_details(req, rsp, m_cntxt_stub); diff --git a/src/gui/qt-daemon/application/view_iface.h b/src/gui/qt-daemon/application/view_iface.h index 0dbfd962..85c1b08e 100644 --- a/src/gui/qt-daemon/application/view_iface.h +++ b/src/gui/qt-daemon/application/view_iface.h @@ -383,12 +383,12 @@ public: struct get_recent_transfers_request { uint64_t wallet_id; - uint64_t offest; + uint64_t offset; uint64_t count; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(wallet_id) - KV_SERIALIZE(offest) + KV_SERIALIZE(offset) KV_SERIALIZE(count) END_KV_SERIALIZE_MAP() }; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 19c2bf64..34d930de 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -277,14 +277,14 @@ namespace nodetool #ifndef TESTNET //TODO: //ADD_HARDCODED_SEED_NODE(std::string("0.0.0.0:") + std::to_string(P2P_DEFAULT_PORT)); - ADD_HARDCODED_SEED_NODE("95.217.43.225:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("94.130.137.230:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("95.217.42.247:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("94.130.160.115:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("195.201.107.230:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("95.217.46.49:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("159.69.76.144:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("144.76.183.143:", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.43.225", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.137.230", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.42.247", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.160.115", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("195.201.107.230", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.46.49", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("159.69.76.144", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("144.76.183.143", P2P_DEFAULT_PORT); #else //TODO: ADD_HARDCODED_SEED_NODE("95.217.43.225", P2P_DEFAULT_PORT); diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index e8ca9317..28706e83 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -793,28 +793,45 @@ namespace currency return false; } - block b = AUTO_VAL_INIT(b); - wide_difficulty_type dt = 0; - currency::pos_entry pe = AUTO_VAL_INIT(pe); - pe.amount = req.pos_amount; - pe.index = req.pos_index; - pe.stake_unlock_time = req.stake_unlock_time; - //pe.keyimage key image will be set in the wallet - //pe.wallet_index is not included in serialization map, TODO: refactoring here - if (!m_core.get_block_template(b, miner_address, stakeholder_address, dt, res.height, req.extra_text, req.pos_block, pe)) + create_block_template_params params = AUTO_VAL_INIT(params); + params.miner_address = miner_address; + params.stakeholder_address = stakeholder_address; + params.ex_nonce = req.extra_text; + params.pos = req.pos_block; + params.pe.amount = req.pos_amount; + params.pe.index = req.pos_index; + params.pe.stake_unlock_time = req.stake_unlock_time; + //params.pe.keyimage key image will be set in the wallet + //params.pe.wallet_index is not included in serialization map, TODO: refactoring here + params.pcustom_fill_block_template_func = nullptr; + if (req.explicit_transaction.size()) + { + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(req.explicit_transaction, tx)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Wrong parameters: explicit_transaction is invalid"; + LOG_ERROR("Failed to parse explicit_transaction blob"); + return false; + } + params.explicit_txs.push_back(tx); + } + + create_block_template_response resp = AUTO_VAL_INIT(resp); + if (!m_core.get_block_template(params, resp)) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; LOG_ERROR("Failed to create block template"); return false; } - res.difficulty = dt.convert_to(); - blobdata block_blob = t_serializable_object_to_blob(b); + res.difficulty = resp.diffic.convert_to(); + blobdata block_blob = t_serializable_object_to_blob(resp.b); res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); - res.prev_hash = string_tools::pod_to_hex(b.prev_id); - + res.prev_hash = string_tools::pod_to_hex(resp.b.prev_id); + res.height = resp.height; //calculate epoch seed res.seed = currency::ethash_epoch_to_seed(currency::ethash_height_to_epoch(res.height)); @@ -851,7 +868,64 @@ namespace currency block_verification_context bvc = AUTO_VAL_INIT(bvc); if(!m_core.handle_block_found(b, &bvc)) { - if (bvc.added_to_altchain) + if (bvc.m_added_to_altchain) + { + error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_ADDED_AS_ALTERNATIVE; + error_resp.message = "Block added as alternative"; + return false; + } + error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_NOT_ACCEPTED; + error_resp.message = "Block not accepted"; + return false; + } + //@#@ + //temporary double check timestamp + if (time(NULL) - get_actual_timestamp(b) > 5) + { + LOG_PRINT_RED_L0("Found block (" << get_block_hash(b) << ") timestamp (" << get_actual_timestamp(b) + << ") is suspiciously less (" << time(NULL) - get_actual_timestamp(b) << ") then curren time( " << time(NULL) << ")"); + //mark node to make it easier to find it via scanner + m_core.get_blockchain_storage().get_performnce_data().epic_failure_happend = true; + } + // + + + res.status = "OK"; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_submitblock2(const COMMAND_RPC_SUBMITBLOCK2::request& req, COMMAND_RPC_SUBMITBLOCK2::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + { + CHECK_CORE_READY(); + + + block b = AUTO_VAL_INIT(b); + if (!parse_and_validate_block_from_blob(req.b, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + + block_verification_context bvc = AUTO_VAL_INIT(bvc); + for (const auto& txblob : req.explicit_txs) + { + + crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash); + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(txblob.blob, tx, tx_hash)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong explicit tx blob"; + return false; + } + bvc.m_onboard_transactions[tx_hash] = tx; + } + + + if (!m_core.handle_block_found(b, &bvc)) + { + if (bvc.m_added_to_altchain) { error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_ADDED_AS_ALTERNATIVE; error_resp.message = "Block added as alternative"; diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 287d16d2..03851812 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -63,6 +63,7 @@ namespace currency bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); + bool on_submitblock2(const COMMAND_RPC_SUBMITBLOCK2::request& req, COMMAND_RPC_SUBMITBLOCK2::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); @@ -125,6 +126,7 @@ namespace currency MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) + MAP_JON_RPC_WE("submitblock2", on_submitblock2, COMMAND_RPC_SUBMITBLOCK2) MAP_JON_RPC_WE("getlastblockheader", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("getblockheaderbyhash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) MAP_JON_RPC_WE("getblockheaderbyheight", on_get_block_header_by_height, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 4ff21dec..c0cb4990 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -5,6 +5,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include "serialization/keyvalue_hexemizer.h" #include "currency_protocol/currency_protocol_defs.h" #include "currency_core/currency_basic.h" #include "currency_core/difficulty.h" @@ -770,7 +771,7 @@ namespace currency { struct request { - //uint64_t reserve_size; //max 255 bytes + blobdata explicit_transaction; std::string extra_text; std::string wallet_address; std::string stakeholder_address; @@ -780,6 +781,7 @@ namespace currency uint64_t stake_unlock_time; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(explicit_transaction) KV_SERIALIZE(extra_text) KV_SERIALIZE(wallet_address) KV_SERIALIZE(stakeholder_address); @@ -824,6 +826,29 @@ namespace currency }; }; + struct COMMAND_RPC_SUBMITBLOCK2 + { + struct request + { + std::string b; //hex encoded block blob + std::list explicit_txs; //hex encoded tx blobs + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(b) + KV_SERIALIZE(explicit_txs) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + struct block_header_response { uint8_t major_version; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 6b7150e6..1cc11544 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -207,6 +207,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("fix_collisions", boost::bind(&simple_wallet::fix_collisions, this, _1), "Rescan transfers for key image collisions"); m_cmd_binder.set_handler("scan_transfers_for_id", boost::bind(&simple_wallet::scan_transfers_for_id, this, _1), "Rescan transfers for tx_id"); m_cmd_binder.set_handler("scan_transfers_for_ki", boost::bind(&simple_wallet::scan_transfers_for_ki, this, _1), "Rescan transfers for key image"); + m_cmd_binder.set_handler("print_utxo_distribution", boost::bind(&simple_wallet::print_utxo_distribution, this, _1), "Prints utxo distribution"); m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), "Show current wallet public address"); m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::integrated_address, this, _1), "integrated_address [| &args) print_td_list(td); return true; } +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::print_utxo_distribution(const std::vector &args) +{ + std::map distribution; + m_wallet->get_utxo_distribution(distribution); + for (auto& e : distribution) + { + message_writer() << std::left << setw(25) << print_money(e.first) << "|" << e.second; + } + return true; +} + //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_transfer_info(const std::vector &args) { diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 34ba13b2..91fccfd3 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -67,6 +67,7 @@ namespace currency bool fix_collisions(const std::vector &args ); bool scan_transfers_for_id(const std::vector &args); bool scan_transfers_for_ki(const std::vector &args); + bool print_utxo_distribution(const std::vector &args); bool show_blockchain_height(const std::vector &args); bool show_wallet_bcheight(const std::vector &args); bool transfer(const std::vector &args); diff --git a/src/stratum/stratum_server.cpp b/src/stratum/stratum_server.cpp index 586d5e09..2b454c47 100644 --- a/src/stratum/stratum_server.cpp +++ b/src/stratum/stratum_server.cpp @@ -532,7 +532,7 @@ namespace r = m_p_core->handle_block_found(m_block_template, &bvc, false); if (r) { - if (!bvc.m_verification_failed && !bvc.added_to_altchain && bvc.m_added_to_main_chain && !bvc.m_already_exists && !bvc.m_marked_as_orphaned) + if (!bvc.m_verification_failed && !bvc.m_added_to_altchain && bvc.m_added_to_main_chain && !bvc.m_already_exists && !bvc.m_marked_as_orphaned) { LP_CC_WORKER_GREEN(p_ph->get_context(), "found block " << block_hash << " at height " << height << " was successfully added to the blockchain, difficulty " << m_network_difficulty, LOG_LEVEL_0); r = update_block_template(); @@ -544,7 +544,7 @@ namespace { LP_CC_WORKER_RED(p_ph->get_context(), "block " << block_hash << " at height " << height << " was NOT added to the blockchain:" << ENDL << " verification_failed: " << bvc.m_verification_failed << ENDL << - " added_to_altchain: " << bvc.added_to_altchain << ENDL << + " added_to_altchain: " << bvc.m_added_to_altchain << ENDL << " added_to_main_chain: " << bvc.m_added_to_main_chain << ENDL << " already_exists: " << bvc.m_already_exists << ENDL << " marked_as_orphaned: " << bvc.m_marked_as_orphaned, LOG_LEVEL_0); diff --git a/src/wallet/core_default_rpc_proxy.cpp b/src/wallet/core_default_rpc_proxy.cpp index 6140270d..ae6123bd 100644 --- a/src/wallet/core_default_rpc_proxy.cpp +++ b/src/wallet/core_default_rpc_proxy.cpp @@ -116,6 +116,11 @@ namespace tools return invoke_http_json_rpc_update_is_disconnect("submitblock", req, rsp); } //------------------------------------------------------------------------------------------------------------------------------ + bool default_http_core_proxy::call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) + { + return invoke_http_json_rpc_update_is_disconnect("submitblock2", req, rsp); + } + //------------------------------------------------------------------------------------------------------------------------------ bool default_http_core_proxy::check_connection() { CRITICAL_REGION_LOCAL(m_lock); diff --git a/src/wallet/core_default_rpc_proxy.h b/src/wallet/core_default_rpc_proxy.h index ec43ab5f..b514adbc 100644 --- a/src/wallet/core_default_rpc_proxy.h +++ b/src/wallet/core_default_rpc_proxy.h @@ -39,6 +39,7 @@ namespace tools bool call_COMMAND_RPC_SCAN_POS(const currency::COMMAND_RPC_SCAN_POS::request& req, currency::COMMAND_RPC_SCAN_POS::response& rsp) override; bool call_COMMAND_RPC_GETBLOCKTEMPLATE(const currency::COMMAND_RPC_GETBLOCKTEMPLATE::request& req, currency::COMMAND_RPC_GETBLOCKTEMPLATE::response& rsp) override; bool call_COMMAND_RPC_SUBMITBLOCK(const currency::COMMAND_RPC_SUBMITBLOCK::request& req, currency::COMMAND_RPC_SUBMITBLOCK::response& rsp) override; + bool call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) override; bool call_COMMAND_RPC_GET_POS_MINING_DETAILS(const currency::COMMAND_RPC_GET_POS_MINING_DETAILS::request& req, currency::COMMAND_RPC_GET_POS_MINING_DETAILS::response& rsp) override; bool call_COMMAND_RPC_GET_BLOCKS_DETAILS(const currency::COMMAND_RPC_GET_BLOCKS_DETAILS::request& req, currency::COMMAND_RPC_GET_BLOCKS_DETAILS::response& res) override; bool call_COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN(const currency::COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN::request& req, currency::COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN::response& res) override; diff --git a/src/wallet/core_rpc_proxy.h b/src/wallet/core_rpc_proxy.h index 6688e75a..828bd509 100644 --- a/src/wallet/core_rpc_proxy.h +++ b/src/wallet/core_rpc_proxy.h @@ -36,6 +36,7 @@ namespace tools virtual bool call_COMMAND_RPC_SCAN_POS(const currency::COMMAND_RPC_SCAN_POS::request& req, currency::COMMAND_RPC_SCAN_POS::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GETBLOCKTEMPLATE(const currency::COMMAND_RPC_GETBLOCKTEMPLATE::request& req, currency::COMMAND_RPC_GETBLOCKTEMPLATE::response& rsp){ return false; } virtual bool call_COMMAND_RPC_SUBMITBLOCK(const currency::COMMAND_RPC_SUBMITBLOCK::request& req, currency::COMMAND_RPC_SUBMITBLOCK::response& rsp){ return false; } + virtual bool call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) { return false; } virtual bool call_COMMAND_RPC_GET_POS_MINING_DETAILS(const currency::COMMAND_RPC_GET_POS_MINING_DETAILS::request& req, currency::COMMAND_RPC_GET_POS_MINING_DETAILS::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_BLOCKS_DETAILS(const currency::COMMAND_RPC_GET_BLOCKS_DETAILS::request& req, currency::COMMAND_RPC_GET_BLOCKS_DETAILS::response& res){ return false; } virtual bool call_COMMAND_RPC_GET_OFFERS_EX(const currency::COMMAND_RPC_GET_OFFERS_EX::request& req, currency::COMMAND_RPC_GET_OFFERS_EX::response& res){ return false; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index a69878e8..68779858 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -147,6 +147,11 @@ bool wallet2::set_core_proxy(const std::shared_ptr& proxy) return true; } //---------------------------------------------------------------------------------------------------- +void wallet2::set_pos_mint_packing_size(uint64_t new_size) +{ + m_pos_mint_packing_size = new_size; +} +//---------------------------------------------------------------------------------------------------- std::shared_ptr wallet2::get_core_proxy() { return m_core_proxy; @@ -2315,6 +2320,34 @@ void wallet2::get_transfers(wallet2::transfer_container& incoming_transfers) con incoming_transfers = m_transfers; } //---------------------------------------------------------------------------------------------------- +bool wallet2::generate_packing_transaction_if_needed(currency::transaction& tx, uint64_t fake_outputs_number) +{ + prepare_free_transfers_cache(0); + auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); + if (it == m_found_free_amounts.end() || it->second.size() < m_pos_mint_packing_size) + return false; + + //let's check if we have at least WALLET_POS_MINT_PACKING_SIZE transactions which is ready to go + size_t count = 0; + for (auto it_ind = it->second.begin(); it_ind != it->second.end() && count < m_pos_mint_packing_size; it_ind++) + { + if (is_transfer_ready_to_go(m_transfers[*it_ind], fake_outputs_number)) + ++count; + } + if (count < m_pos_mint_packing_size) + return false; + construct_tx_param ctp = get_default_construct_tx_param(); + currency::tx_destination_entry de = AUTO_VAL_INIT(de); + de.addr.push_back(m_account.get_public_address()); + de.amount = m_pos_mint_packing_size*CURRENCY_BLOCK_REWARD; + ctp.dsts.push_back(de); + ctp.perform_packing = true; + + transfer(ctp, tx, false, nullptr); + + return true; +} +//---------------------------------------------------------------------------------------------------- std::string wallet2::get_transfers_str(bool include_spent /*= true*/, bool include_unspent /*= true*/) const { static const char* header = "index amount g_index flags block tx out# key image"; @@ -2433,6 +2466,16 @@ void wallet2::sign_transfer_files(const std::string& tx_sources_file, const std: THROW_IF_FALSE_WALLET_CMN_ERR_EX(r, "failed to store signed tx to file " << signed_tx_file); } //---------------------------------------------------------------------------------------------------- +bool wallet2::get_utxo_distribution(std::map& distribution) +{ + prepare_free_transfers_cache(0); + for (auto ent : m_found_free_amounts) + { + distribution[ent.first] = ent.second.size(); + } + return true; +} +//---------------------------------------------------------------------------------------------------- void wallet2::submit_transfer(const std::string& signed_tx_blob, currency::transaction& tx) { // decrypt sources @@ -2783,6 +2826,13 @@ bool wallet2::build_minted_block(const currency::COMMAND_RPC_SCAN_POS::request& tmpl_req.pos_index = req.pos_entries[rsp.index].index; tmpl_req.extra_text = m_miner_text_info; tmpl_req.stake_unlock_time = req.pos_entries[rsp.index].stake_unlock_time; + //generate packing tx + transaction pack_tx = AUTO_VAL_INIT(pack_tx); + if (generate_packing_transaction_if_needed(pack_tx, 0)) + { + tx_to_blob(pack_tx, tmpl_req.explicit_transaction); + WLT_LOG_GREEN("Pacling inputs: " << pack_tx.vin.size() << " inputs consolidated", LOG_LEVEL_0); + } m_core_proxy->call_COMMAND_RPC_GETBLOCKTEMPLATE(tmpl_req, tmpl_rsp); WLT_CHECK_AND_ASSERT_MES(tmpl_rsp.status == CORE_RPC_STATUS_OK, false, "Failed to create block template after kernel hash found!"); @@ -2826,10 +2876,13 @@ bool wallet2::build_minted_block(const currency::COMMAND_RPC_SCAN_POS::request& WLT_LOG_GREEN("Block constructed <" << get_block_hash(b) << ">, sending to core...", LOG_LEVEL_0); - currency::COMMAND_RPC_SUBMITBLOCK::request subm_req = AUTO_VAL_INIT(subm_req); - currency::COMMAND_RPC_SUBMITBLOCK::response subm_rsp = AUTO_VAL_INIT(subm_rsp); - subm_req.push_back(epee::string_tools::buff_to_hex_nodelimer(t_serializable_object_to_blob(b))); - m_core_proxy->call_COMMAND_RPC_SUBMITBLOCK(subm_req, subm_rsp); + currency::COMMAND_RPC_SUBMITBLOCK2::request subm_req = AUTO_VAL_INIT(subm_req); + currency::COMMAND_RPC_SUBMITBLOCK2::response subm_rsp = AUTO_VAL_INIT(subm_rsp); + subm_req.b = t_serializable_object_to_blob(b); + if (tmpl_req.explicit_transaction.size()) + subm_req.explicit_txs.push_back(hexemizer{ tmpl_req.explicit_transaction }); + + m_core_proxy->call_COMMAND_RPC_SUBMITBLOCK2(subm_req, subm_rsp); if (subm_rsp.status != CORE_RPC_STATUS_OK) { WLT_LOG_ERROR("Constructed block is not accepted by core, status: " << subm_rsp.status); @@ -3417,11 +3470,41 @@ void wallet2::send_escrow_proposal(const bc_services::contract_private_details& print_tx_sent_message(tx, "(from multisig)", fee); } //---------------------------------------------------------------------------------------------------- +bool wallet2::prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) +{ + prepare_free_transfers_cache(fake_outputs_count); + auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); + if (it == m_found_free_amounts.end() || it->second.size() < m_pos_mint_packing_size) + return false; + + for (auto set_it = it->second.begin(); set_it != it->second.end() && selected_indicies.size() <= m_pos_mint_packing_size; ) + { + if (is_transfer_ready_to_go(m_transfers[*set_it], fake_outputs_count)) + { + found_money += it->first; + selected_indicies.push_back(*set_it); + WLT_LOG_L2("Selected index: " << *set_it << ", transfer_details: " << ENDL << epee::serialization::store_t_to_json(m_transfers[*set_it])); + + it->second.erase(set_it++); + } + else + set_it++; + } + if (!it->second.size()) + m_found_free_amounts.erase(it); + + return prepare_tx_sources(fake_outputs_count, sources, selected_indicies, found_money); +} +//---------------------------------------------------------------------------------------------------- bool wallet2::prepare_tx_sources(uint64_t needed_money, size_t fake_outputs_count, uint64_t dust_threshold, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) { found_money = select_transfers(needed_money, fake_outputs_count, dust_threshold, selected_indicies); THROW_IF_FALSE_WALLET_EX_MES(found_money >= needed_money, error::not_enough_money, "wallet_dump: " << ENDL << dump_trunsfers(false), found_money, needed_money, 0); - + return prepare_tx_sources(fake_outputs_count, sources, selected_indicies, found_money); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::prepare_tx_sources(size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) +{ typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; typedef currency::tx_source_entry::output_entry tx_output_entry; @@ -4060,7 +4143,9 @@ void wallet2::prepare_transaction(const construct_tx_param& ctp, finalize_tx_par uint64_t found_money = 0; TIME_MEASURE_START_MS(prepare_tx_sources_time); - if (ctp.multisig_id == currency::null_hash) + if (ctp.perform_packing) + prepare_tx_sources_for_packing(WALLET_DEFAULT_POS_MINT_PACKING_SIZE, 0, ftp.sources, ftp.selected_transfers, found_money); + else if (ctp.multisig_id == currency::null_hash) prepare_tx_sources(needed_money, ctp.fake_outputs_count, ctp.dust_policy.dust_threshold, ftp.sources, ftp.selected_transfers, found_money); else prepare_tx_sources(ctp.multisig_id, ftp.sources, found_money); @@ -4174,7 +4259,7 @@ void wallet2::transfer(const std::vector& dsts, bool send_to_network, std::string* p_signed_tx_blob_str) { - TIME_MEASURE_START(precalculation_time); + //TIME_MEASURE_START(precalculation_time); construct_tx_param ctp = AUTO_VAL_INIT(ctp); ctp.attachments = attachments; ctp.crypt_address = currency::get_crypt_address_from_destinations(m_account.get_keys(), dsts); @@ -4190,8 +4275,33 @@ void wallet2::transfer(const std::vector& dsts, ctp.split_strategy_id = destination_split_strategy_id; ctp.tx_outs_attr = tx_outs_attr; ctp.unlock_time = unlock_time; - TIME_MEASURE_FINISH(precalculation_time); + //TIME_MEASURE_FINISH(precalculation_time); + transfer(ctp, tx, send_to_network, p_signed_tx_blob_str); +} +//---------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------- +construct_tx_param wallet2::get_default_construct_tx_param_inital() +{ + construct_tx_param ctp = AUTO_VAL_INIT(ctp); + ctp.fee = m_core_runtime_config.tx_default_fee; + ctp.dust_policy = tools::tx_dust_policy(DEFAULT_DUST_THRESHOLD); + ctp.split_strategy_id = tools::detail::ssi_digit; + ctp.tx_outs_attr = CURRENCY_TO_KEY_OUT_RELAXED; + ctp.shuffle = 0; + return ctp; +} +const construct_tx_param& wallet2::get_default_construct_tx_param() +{ + static construct_tx_param ctp = get_default_construct_tx_param_inital(); + return ctp; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::transfer(const construct_tx_param& ctp, + currency::transaction &tx, + bool send_to_network, + std::string* p_signed_tx_blob_str) +{ TIME_MEASURE_START(prepare_transaction_time); finalize_tx_param ftp = AUTO_VAL_INIT(ftp); prepare_transaction(ctp, ftp); @@ -4216,7 +4326,7 @@ void wallet2::transfer(const std::vector& dsts, TIME_MEASURE_FINISH(mark_transfers_as_spent_time); WLT_LOG_GREEN("[wallet::transfer]" - << " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) + //<< " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) << ", prepare_transaction_time: " << print_fixed_decimal_point(prepare_transaction_time, 3) << ", store_unsigned_tx_time: " << print_fixed_decimal_point(store_unsigned_tx_time, 3) << ", mark_transfers_as_spent_time: " << print_fixed_decimal_point(mark_transfers_as_spent_time, 3) @@ -4244,13 +4354,13 @@ void wallet2::transfer(const std::vector& dsts, WLT_LOG_GREEN("[wallet::transfer]" - << " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) + //<< " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) << ", prepare_transaction_time: " << print_fixed_decimal_point(prepare_transaction_time, 3) << ", finalize_transaction_time: " << print_fixed_decimal_point(finalize_transaction_time, 3) << ", mark_transfers_as_spent_time: " << print_fixed_decimal_point(mark_transfers_as_spent_time, 3) , LOG_LEVEL_0); - print_tx_sent_message(tx, std::string() + "(transfer)", fee); + print_tx_sent_message(tx, std::string() + "(transfer)", ctp.fee); } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 26368f0a..e214f5a7 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -42,6 +42,8 @@ #define WALLET_DEFAULT_TX_SPENDABLE_AGE 10 #define WALLET_POS_MINT_CHECK_HEIGHT_INTERVAL 1 +#define WALLET_DEFAULT_POS_MINT_PACKING_SIZE 100 + #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "wallet" ENABLE_CHANNEL_BY_DEFAULT("wallet"); @@ -256,6 +258,7 @@ namespace tools currency::account_public_address crypt_address; uint8_t tx_outs_attr; bool shuffle; + bool perform_packing; }; struct finalize_tx_param @@ -313,7 +316,8 @@ namespace tools m_last_sync_percent(0), m_do_rise_transfer(false), m_watch_only(false), - m_last_pow_block_h(0) + m_last_pow_block_h(0), + m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE) {}; public: wallet2() : m_stop(false), @@ -326,7 +330,8 @@ namespace tools m_do_rise_transfer(false), m_log_prefix("???"), m_watch_only(false), - m_last_pow_block_h(0) + m_last_pow_block_h(0), + m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE) { m_core_runtime_config = currency::get_default_core_runtime_config(); }; @@ -491,6 +496,7 @@ namespace tools bool set_core_proxy(const std::shared_ptr& proxy); + void set_pos_mint_packing_size(uint64_t new_size); std::shared_ptr get_core_proxy(); uint64_t balance() const; uint64_t balance(uint64_t& unloked, uint64_t& awaiting_in, uint64_t& awaiting_out, uint64_t& mined) const; @@ -538,6 +544,11 @@ namespace tools const std::vector& attachments, currency::transaction& tx); + void transfer(const construct_tx_param& ctp, + currency::transaction &tx, + bool send_to_network, + std::string* p_signed_tx_blob_str); + template void transfer_from_contract( const std::list& owner_keys, @@ -728,6 +739,7 @@ namespace tools std::string get_log_prefix() const { return m_log_prefix; } static uint64_t get_max_unlock_time_from_receive_indices(const currency::transaction& tx, const money_transfer2_details& td); + bool get_utxo_distribution(std::map& distribution); private: void add_transfers_to_expiration_list(const std::vector& selected_transfers, uint64_t expiration, uint64_t change_amount, const crypto::hash& related_tx_id); @@ -783,7 +795,9 @@ private: void process_genesis_if_needed(const currency::block& genesis); bool build_escrow_proposal(bc_services::contract_private_details& ecrow_details, uint64_t fee, uint64_t unlock_time, currency::tx_service_attachment& att, std::vector& selected_indicies); bool prepare_tx_sources(uint64_t needed_money, size_t fake_outputs_count, uint64_t dust_threshold, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); + bool prepare_tx_sources(size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); bool prepare_tx_sources(crypto::hash multisig_id, std::vector& sources, uint64_t& found_money); + bool prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); uint64_t get_needed_money(uint64_t fee, const std::vector& dsts); void prepare_tx_destinations(uint64_t needed_money, uint64_t found_money, @@ -799,7 +813,9 @@ private: void change_contract_state(wallet_public::escrow_contract_details_basic& contract, uint32_t new_state, const crypto::hash& contract_id, const wallet_public::wallet_transfer_info& wti) const; void change_contract_state(wallet_public::escrow_contract_details_basic& contract, uint32_t new_state, const crypto::hash& contract_id, const std::string& reason = "internal intention") const; - + + construct_tx_param get_default_construct_tx_param_inital(); + const construct_tx_param& get_default_construct_tx_param(); uint64_t get_tx_expiration_median() const; @@ -841,7 +857,7 @@ private: void exception_handler() const; uint64_t get_minimum_allowed_fee_for_contract(const crypto::hash& ms_id); void check_for_free_space_and_throw_if_it_lacks(const std::wstring& path, uint64_t exact_size_needed_if_known = UINT64_MAX); - + bool generate_packing_transaction_if_needed(currency::transaction& tx, uint64_t fake_outputs_number); currency::account_base m_account; @@ -854,6 +870,7 @@ private: std::atomic m_local_bc_height; //temporary workaround std::atomic m_last_bc_timestamp; bool m_do_rise_transfer; + uint64_t m_pos_mint_packing_size; transfer_container m_transfers; multisig_transfer_container m_multisig_transfers; diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 761cc26a..27bb39f9 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -250,7 +250,7 @@ bool gen_and_play_intermitted_by_blockchain_saveload(const char* const genclass_ #define GENERATE_AND_PLAY(genclass) \ - if(!postponed_tests.count(#genclass) && (run_single_test.empty() || run_single_test == #genclass)) \ + if(!postponed_tests.count(#genclass) && (run_single_test.empty() || std::string::npos != std::string(#genclass).find(run_single_test))) \ { \ TIME_MEASURE_START_MS(t); \ ++tests_count; \ @@ -359,6 +359,7 @@ private: size_t m_ev_index; test_core_listener* m_core_listener; + mutable std::unordered_map m_onboard_txs; bool m_txs_kept_by_block; bool m_skip_txs_blobsize_check; @@ -389,11 +390,19 @@ public: size_t pool_size = m_c.get_pool_transactions_count(); currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_c.handle_incoming_tx(tx_blob, tvc, m_txs_kept_by_block); - bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); - bool r = check_tx_verification_context(tvc, tx_added, m_ev_index, tx, m_validator); - LOCAL_ASSERT(r); - CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed"); + if (m_txs_kept_by_block) + { + m_onboard_txs[get_transaction_hash(tx)] = tx; + } + else + { + m_c.handle_incoming_tx(tx_blob, tvc, m_txs_kept_by_block); + bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); + bool r = check_tx_verification_context(tvc, tx_added, m_ev_index, tx, m_validator); + LOCAL_ASSERT(r); + CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed"); + } + return true; } @@ -403,6 +412,7 @@ public: m_core_listener->before_block_pushed_to_core(b, blob_blk, m_c); currency::block_verification_context bvc = AUTO_VAL_INIT(bvc); + bvc.m_onboard_transactions.swap(m_onboard_txs); m_c.handle_incoming_block(blob_blk, bvc); bool r = check_block_verification_context(bvc, m_ev_index, b, m_validator); CHECK_AND_NO_ASSERT_MES(r, false, "block verification context check failed"); @@ -846,6 +856,7 @@ int main(int argc, char* argv[]) GENERATE_AND_PLAY(wallet_outputs_with_same_key_image); GENERATE_AND_PLAY(wallet_unconfirmed_tx_expiration); GENERATE_AND_PLAY(wallet_unconfimed_tx_balance); + GENERATE_AND_PLAY(packing_outputs_on_pos_minting_wallet); GENERATE_AND_PLAY(wallet_rpc_integrated_address); GENERATE_AND_PLAY(wallet_rpc_integrated_address_transfer); diff --git a/tests/core_tests/double_spend.h b/tests/core_tests/double_spend.h index 3ca9ba9d..901f1ac5 100644 --- a/tests/core_tests/double_spend.h +++ b/tests/core_tests/double_spend.h @@ -42,7 +42,7 @@ struct gen_double_spend_in_the_same_block : public gen_double_spend_base< gen_do { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 1 : 2; + static const size_t expected_pool_txs_count = 1; static const uint64_t expected_bob_balance = send_amount; static const uint64_t expected_alice_balance = 0; @@ -55,7 +55,7 @@ struct gen_double_spend_in_different_blocks : public gen_double_spend_base< gen_ { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 0 : 1; + static const size_t expected_pool_txs_count = 0; static const uint64_t expected_bob_balance = 0; static const uint64_t expected_alice_balance = send_amount - TESTS_DEFAULT_FEE; @@ -68,7 +68,7 @@ struct gen_double_spend_in_alt_chain_in_the_same_block : public gen_double_spend { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 1 : 2; + static const size_t expected_pool_txs_count = txs_kept_by_block ? 0 :1; static const uint64_t expected_bob_balance = send_amount; static const uint64_t expected_alice_balance = 0; @@ -81,14 +81,12 @@ struct gen_double_spend_in_alt_chain_in_different_blocks : public gen_double_spe { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 1 : 2; + static const size_t expected_pool_txs_count = txs_kept_by_block ? 0:1; static const uint64_t expected_bob_balance = send_amount; - static const uint64_t expected_alice_balance = 0; - + static const uint64_t expected_alice_balance = 0; bool generate(std::vector& events) const; }; - class gen_double_spend_in_different_chains : public test_chain_unit_base { public: diff --git a/tests/core_tests/multisig_wallet_tests.cpp b/tests/core_tests/multisig_wallet_tests.cpp index e69cd871..0582d3ab 100644 --- a/tests/core_tests/multisig_wallet_tests.cpp +++ b/tests/core_tests/multisig_wallet_tests.cpp @@ -2361,7 +2361,7 @@ bool multisig_out_make_and_spent_in_altchain::generate(std::vector& events) events.push_back(tx_2); events.push_back(event_visitor_settings(event_visitor_settings::set_txs_kept_by_block, false)); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(3)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // make a block with tx_0 and put tx_0 to the blockchain MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, m_miner_acc, tx_0); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // tx_1 and tx_2 is still in the pool // it can never be added to any block as long as blk_1 is in the blockchain due to key image conflict @@ -1531,7 +1531,7 @@ bool tx_key_image_pool_conflict::generate(std::vector& events) MAKE_NEXT_BLOCK_TX1(events, blk_1a, blk_0r, m_miner_acc, tx_1); // however, it does not remove tx from the pool - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // // make sure stuck tx will be removed from the pool when it's too old @@ -1541,7 +1541,7 @@ bool tx_key_image_pool_conflict::generate(std::vector& events) // remove_stuck_txs should not remove anything, tx_1 and tx_2 should be in the pool DO_CALLBACK(events, "remove_stuck_txs"); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // shift time by CURRENCY_MEMPOOL_TX_LIVETIME events.push_back(event_core_time(CURRENCY_MEMPOOL_TX_LIVETIME + 1, true)); @@ -1562,11 +1562,11 @@ bool tx_key_image_pool_conflict::generate(std::vector& events) events.push_back(tx_2); events.push_back(event_visitor_settings(event_visitor_settings::set_txs_kept_by_block, false)); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // remove_stuck_txs should not remove anything, tx_1 and tx_2 should be in the pool DO_CALLBACK(events, "remove_stuck_txs"); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // rewind 50 blocks so tx_0 spending its key image will be deep enough REWIND_BLOCKS_N_WITH_TIME(events, blk_3r, blk_3, m_miner_acc, 50); diff --git a/tests/core_tests/wallet_tests.cpp b/tests/core_tests/wallet_tests.cpp index 05dd6f19..870f3099 100644 --- a/tests/core_tests/wallet_tests.cpp +++ b/tests/core_tests/wallet_tests.cpp @@ -3297,3 +3297,79 @@ bool wallet_unconfimed_tx_balance::c1(currency::core& c, size_t ev_index, const return true; } + +//------------------------------------------------------------------------------ + +packing_outputs_on_pos_minting_wallet::packing_outputs_on_pos_minting_wallet() +{ + REGISTER_CALLBACK_METHOD(packing_outputs_on_pos_minting_wallet, c1); + REGISTER_CALLBACK_METHOD(packing_outputs_on_pos_minting_wallet, set_core_config); +} +bool packing_outputs_on_pos_minting_wallet::generate(std::vector& events) const +{ + + // 0 10 11 21 22 <- blockchain height (assuming CURRENCY_MINED_MONEY_UNLOCK_WINDOW == 10) + // (0 )... (0r)- (1 )... (1r)- <- main chain + // tx_0 <- txs + + GENERATE_ACCOUNT(miner_acc); + m_accounts.push_back(miner_acc); + //GENERATE_ACCOUNT(alice_acc); + //m_accounts.push_back(alice_acc); + + // don't use MAKE_GENESIS_BLOCK here because it will mask 'generator' + currency::block blk_0 = AUTO_VAL_INIT(blk_0); + generator.construct_genesis_block(blk_0, miner_acc, test_core_time::get_time()); + events.push_back(blk_0); + + DO_CALLBACK(events, "set_core_config"); + + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW+5); + + //MAKE_TX_FEE(events, tx_0, miner_acc, alice_acc, MK_TEST_COINS(2000), TESTS_DEFAULT_FEE, blk_0r); + //MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_acc, tx_0); + //REWIND_BLOCKS_N_WITH_TIME(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); + + DO_CALLBACK(events, "c1"); + + return true; +} + +bool packing_outputs_on_pos_minting_wallet::set_core_config(currency::core& c, size_t ev_index, const std::vector& events) +{ + core_runtime_config crc = c.get_blockchain_storage().get_core_runtime_config(); + crc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; + crc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; + c.get_blockchain_storage().set_core_runtime_config(crc); + return true; +} + +bool packing_outputs_on_pos_minting_wallet::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + std::shared_ptr miner_wlt = init_playtime_test_wallet(events, c, MINER_ACC_IDX); + size_t blocks_fetched = 0; + bool received_money; + std::atomic atomic_false = ATOMIC_VAR_INIT(false); + miner_wlt->refresh(blocks_fetched, received_money, atomic_false); + CHECK_AND_ASSERT_MES(blocks_fetched == CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 5, false, "Incorrect numbers of blocks fetched"); + + miner_wlt->set_pos_mint_packing_size(4); + check_balance_via_wallet(*miner_wlt.get(), "miner_wlt", MK_TEST_COINS(2000), 0, MK_TEST_COINS(2000), 0, 0); + + miner_wlt->try_mint_pos(); + + CHECK_AND_ASSERT_MES(c.get_current_blockchain_size() == CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 7, false, "Incorrect blockchain height:" << c.get_current_blockchain_size()); + miner_wlt->refresh(blocks_fetched, received_money, atomic_false); + CHECK_AND_ASSERT_MES(blocks_fetched == 1, false, "Incorrect numbers of blocks fetched"); + + block top_block = AUTO_VAL_INIT(top_block); + bool r = c.get_blockchain_storage().get_top_block(top_block); + CHECK_AND_ASSERT_MES(r && is_pos_block(top_block), false, "get_top_block failed or smth goes wrong"); + uint64_t top_block_reward = get_outs_money_amount(top_block.miner_tx); + check_balance_via_wallet(*miner_wlt.get(), "miner_wlt", uint64_max, MK_TEST_COINS(2000) + top_block_reward, 0, 0, 0); + + miner_wlt->reset_password(g_wallet_password); + miner_wlt->store(g_wallet_filename); + + return true; +} \ No newline at end of file diff --git a/tests/core_tests/wallet_tests.h b/tests/core_tests/wallet_tests.h index 6fbd37bd..89dfb05f 100644 --- a/tests/core_tests/wallet_tests.h +++ b/tests/core_tests/wallet_tests.h @@ -251,3 +251,11 @@ struct wallet_unconfimed_tx_balance : public wallet_test bool generate(std::vector& events) const; bool c1(currency::core& c, size_t ev_index, const std::vector& events); }; + +struct packing_outputs_on_pos_minting_wallet : public wallet_test +{ + packing_outputs_on_pos_minting_wallet(); + bool generate(std::vector& events) const; + bool set_core_config(currency::core& c, size_t ev_index, const std::vector& events); + bool c1(currency::core& c, size_t ev_index, const std::vector& events); +};