From d1e6ef429bf8568d09baa54a40f4b953b4b987de Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 15 Nov 2019 01:04:51 +0100 Subject: [PATCH 01/21] refactoring of tx pool/core block's transactions handling --- src/currency_core/blockchain_storage.cpp | 2 +- .../currency_format_utils_transactions.cpp | 10 ++++++++++ .../currency_format_utils_transactions.h | 1 + src/currency_core/tx_pool.cpp | 17 +++++++++++++---- src/currency_core/tx_pool.h | 2 +- src/currency_core/verification_context.h | 4 ++++ 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index eaa25474..bd72ce8c 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -1351,7 +1351,7 @@ bool blockchain_storage::create_block_template(const create_block_template_param uint64_t fee; bool block_filled = false; if (pcustom_fill_block_template_func == nullptr) - block_filled = m_tx_pool.fill_block_template(b, pos, median_size, already_generated_coins, txs_size, fee, height); + block_filled = m_tx_pool.fill_block_template(b, pos, median_size, already_generated_coins, txs_size, fee, height, params.explicit_txs); else block_filled = (*pcustom_fill_block_template_func)(b, pos, median_size, already_generated_coins, txs_size, fee, height); diff --git a/src/currency_core/currency_format_utils_transactions.cpp b/src/currency_core/currency_format_utils_transactions.cpp index ba08914b..20ecd4e0 100644 --- a/src/currency_core/currency_format_utils_transactions.cpp +++ b/src/currency_core/currency_format_utils_transactions.cpp @@ -193,6 +193,16 @@ namespace currency return get_object_blobsize(t, tx_blob_size); } //--------------------------------------------------------------- + size_t get_objects_blobsize(const std::list& ls) + { + size_t total = 0; + for (const auto& tx : ls) + { + total += get_object_blobsize(tx); + } + return total; + } + //--------------------------------------------------------------- size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size) { size_t tx_blob_size = prefix_blob_size; diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index 4b993394..626fc156 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -104,6 +104,7 @@ namespace currency bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, uint64_t& blob_size); size_t get_object_blobsize(const transaction& t); + size_t get_objects_blobsize(const std::list& ls); size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size); blobdata tx_to_blob(const transaction& b); bool tx_to_blob(const transaction& b, blobdata& b_blob); diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index ae8737a8..aef91d5a 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -987,7 +987,9 @@ namespace currency const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, - uint64_t height) + uint64_t height, + const std::list& explicit_txs + ) { LOCAL_READONLY_TRANSACTION(); //typedef transactions_container::value_type txv; @@ -998,8 +1000,8 @@ namespace currency txs_v.reserve(m_db_transactions.size()); std::vector txs; - //std::transform(m_transactions.begin(), m_transactions.end(), txs.begin(), [](txv &a) -> txv * { return &a; }); - //keep getting it as a values cz db items cache will keep it as unserialied object stored by shared ptrs + + //keep getting it as a values cz db items cache will keep it as unserialised object stored by shared ptrs m_db_transactions.enumerate_keys([&](uint64_t i, crypto::hash& k){txs_v.resize(i + 1); txs_v[i].first = k; return true;}); txs.resize(txs_v.size(), nullptr); @@ -1024,7 +1026,9 @@ namespace currency return a_ > b_; }); - size_t current_size = 0; + + size_t explicit_total_size = get_objects_blobsize(explicit_txs); + size_t current_size = explicit_total_size; uint64_t current_fee = 0; uint64_t best_money; if (!get_block_reward(pos, median_size, CURRENCY_COINBASE_BLOB_RESERVED_SIZE, already_generated_coins, best_money, height)) { @@ -1133,6 +1137,11 @@ namespace currency } } } + // add explicit transactions + for (const auto& tx : explicit_txs) + { + bl.tx_hashes.push_back(get_transaction_hash(tx)); + } return true; } //--------------------------------------------------------------------------------- diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index 6d9bad9d..d8642e8e 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -117,7 +117,7 @@ namespace currency // load/store operations bool init(const std::string& config_folder, const boost::program_options::variables_map& vm); bool deinit(); - bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height); + bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height, const std::list& explicit_txs); bool get_transactions(std::list& txs) const; bool get_all_transactions_details(std::list& txs)const; bool get_all_transactions_brief_details(std::list& txs)const; diff --git a/src/currency_core/verification_context.h b/src/currency_core/verification_context.h index a9f1f646..f952e3bd 100644 --- a/src/currency_core/verification_context.h +++ b/src/currency_core/verification_context.h @@ -27,5 +27,9 @@ namespace currency bool m_already_exists; bool added_to_altchain; uint64_t height_difference; + //this is work like a first-level cache for transactions while block is getting handled. It lets transactions + //associated with the block to get handled directly to core without being handled by tx_pool(which makes full + //inputs validation, including signatures check) + std::unordered_map m_onboard_transactions; }; } From 499f822e97704d464d8f7374d68672de584aaa98 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 15 Nov 2019 23:58:09 +0100 Subject: [PATCH 02/21] moved tx semantics validation to separate funtion, added to pool validation and to core validation --- src/currency_core/blockchain_storage.cpp | 47 ++++- src/currency_core/currency_core.cpp | 103 +--------- src/currency_core/currency_core.h | 7 - src/currency_core/currency_format_utils.cpp | 10 +- .../currency_format_utils_transactions.h | 2 +- src/currency_core/tx_semantic_validation.cpp | 179 ++++++++++++++++++ src/currency_core/tx_semantic_validation.h | 16 ++ 7 files changed, 242 insertions(+), 122 deletions(-) create mode 100644 src/currency_core/tx_semantic_validation.cpp create mode 100644 src/currency_core/tx_semantic_validation.h diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index bd72ce8c..b90cc1bc 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -32,6 +32,7 @@ #include "storages/portable_storage_template_helper.h" #include "basic_pow_helpers.h" #include "version.h" +#include "tx_semantic_validation.h" #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "core" @@ -4762,6 +4763,18 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise return 0; } //------------------------------------------------------------------ +bool get_tx_from_cache(const crypto::hash& tx_id, std::unordered_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t fee) +{ + auto it = tx_cache.find(tx_id); + if (it == tx_cache.end()) + return false; + + tx = it->second; + blob_size = get_object_blobsize(tx); + fee = get_tx_fee(tx); + return true; +} +//------------------------------------------------------------------ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) { TIME_MEASURE_START_PD_MS(block_processing_time_0_ms); @@ -4882,7 +4895,10 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt transaction tx; size_t blob_size = 0; uint64_t fee = 0; - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + + bool taken_from_cache = get_tx_from_cache(tx_id, bvc.m_onboard_transactions, tx, blob_size, fee); + bool taken_from_pool = m_tx_pool.take_tx(tx_id, tx, blob_size, fee); + if(!taken_from_cache && !taken_from_pool) { LOG_PRINT_L0("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); purge_block_data_from_blockchain(bl, tx_processed_count); @@ -4891,6 +4907,15 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } + if (!validate_tx_semantic(tx, blob_size)) + { + LOG_PRINT_L0("Block with id: " << id << " has at least one transaction with wrong semantic, tx_id: " << tx_id); + purge_block_data_from_blockchain(bl, tx_processed_count); + //add_block_as_invalid(bl, id); + bvc.m_verification_failed = true; + return false; + } + append_per_block_increments_for_tx(tx, gindices); //If we under checkpoints, ring signatures should be pruned @@ -4905,9 +4930,12 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt { LOG_PRINT_L0("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs."); currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); - m_tx_pool.add_transaction_to_black_list(tx); - CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + if (taken_from_pool) + { + bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); + m_tx_pool.add_transaction_to_black_list(tx); + CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + } purge_block_data_from_blockchain(bl, tx_processed_count); add_block_as_invalid(bl, id); LOG_PRINT_L0("Block with id " << id << " added as invalid because of wrong inputs in transactions"); @@ -4925,10 +4953,13 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if(!add_transaction_from_block(tx, tx_id, id, current_bc_size, actual_timestamp)) { LOG_PRINT_L0("Block with id: " << id << " failed to add transaction to blockchain storage"); - currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); - m_tx_pool.add_transaction_to_black_list(tx); - CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + if (taken_from_pool) + { + currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); + m_tx_pool.add_transaction_to_black_list(tx); + CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + } purge_block_data_from_blockchain(bl, tx_processed_count); bvc.m_verification_failed = true; return false; diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index 20e81fed..77052259 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -19,6 +19,7 @@ using namespace epee; #include "currency_format_utils.h" #include "misc_language.h" #include "string_coding.h" +#include "tx_semantic_validation.h" #define MINIMUM_REQUIRED_FREE_SPACE_BYTES (1024 * 1024 * 100) @@ -185,13 +186,15 @@ namespace currency //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block) { + CHECK_AND_ASSERT_MES(!kept_by_block, false, "Transaction associated with block came throw handle_incoming_tx!(not allowed anymore)"); + tvc = boost::value_initialized(); //want to process all transactions sequentially TIME_MEASURE_START_MS(wait_lock_time); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); TIME_MEASURE_FINISH_MS(wait_lock_time); - if(tx_blob.size() > get_max_tx_size()) + if(tx_blob.size() > CURRENCY_MAX_TRANSACTION_BLOB_SIZE) { LOG_PRINT_L0("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected"); tvc.m_verification_failed = true; @@ -210,19 +213,10 @@ namespace currency TIME_MEASURE_FINISH_MS(parse_tx_time); - TIME_MEASURE_START_MS(check_tx_syntax_time); - if(!check_tx_syntax(tx)) - { - LOG_PRINT_L0("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " syntax, rejected"); - tvc.m_verification_failed = true; - return false; - } - TIME_MEASURE_FINISH_MS(check_tx_syntax_time); - TIME_MEASURE_START_MS(check_tx_semantic_time); - if(!check_tx_semantic(tx, kept_by_block)) + if(!validate_tx_semantic(tx, tx_blob.size())) { - LOG_PRINT_L0("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + LOG_PRINT_L0("WRONG TRANSACTION SEMANTICS, Failed to check tx " << tx_hash << " semantic, rejected"); tvc.m_verification_failed = true; return false; } @@ -243,7 +237,6 @@ namespace currency } LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX]: timing " << wait_lock_time << "/" << parse_tx_time - << "/" << check_tx_syntax_time << "/" << check_tx_semantic_time << "/" << add_new_tx_time); return r; @@ -296,88 +289,9 @@ namespace currency return true; } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_semantic(const transaction& tx, bool kept_by_block) - { - if(!tx.vin.size()) - { - LOG_PRINT_RED_L0("tx with empty inputs, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_inputs_types_supported(tx)) - { - LOG_PRINT_RED_L0("unsupported input types for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_outs_valid(tx)) - { - LOG_PRINT_RED_L0("tx with invalid outputs, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_money_overflow(tx)) - { - LOG_PRINT_RED_L0("tx has money overflow, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - - uint64_t amount_in = 0; - get_inputs_money_amount(tx, amount_in); - uint64_t amount_out = get_outs_money_amount(tx); - - if(amount_in < amount_out) - { - LOG_PRINT_RED_L0("tx with wrong amounts: ins " << amount_in << ", outs " << amount_out << ", rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - - if(!kept_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - CURRENCY_COINBASE_BLOB_RESERVED_SIZE) - { - LOG_PRINT_RED_L0("tx has too big size " << get_object_blobsize(tx) << ", expected no bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CURRENCY_COINBASE_BLOB_RESERVED_SIZE); - return false; - } - - //check if tx use different key images - if(!check_tx_inputs_keyimages_diff(tx)) - { - LOG_PRINT_RED_L0("tx inputs have the same key images"); - return false; - } - - if(!check_tx_extra(tx)) - { - LOG_PRINT_RED_L0("tx has wrong extra, rejected"); - return false; - } - - return true; - } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_extra(const transaction& tx) - { - tx_extra_info ei = AUTO_VAL_INIT(ei); - bool r = parse_and_validate_tx_extra(tx, ei); - if(!r) - return false; - return true; - } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_inputs_keyimages_diff(const transaction& tx) - { - std::unordered_set ki; - BOOST_FOREACH(const auto& in, tx.vin) - { - if (in.type() == typeid(txin_to_key)) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); - if (!ki.insert(tokey_in.k_image).second) - return false; - } - } - return true; - } //----------------------------------------------------------------------------------------------- bool core::add_new_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block) { @@ -627,11 +541,6 @@ namespace currency { return parse_and_validate_tx_from_blob(blob, tx, tx_hash); } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_syntax(const transaction& tx) - { - return true; - } //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions(std::list& txs) { diff --git a/src/currency_core/currency_core.h b/src/currency_core/currency_core.h index 7fa9313a..cdadf317 100644 --- a/src/currency_core/currency_core.h +++ b/src/currency_core/currency_core.h @@ -118,12 +118,6 @@ namespace currency bool add_new_block(const block& b, block_verification_context& bvc); bool load_state_data(); bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob); - bool check_tx_extra(const transaction& tx); - - bool check_tx_syntax(const transaction& tx); - //check correct values, amounts and all lightweight checks not related with database - bool check_tx_semantic(const transaction& tx, bool kept_by_block); - //check if tx already in memory pool or in main blockchain bool is_key_image_spent(const crypto::key_image& key_im); @@ -132,7 +126,6 @@ namespace currency bool update_miner_block_template(); bool handle_command_line(const boost::program_options::variables_map& vm); bool on_update_blocktemplate_interval(); - bool check_tx_inputs_keyimages_diff(const transaction& tx); void notify_blockchain_update_listeners(); diff --git a/src/currency_core/currency_format_utils.cpp b/src/currency_core/currency_format_utils.cpp index 79a63b06..e6ec0d01 100644 --- a/src/currency_core/currency_format_utils.cpp +++ b/src/currency_core/currency_format_utils.cpp @@ -2426,15 +2426,7 @@ namespace currency return true; return false; } - size_t get_max_block_size() - { - return CURRENCY_MAX_BLOCK_SIZE; - } - //----------------------------------------------------------------------------------------------- - size_t get_max_tx_size() - { - return CURRENCY_MAX_TRANSACTION_BLOB_SIZE; - } + //----------------------------------------------------------------------------------------------- uint64_t get_base_block_reward(bool is_pos, const boost::multiprecision::uint128_t& already_generated_coins, uint64_t height) { diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index 626fc156..d903f38e 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -92,7 +92,7 @@ namespace currency inline void set_tx_flags(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } inline void set_tx_expiration_time(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations); - + //----------------------------------------------------------------------------------------------- bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median); uint64_t get_burned_amount(const transaction& tx); diff --git a/src/currency_core/tx_semantic_validation.cpp b/src/currency_core/tx_semantic_validation.cpp new file mode 100644 index 00000000..db29bce1 --- /dev/null +++ b/src/currency_core/tx_semantic_validation.cpp @@ -0,0 +1,179 @@ +// Copyright (c) 2018-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + + +#include "tx_semantic_validation.h" +#include "currency_format_utils.h" + +namespace currency +{ + //----------------------------------------------------------------------------------------------- + bool check_tx_extra(const transaction& tx) + { + tx_extra_info ei = AUTO_VAL_INIT(ei); + bool r = parse_and_validate_tx_extra(tx, ei); + if (!r) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool check_tx_inputs_keyimages_diff(const transaction& tx) + { + std::unordered_set ki; + BOOST_FOREACH(const auto& in, tx.vin) + { + if (in.type() == typeid(txin_to_key)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); + if (!ki.insert(tokey_in.k_image).second) + return false; + } + } + return true; + } + //----------------------------------------------------------------------------------------------- + bool validate_tx_semantic(const transaction& tx, size_t tx_block_size) + { + if (!tx.vin.size()) + { + LOG_PRINT_RED_L0("tx with empty inputs, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_inputs_types_supported(tx)) + { + LOG_PRINT_RED_L0("unsupported input types for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_outs_valid(tx)) + { + LOG_PRINT_RED_L0("tx with invalid outputs, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_money_overflow(tx)) + { + LOG_PRINT_RED_L0("tx has money overflow, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + uint64_t amount_in = 0; + get_inputs_money_amount(tx, amount_in); + uint64_t amount_out = get_outs_money_amount(tx); + + if (amount_in < amount_out) + { + LOG_PRINT_RED_L0("tx with wrong amounts: ins " << amount_in << ", outs " << amount_out << ", rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (tx_block_size >= CURRENCY_MAX_TRANSACTION_BLOB_SIZE) + { + LOG_PRINT_RED_L0("tx has too big size " << tx_block_size << ", expected no bigger than " << CURRENCY_BLOCK_GRANTED_FULL_REWARD_ZONE); + return false; + } + + //check if tx use different key images + if (!check_tx_inputs_keyimages_diff(tx)) + { + LOG_PRINT_RED_L0("tx inputs have the same key images"); + return false; + } + + if (!check_tx_extra(tx)) + { + LOG_PRINT_RED_L0("tx has wrong extra, rejected"); + return false; + } + + return true; + } + //--------------------------------------------------------------- + bool get_transaction_hash(const transaction& t, crypto::hash& res) + { + uint64_t blob_size = 0; + return get_object_hash(static_cast(t), res, blob_size); + } + //--------------------------------------------------------------- + bool get_transaction_hash(const transaction& t, crypto::hash& res, uint64_t& blob_size) + { + blob_size = 0; + bool r = get_object_hash(static_cast(t), res, blob_size); + blob_size = get_object_blobsize(t, blob_size); + return r; + } + //--------------------------------------------------------------- + size_t get_object_blobsize(const transaction& t) + { + size_t tx_blob_size = get_object_blobsize(static_cast(t)); + return get_object_blobsize(t, tx_blob_size); + } + //--------------------------------------------------------------- + size_t get_objects_blobsize(const std::list& ls) + { + size_t total = 0; + for (const auto& tx : ls) + { + total += get_object_blobsize(tx); + } + return total; + } + //--------------------------------------------------------------- + size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size) + { + size_t tx_blob_size = prefix_blob_size; + + if (is_coinbase(t)) + return tx_blob_size; + + // for purged tx, with empty signatures and attachments, this function should return the blob size + // which the tx would have if the signatures and attachments were correctly filled with actual data + + // 1. signatures + bool separately_signed_tx = get_tx_flags(t) & TX_FLAG_SIGNATURE_MODE_SEPARATE; + + tx_blob_size += tools::get_varint_packed_size(t.vin.size()); // size of transaction::signatures (equals to total inputs count) + + for (size_t i = 0; i != t.vin.size(); i++) + { + size_t sig_count = get_input_expected_signatures_count(t.vin[i]); + if (separately_signed_tx && i == t.vin.size() - 1) + ++sig_count; // count in one more signature for the last input in a complete separately signed tx + tx_blob_size += tools::get_varint_packed_size(sig_count); // size of transaction::signatures[i] + tx_blob_size += sizeof(crypto::signature) * sig_count; // size of signatures' data itself + } + + // 2. attachments (try to find extra_attachment_info in tx prefix and count it in if succeed) + extra_attachment_info eai = AUTO_VAL_INIT(eai); + bool got_eai = false; + if (separately_signed_tx) + { + // for separately-signed tx, try to obtain extra_attachment_info from the last input's etc_details + const std::vector* p_etc_details = get_input_etc_details(t.vin.back()); + got_eai = p_etc_details != nullptr && get_type_in_variant_container(*p_etc_details, eai); + } + if (!got_eai) + got_eai = get_type_in_variant_container(t.extra, eai); // then from the extra + + if (got_eai) + tx_blob_size += eai.sz; // sz is a size of whole serialized attachment blob, including attachments vector size + else + tx_blob_size += tools::get_varint_packed_size(static_cast(0)); // no extra_attachment_info found - just add zero vector's size, 'cause it's serialized anyway + + return tx_blob_size; + } + //--------------------------------------------------------------- + blobdata tx_to_blob(const transaction& tx) + { + return t_serializable_object_to_blob(tx); + } + //--------------------------------------------------------------- + bool tx_to_blob(const transaction& tx, blobdata& b_blob) + { + return t_serializable_object_to_blob(tx, b_blob); + } + +} \ No newline at end of file diff --git a/src/currency_core/tx_semantic_validation.h b/src/currency_core/tx_semantic_validation.h new file mode 100644 index 00000000..2e6f4790 --- /dev/null +++ b/src/currency_core/tx_semantic_validation.h @@ -0,0 +1,16 @@ +// Copyright (c) 2018-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + + +#include "include_base_utils.h" +#include "currency_format_utils_transactions.h" + + +namespace currency +{ + //check correct values, amounts and all lightweight checks not related with database + bool validate_tx_semantic(const transaction& tx, size_t tx_block_size); +} From 17e36012de9baa6faacb4fb80aab75b5010a11c6 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 16 Nov 2019 01:06:21 +0100 Subject: [PATCH 03/21] transaction pool optimisation: basic implemntation --- src/currency_core/blockchain_storage.cpp | 56 +++++++++++-------- src/currency_core/blockchain_storage.h | 11 ++-- src/currency_core/blockchain_storage_basic.h | 8 ++- src/currency_core/verification_context.h | 6 +- .../currency_protocol_handler.inl | 4 +- src/rpc/core_rpc_server.cpp | 2 +- src/stratum/stratum_server.cpp | 4 +- 7 files changed, 56 insertions(+), 35 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index b90cc1bc..571e7345 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -439,7 +439,7 @@ bool blockchain_storage::deinit() return true; } //------------------------------------------------------------------ -bool blockchain_storage::pop_block_from_blockchain() +bool blockchain_storage::pop_block_from_blockchain(transactions_map& onboard_transactions) { CRITICAL_REGION_LOCAL(m_read_lock); @@ -449,7 +449,7 @@ bool blockchain_storage::pop_block_from_blockchain() CHECK_AND_ASSERT_MES(bei_ptr.get(), false, "pop_block_from_blockchain: can't pop from blockchain"); uint64_t fee_total = 0; - bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total); + bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total, onboard_transactions); CHECK_AND_ASSERT_MES(r, false, "Failed to purge_block_data_from_blockchain for block " << get_block_hash(bei_ptr->bl) << " on height " << h); pop_block_from_per_block_increments(bei_ptr->height); @@ -649,7 +649,7 @@ bool blockchain_storage::purge_transaction_keyimages_from_blockchain(const trans return true; } //------------------------------------------------------------------ -bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee) +bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx_) { fee = 0; CRITICAL_REGION_LOCAL(m_read_lock); @@ -657,6 +657,7 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t auto tx_res_ptr = m_db_transactions.find(tx_id); CHECK_AND_ASSERT_MES(tx_res_ptr != m_db_transactions.end(), false, "transaction " << tx_id << " is not found in blockchain index!!"); const transaction& tx = tx_res_ptr->tx; + tx_ = tx; fee = get_tx_fee(tx_res_ptr->tx); purge_transaction_keyimages_from_blockchain(tx, true); @@ -687,10 +688,11 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t bool blockchain_storage::purge_block_data_from_blockchain(const block& b, size_t processed_tx_count) { uint64_t total_fee = 0; - return purge_block_data_from_blockchain(b, processed_tx_count, total_fee); + transactions_map onboard_transactions; + return purge_block_data_from_blockchain(b, processed_tx_count, total_fee, onboard_transactions); } //------------------------------------------------------------------ -bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total) +bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total, transactions_map& onboard_transactions) { CRITICAL_REGION_LOCAL(m_read_lock); fee_total = 0; @@ -699,11 +701,13 @@ bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_ CHECK_AND_ASSERT_MES(processed_tx_count <= bl.tx_hashes.size(), false, "wrong processed_tx_count in purge_block_data_from_blockchain"); for(size_t count = 0; count != processed_tx_count; count++) { - res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee) && res; + transaction tx = AUTO_VAL_INIT(tx); + res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee, tx) && res; fee_total += fee; + onboard_transactions[bl.tx_hashes[(processed_tx_count - 1) - count]] = tx; } - - res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee) && res; + transaction tx = AUTO_VAL_INIT(tx); + res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee, tx) && res; return res; } //------------------------------------------------------------------ @@ -866,20 +870,22 @@ bool blockchain_storage::get_block_by_height(uint64_t h, block &blk) const // invalid.push_back(v.first); // } //------------------------------------------------------------------ -bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) +bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) { CRITICAL_REGION_LOCAL(m_read_lock); //remove failed subchain for(size_t i = m_db_blocks.size()-1; i >=rollback_height; i--) { - bool r = pop_block_from_blockchain(); + transactions_map ot; + bool r = pop_block_from_blockchain(ot); CHECK_AND_ASSERT_MES(r, false, "PANIC!!! failed to remove block while chain switching during the rollback!"); } //return back original chain - BOOST_FOREACH(auto& bl, original_chain) + BOOST_FOREACH(auto& oce, original_chain) { block_verification_context bvc = boost::value_initialized(); - bool r = handle_block_to_main_chain(bl, bvc); + bvc.m_onboard_transactions.swap(oce.onboard_transactions); + bool r = handle_block_to_main_chain(oce.b, bvc); CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC!!! failed to add (again) block while chain switching during the rollback!"); } @@ -942,13 +948,15 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch ); //disconnecting old chain - std::list disconnected_chain; + std::list disconnected_chain; for(size_t i = m_db_blocks.size()-1; i >=split_height; i--) { - block b = m_db_blocks[i]->bl; - bool r = pop_block_from_blockchain(); - CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(b) << " @ " << get_block_height(b) << " on chain switching"); - disconnected_chain.push_front(b); + disconnected_chain.push_front(block_ws_txs()); + block_ws_txs& bwt = disconnected_chain.front(); + bwt.b = m_db_blocks[i]->bl; + bool r = pop_block_from_blockchain(bwt.onboard_transactions); + CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(bwt.b) << " @ " << get_block_height(bwt.b) << " on chain switching"); + CHECK_AND_ASSERT_MES(validate_blockchain_prev_links(), false, "EPIC FAIL!"); } @@ -957,6 +965,7 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch { auto ch_ent = *alt_ch_iter; block_verification_context bvc = boost::value_initialized(); + bvc.m_onboard_transactions = ch_ent->second.onboard_transactions; bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc); if(!r || !bvc.m_added_to_main_chain) { @@ -978,7 +987,8 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch for(auto& old_ch_ent : disconnected_chain) { block_verification_context bvc = boost::value_initialized(); - bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc); + bvc.m_onboard_transactions.swap(old_ch_ent.onboard_transactions); + bool r = handle_alternative_block(old_ch_ent.b, get_block_hash(old_ch_ent.b), bvc); if(!r) { LOG_ERROR("Failed to push ex-main chain blocks to alternative chain "); @@ -1602,6 +1612,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: alt_block_extended_info abei = AUTO_VAL_INIT(abei); abei.bl = b; + abei.onboard_transactions.swap(bvc.m_onboard_transactions); abei.timestamp = m_core_runtime_config.get_core_time(); abei.height = alt_chain.size() ? it_prev->second.height + 1 : *ptr_main_prev + 1; CHECK_AND_ASSERT_MES_CUSTOM(coinbase_height == abei.height, false, bvc.m_verification_failed = true, "block coinbase height doesn't match with altchain height, declined"); @@ -1716,7 +1727,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: alt_chain.push_back(i_res.first); //check if difficulty bigger then in main chain - bvc.height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0; + bvc.m_height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0; crypto::hash proof = null_hash; std::stringstream ss_pow_pos_info; @@ -1753,7 +1764,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: bvc.m_verification_failed = true; return r; } - bvc.added_to_altchain = true; + bvc.m_added_to_altchain = true; //protect ourself from altchains container flood if (m_alternative_chains.size() > m_core_runtime_config.max_alt_blocks) @@ -4763,7 +4774,7 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise return 0; } //------------------------------------------------------------------ -bool get_tx_from_cache(const crypto::hash& tx_id, std::unordered_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t fee) +bool get_tx_from_cache(const crypto::hash& tx_id, transactions_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t fee) { auto it = tx_cache.find(tx_id); if (it == tx_cache.end()) @@ -5344,7 +5355,8 @@ bool blockchain_storage::truncate_blockchain(uint64_t to_height) uint64_t inital_height = get_current_blockchain_size(); while (get_current_blockchain_size() > to_height) { - pop_block_from_blockchain(); + transactions_map ot; + pop_block_from_blockchain(ot); } CRITICAL_REGION_LOCAL(m_alternative_chains_lock); m_alternative_chains.clear(); diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 46af15bd..70187fda 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -156,6 +156,9 @@ namespace currency //date added to alt chain storage uint64_t timestamp; + + //transactions associated with the block + transactions_map onboard_transactions; }; typedef std::unordered_map alt_chain_container; //typedef std::list alt_chain_type; @@ -543,10 +546,10 @@ namespace currency bool switch_to_alternative_blockchain(alt_chain_type& alt_chain); void purge_alt_block_txs_hashs(const block& b); void add_alt_block_txs_hashs(const block& b); - bool pop_block_from_blockchain(); + bool pop_block_from_blockchain(transactions_map& onboard_transactions); bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count); - bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee); - bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee); + bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee, transactions_map& onboard_transactions); + bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx); bool purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check); wide_difficulty_type get_next_difficulty_for_alternative_chain(const alt_chain_type& alt_chain, block_extended_info& bei, bool pos) const; bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); @@ -565,7 +568,7 @@ namespace currency bool get_transaction_from_pool_or_db(const crypto::hash& tx_id, std::shared_ptr& tx_ptr, uint64_t min_allowed_block_height = 0) const; void get_last_n_x_blocks(uint64_t n, bool pos_blocks, std::list>& blocks) const; bool prevalidate_miner_transaction(const block& b, uint64_t height, bool pos)const; - bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); + bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, uint64_t timestamp); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); diff --git a/src/currency_core/blockchain_storage_basic.h b/src/currency_core/blockchain_storage_basic.h index b232b32e..70d2ada9 100644 --- a/src/currency_core/blockchain_storage_basic.h +++ b/src/currency_core/blockchain_storage_basic.h @@ -145,6 +145,12 @@ namespace currency uint64_t height; }; - + typedef std::unordered_map transactions_map; + + struct block_ws_txs + { + block b; + transactions_map onboard_transactions; + }; } \ No newline at end of file diff --git a/src/currency_core/verification_context.h b/src/currency_core/verification_context.h index f952e3bd..85533cc5 100644 --- a/src/currency_core/verification_context.h +++ b/src/currency_core/verification_context.h @@ -25,11 +25,11 @@ namespace currency bool m_verification_failed; //bad block, should drop connection bool m_marked_as_orphaned; bool m_already_exists; - bool added_to_altchain; - uint64_t height_difference; + bool m_added_to_altchain; + uint64_t m_height_difference; //this is work like a first-level cache for transactions while block is getting handled. It lets transactions //associated with the block to get handled directly to core without being handled by tx_pool(which makes full //inputs validation, including signatures check) - std::unordered_map m_onboard_transactions; + transactions_map m_onboard_transactions; }; } diff --git a/src/currency_protocol/currency_protocol_handler.inl b/src/currency_protocol/currency_protocol_handler.inl index 2566bd24..c2991eb2 100644 --- a/src/currency_protocol/currency_protocol_handler.inl +++ b/src/currency_protocol/currency_protocol_handler.inl @@ -327,10 +327,10 @@ namespace currency LOG_PRINT_GREEN("[HANDLE]NOTIFY_NEW_BLOCK EXTRA " << block_id << " bvc.m_added_to_main_chain=" << bvc.m_added_to_main_chain //<< ", prevalidate_result=" << prevalidate_relayed - << ", bvc.added_to_altchain=" << bvc.added_to_altchain + << ", bvc.added_to_altchain=" << bvc.m_added_to_altchain << ", bvc.m_marked_as_orphaned=" << bvc.m_marked_as_orphaned, LOG_LEVEL_2); - if (bvc.m_added_to_main_chain || (bvc.added_to_altchain && bvc.height_difference < 2)) + if (bvc.m_added_to_main_chain || (bvc.m_added_to_altchain && bvc.m_height_difference < 2)) { if (true/*!prevalidate_relayed*/) { diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index e8ca9317..40ef799c 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -851,7 +851,7 @@ namespace currency block_verification_context bvc = AUTO_VAL_INIT(bvc); if(!m_core.handle_block_found(b, &bvc)) { - if (bvc.added_to_altchain) + if (bvc.m_added_to_altchain) { error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_ADDED_AS_ALTERNATIVE; error_resp.message = "Block added as alternative"; diff --git a/src/stratum/stratum_server.cpp b/src/stratum/stratum_server.cpp index 586d5e09..2b454c47 100644 --- a/src/stratum/stratum_server.cpp +++ b/src/stratum/stratum_server.cpp @@ -532,7 +532,7 @@ namespace r = m_p_core->handle_block_found(m_block_template, &bvc, false); if (r) { - if (!bvc.m_verification_failed && !bvc.added_to_altchain && bvc.m_added_to_main_chain && !bvc.m_already_exists && !bvc.m_marked_as_orphaned) + if (!bvc.m_verification_failed && !bvc.m_added_to_altchain && bvc.m_added_to_main_chain && !bvc.m_already_exists && !bvc.m_marked_as_orphaned) { LP_CC_WORKER_GREEN(p_ph->get_context(), "found block " << block_hash << " at height " << height << " was successfully added to the blockchain, difficulty " << m_network_difficulty, LOG_LEVEL_0); r = update_block_template(); @@ -544,7 +544,7 @@ namespace { LP_CC_WORKER_RED(p_ph->get_context(), "block " << block_hash << " at height " << height << " was NOT added to the blockchain:" << ENDL << " verification_failed: " << bvc.m_verification_failed << ENDL << - " added_to_altchain: " << bvc.added_to_altchain << ENDL << + " added_to_altchain: " << bvc.m_added_to_altchain << ENDL << " added_to_main_chain: " << bvc.m_added_to_main_chain << ENDL << " already_exists: " << bvc.m_already_exists << ENDL << " marked_as_orphaned: " << bvc.m_marked_as_orphaned, LOG_LEVEL_0); From 57e0aa063d103f734513c7e9bb57b7cd74e2d197 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 16 Nov 2019 20:26:40 +0100 Subject: [PATCH 04/21] pool code modificated to work with onboard transactions --- contrib/epee/include/misc_language.h | 16 ++- contrib/epee/include/syncobj.h | 3 + src/currency_core/blockchain_storage.cpp | 11 +- src/currency_core/blockchain_storage.h | 2 +- .../currency_format_utils_transactions.cpp | 15 +++ .../currency_format_utils_transactions.h | 1 + src/currency_core/tx_pool.cpp | 102 +++++++++--------- src/currency_core/tx_pool.h | 21 ++-- 8 files changed, 107 insertions(+), 64 deletions(-) diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index b0cfd15c..6c2a287d 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -393,7 +393,21 @@ namespace misc_utils auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); return res.first->second; - } + } + + template + typename t_container_type::iterator it_get_or_insert_value_initialized(t_container_type& container, const typename t_container_type::key_type& key) + { + auto it = container.find(key); + if (it != container.end()) + { + return it; + } + + auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); + return res.first; + } + } // namespace misc_utils } // namespace epee diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index dffbd614..b5907538 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -698,6 +698,9 @@ namespace epee #define CRITICAL_REGION_BEGIN1(x) CRITICAL_REGION_BEGIN_VAR(x, critical_region_var1) #define CRITICAL_REGION_END() } + +#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj##.##op;obj##_lock.unlock();} + #define SHARED_CRITICAL_REGION_LOCAL(x) boost::shared_lock< boost::shared_mutex > critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_LOCAL(x) boost::unique_lock< boost::shared_mutex > critical_region_var(x) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 571e7345..6743337b 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -4900,6 +4900,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t tx_processed_count = 0; uint64_t fee_summary = 0; uint64_t burned_coins = 0; + std::list block_summary_kimages; for(const crypto::hash& tx_id : bl.tx_hashes) { @@ -4983,6 +4984,8 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt ++tx_processed_count; if (fee) block_fees.push_back(fee); + + read_keyimages_from_tx(tx, block_summary_kimages); } TIME_MEASURE_FINISH_PD(all_txs_insert_time_5); @@ -5017,8 +5020,6 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if (is_pos_bl) bei.stake_hash = proof_hash; - - ////////////////////////////////////////////////////////////////////////// //old style cumulative difficulty collecting @@ -5156,17 +5157,17 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "/" << etc_stuff_6 << "))"); - on_block_added(bei, id); + on_block_added(bei, id, block_summary_kimages); bvc.m_added_to_main_chain = true; return true; } //------------------------------------------------------------------ -void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id) +void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk) { update_next_comulative_size_limit(); m_timestamps_median_cache.clear(); - m_tx_pool.on_blockchain_inc(bei.height, id); + m_tx_pool.on_blockchain_inc(bei.height, id, bsk); update_targetdata_cache_on_block_added(bei); diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 70187fda..bc595256 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -581,7 +581,7 @@ namespace currency bool check_block_timestamp(std::vector timestamps, const block& b)const; std::vector get_last_n_blocks_timestamps(size_t n)const; const std::vector& get_txin_etc_options(const txin_v& in)const; - void on_block_added(const block_extended_info& bei, const crypto::hash& id); + void on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk); void on_block_removed(const block_extended_info& bei); void update_targetdata_cache_on_block_added(const block_extended_info& bei); void update_targetdata_cache_on_block_removed(const block_extended_info& bei); diff --git a/src/currency_core/currency_format_utils_transactions.cpp b/src/currency_core/currency_format_utils_transactions.cpp index 20ecd4e0..da78894f 100644 --- a/src/currency_core/currency_format_utils_transactions.cpp +++ b/src/currency_core/currency_format_utils_transactions.cpp @@ -256,5 +256,20 @@ namespace currency { return t_serializable_object_to_blob(tx, b_blob); } + //--------------------------------------------------------------- + bool read_keyimages_from_tx(const transaction& tx, std::list& kil) + { + std::unordered_set ki; + BOOST_FOREACH(const auto& in, tx.vin) + { + if (in.type() == typeid(txin_to_key)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); + if (!ki.insert(tokey_in.k_image).second) + return false; + } + } + return true; + } } \ No newline at end of file diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index d903f38e..603329e2 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -108,4 +108,5 @@ namespace currency size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size); blobdata tx_to_blob(const transaction& b); bool tx_to_blob(const transaction& b, blobdata& b_blob); + bool read_keyimages_from_tx(const transaction& tx, std::list& kil); } diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index aef91d5a..8a3045b1 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -50,7 +50,7 @@ namespace currency m_db_transactions(m_db), m_db_black_tx_list(m_db), m_db_solo_options(m_db), - m_db_key_images_set(m_db), +// m_db_key_images_set(m_db), m_db_alias_names(m_db), m_db_alias_addresses(m_db), m_db_storage_major_compatibility_version(TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION, m_db_solo_options) @@ -255,7 +255,7 @@ namespace currency td.receive_time = get_core_time(); m_db_transactions.set(id, td); - on_tx_add(tx, kept_by_block); + on_tx_add(id, tx, kept_by_block); TIME_MEASURE_FINISH_PD(update_db_time); return true; @@ -385,7 +385,7 @@ namespace currency blob_size = txe_tr->blob_size; fee = txe_tr->fee; m_db_transactions.erase(id); - on_tx_remove(tx, txe_tr->kept_by_block); + on_tx_remove(id, tx, txe_tr->kept_by_block); set_taken(id); return true; } @@ -472,7 +472,7 @@ namespace currency for (auto& e : to_delete) { m_db_transactions.erase(e.hash); - on_tx_remove(e.tx, e.kept_by_block); + on_tx_remove(e.hash, e.tx, e.kept_by_block); } @@ -613,8 +613,10 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) + bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk) { + + return true; } //--------------------------------------------------------------------------------- @@ -694,36 +696,33 @@ namespace currency return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::insert_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::insert_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (ki_entry_ptr.get()) - count = *ki_entry_ptr; - uint64_t count_before = count; - ++count; - m_db_key_images_set.set(tokey_in.k_image, count); - LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + const txin_to_key& tokey_in = boost::get(in); + auto& id_set = m_key_images[tokey_in.k_image]; + size_t sz_before = id_set.size(); + id_set.insert(tx_id); + LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << sz_before << " -> " << id_set.size()); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_add(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block) { - insert_key_images(tx, kept_by_block); + insert_key_images(tx_id, tx, kept_by_block); insert_alias_info(tx); return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_remove(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_remove(const crypto::hash &id, const transaction& tx, bool kept_by_block) { - remove_key_images(tx, kept_by_block); + remove_key_images(id, tx, kept_by_block); remove_alias_info(tx); return true; } @@ -757,34 +756,33 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::remove_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (!ki_entry_ptr.get() || *ki_entry_ptr == 0) - { - LOG_ERROR("INTERNAL_ERROR: for tx " << get_transaction_hash(tx) << " key image " << tokey_in.k_image << " not found"); - continue; - } - count = *ki_entry_ptr; - uint64_t count_before = count; - --count; - if (count) - m_db_key_images_set.set(tokey_in.k_image, count); - else - m_db_key_images_set.erase(tokey_in.k_image); - LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + { + const txin_to_key& tokey_in = boost::get(in); + + auto it_map = epee::misc_utils::it_get_or_insert_value_initialized(m_key_images, tokey_in.k_image); + auto& id_set = it_map->second; + size_t count_before = id_set.size(); + auto it_set = id_set.find(tx_id); + if(it_set != id_set.end()) + id_set.erase(it_set); + + size_t count_after = id_set.size(); + if (id_set.size() == 0) + m_key_images.erase(it_map); + + LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << count_before << " -> " << count_after); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_key_images_from_tx_pool(std::unordered_set& key_images) const + bool tx_memory_pool::get_key_images_from_tx_pool(key_image_cache& key_images) const { m_db_transactions.enumerate_items([&](uint64_t i, const crypto::hash& h, const tx_details &tx_entry) @@ -793,7 +791,7 @@ namespace currency { if (in.type() == typeid(txin_to_key)) { - key_images.insert(boost::get(in).k_image); + key_images[boost::get(in).k_image].insert(h); } } return true; @@ -804,9 +802,9 @@ namespace currency //--------------------------------------------------------------------------------- bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im)const { - - auto ptr = m_db_key_images_set.find(key_im); - if (ptr) + CRITICAL_REGION_LOCAL(m_key_images_lock); + auto it = m_key_images.find(key_im); + if (it != m_key_images.end()) return true; return false; } @@ -826,9 +824,9 @@ namespace currency m_db.begin_transaction(); m_db_transactions.clear(); - m_db_key_images_set.clear(); m_db.commit_transaction(); // should m_db_black_tx_list be cleared here? + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- void tx_memory_pool::clear() @@ -836,8 +834,8 @@ namespace currency m_db.begin_transaction(); m_db_transactions.clear(); m_db_black_tx_list.clear(); - m_db_key_images_set.clear(); m_db.commit_transaction(); + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- bool tx_memory_pool::is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const @@ -1193,8 +1191,8 @@ namespace currency res = m_db_transactions.init(TRANSACTION_POOL_CONTAINER_TRANSACTIONS); CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); +// res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); +// CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); res = m_db_black_tx_list.init(TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST); CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); res = m_db_alias_names.init(TRANSACTION_POOL_CONTAINER_ALIAS_NAMES); @@ -1220,7 +1218,7 @@ namespace currency { LOG_PRINT_L1("DB at " << db_folder_path << " is about to be deleted and re-created..."); m_db_transactions.deinit(); - m_db_key_images_set.deinit(); +// m_db_key_images_set.deinit(); m_db_black_tx_list.deinit(); m_db_alias_names.deinit(); m_db_alias_addresses.deinit(); @@ -1252,9 +1250,17 @@ namespace currency }); LOG_PRINT_L2(ss.str()); } + + load_keyimages_cache(); + return true; } - + //--------------------------------------------------------------------------------- + bool tx_memory_pool::load_keyimages_cache() + { + CRITICAL_REGION_LOCAL(m_key_images_lock); + return get_key_images_from_tx_pool(m_key_images); + } //--------------------------------------------------------------------------------- bool tx_memory_pool::deinit() { diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index d8642e8e..b3031566 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -81,6 +81,7 @@ namespace currency epee::math_helper::average db_commit_time; }; + typedef std::unordered_map> key_image_cache; tx_memory_pool(blockchain_storage& bchs, i_currency_protocol* pprotocol); bool add_tx(const transaction &tx, const crypto::hash &id, uint64_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool from_core = false); @@ -99,7 +100,7 @@ namespace currency bool check_tx_multisig_ins_and_outs(const transaction& tx, bool check_against_pool_txs)const; - bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id); + bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk); bool on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id); bool on_finalize_db_transaction(); bool add_transaction_to_black_list(const transaction& tx); @@ -139,10 +140,10 @@ namespace currency bool remove_stuck_transactions(); // made public to be called from coretests private: - bool on_tx_add(const transaction& tx, bool kept_by_block); - bool on_tx_remove(const transaction& tx, bool kept_by_block); - bool insert_key_images(const transaction& tx, bool kept_by_block); - bool remove_key_images(const transaction& tx, bool kept_by_block); + bool on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block); + bool on_tx_remove(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); + bool insert_key_images(const crypto::hash& tx_id, const transaction& tx, bool kept_by_block); + bool remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); bool insert_alias_info(const transaction& tx); bool remove_alias_info(const transaction& tx); @@ -150,12 +151,11 @@ namespace currency void store_db_solo_options_values(); bool is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const; bool validate_alias_info(const transaction& tx, bool is_in_block)const; - bool get_key_images_from_tx_pool(std::unordered_set& key_images)const; - //bool push_alias_info(const transaction& tx); - //bool pop_alias_info(const transaction& tx); + bool get_key_images_from_tx_pool(key_image_cache& key_images) const; bool check_is_taken(const crypto::hash& id) const; void set_taken(const crypto::hash& id); void reset_all_taken(); + bool load_keyimages_cache(); typedef tools::db::cached_key_value_accessor transactions_container; typedef tools::db::cached_key_value_accessor hash_container; @@ -172,7 +172,7 @@ namespace currency transactions_container m_db_transactions; hash_container m_db_black_tx_list; - key_images_container m_db_key_images_set; + //key_images_container m_db_key_images_set; aliases_container m_db_alias_names; address_to_aliases_container m_db_alias_addresses; solo_options_container m_db_solo_options; @@ -189,6 +189,9 @@ namespace currency //in memory containers mutable epee::critical_section m_taken_txs_lock; std::unordered_set m_taken_txs; + + mutable epee::critical_section m_key_images_lock; + key_image_cache m_key_images; mutable epee::critical_section m_remove_stuck_txs_lock; From 6019ffccf89ff154fd3a907a3be07cf17578185a Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 16 Nov 2019 21:25:48 +0100 Subject: [PATCH 05/21] changed currency_protocol_handler for onboard transactions --- src/CMakeLists.txt | 1 - src/currency_core/blockchain_storage_basic.h | 1 + src/currency_core/currency_format_utils.cpp | 5 +- src/currency_core/tx_semantic_validation.cpp | 85 ------------------- .../currency_protocol_handler.inl | 16 +++- 5 files changed, 17 insertions(+), 91 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ba358dd6..630a41c0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,6 @@ if(POLICY CMP0043) cmake_policy(SET CMP0043 OLD) endif() - ########### # using shared PCH -- this is unusual case for MSVC... so mystery, such hack, many wow. See also: https://stackoverflow.com/questions/645747/sharing-precompiled-headers-between-projects-in-visual-studio/4170902#4170902 # define USE_PCH to YES for using precomiled headers diff --git a/src/currency_core/blockchain_storage_basic.h b/src/currency_core/blockchain_storage_basic.h index 70d2ada9..6d2a1300 100644 --- a/src/currency_core/blockchain_storage_basic.h +++ b/src/currency_core/blockchain_storage_basic.h @@ -6,6 +6,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include #include #include diff --git a/src/currency_core/currency_format_utils.cpp b/src/currency_core/currency_format_utils.cpp index e6ec0d01..f9716bd1 100644 --- a/src/currency_core/currency_format_utils.cpp +++ b/src/currency_core/currency_format_utils.cpp @@ -2426,7 +2426,10 @@ namespace currency return true; return false; } - + size_t get_max_block_size() + { + return CURRENCY_MAX_BLOCK_SIZE; + } //----------------------------------------------------------------------------------------------- uint64_t get_base_block_reward(bool is_pos, const boost::multiprecision::uint128_t& already_generated_coins, uint64_t height) { diff --git a/src/currency_core/tx_semantic_validation.cpp b/src/currency_core/tx_semantic_validation.cpp index db29bce1..91b700b5 100644 --- a/src/currency_core/tx_semantic_validation.cpp +++ b/src/currency_core/tx_semantic_validation.cpp @@ -91,89 +91,4 @@ namespace currency return true; } - //--------------------------------------------------------------- - bool get_transaction_hash(const transaction& t, crypto::hash& res) - { - uint64_t blob_size = 0; - return get_object_hash(static_cast(t), res, blob_size); - } - //--------------------------------------------------------------- - bool get_transaction_hash(const transaction& t, crypto::hash& res, uint64_t& blob_size) - { - blob_size = 0; - bool r = get_object_hash(static_cast(t), res, blob_size); - blob_size = get_object_blobsize(t, blob_size); - return r; - } - //--------------------------------------------------------------- - size_t get_object_blobsize(const transaction& t) - { - size_t tx_blob_size = get_object_blobsize(static_cast(t)); - return get_object_blobsize(t, tx_blob_size); - } - //--------------------------------------------------------------- - size_t get_objects_blobsize(const std::list& ls) - { - size_t total = 0; - for (const auto& tx : ls) - { - total += get_object_blobsize(tx); - } - return total; - } - //--------------------------------------------------------------- - size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size) - { - size_t tx_blob_size = prefix_blob_size; - - if (is_coinbase(t)) - return tx_blob_size; - - // for purged tx, with empty signatures and attachments, this function should return the blob size - // which the tx would have if the signatures and attachments were correctly filled with actual data - - // 1. signatures - bool separately_signed_tx = get_tx_flags(t) & TX_FLAG_SIGNATURE_MODE_SEPARATE; - - tx_blob_size += tools::get_varint_packed_size(t.vin.size()); // size of transaction::signatures (equals to total inputs count) - - for (size_t i = 0; i != t.vin.size(); i++) - { - size_t sig_count = get_input_expected_signatures_count(t.vin[i]); - if (separately_signed_tx && i == t.vin.size() - 1) - ++sig_count; // count in one more signature for the last input in a complete separately signed tx - tx_blob_size += tools::get_varint_packed_size(sig_count); // size of transaction::signatures[i] - tx_blob_size += sizeof(crypto::signature) * sig_count; // size of signatures' data itself - } - - // 2. attachments (try to find extra_attachment_info in tx prefix and count it in if succeed) - extra_attachment_info eai = AUTO_VAL_INIT(eai); - bool got_eai = false; - if (separately_signed_tx) - { - // for separately-signed tx, try to obtain extra_attachment_info from the last input's etc_details - const std::vector* p_etc_details = get_input_etc_details(t.vin.back()); - got_eai = p_etc_details != nullptr && get_type_in_variant_container(*p_etc_details, eai); - } - if (!got_eai) - got_eai = get_type_in_variant_container(t.extra, eai); // then from the extra - - if (got_eai) - tx_blob_size += eai.sz; // sz is a size of whole serialized attachment blob, including attachments vector size - else - tx_blob_size += tools::get_varint_packed_size(static_cast(0)); // no extra_attachment_info found - just add zero vector's size, 'cause it's serialized anyway - - return tx_blob_size; - } - //--------------------------------------------------------------- - blobdata tx_to_blob(const transaction& tx) - { - return t_serializable_object_to_blob(tx); - } - //--------------------------------------------------------------- - bool tx_to_blob(const transaction& tx, blobdata& b_blob) - { - return t_serializable_object_to_blob(tx, b_blob); - } - } \ No newline at end of file diff --git a/src/currency_protocol/currency_protocol_handler.inl b/src/currency_protocol/currency_protocol_handler.inl index c2991eb2..c3c224d2 100644 --- a/src/currency_protocol/currency_protocol_handler.inl +++ b/src/currency_protocol/currency_protocol_handler.inl @@ -305,14 +305,22 @@ namespace currency //now actually process block for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) { - currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, true); - if(tvc.m_verification_failed) + if (tx_blob_it->size() > CURRENCY_MAX_TRANSACTION_BLOB_SIZE) { - LOG_PRINT_L0("Block verification failed: transaction verification failed, dropping connection"); + LOG_ERROR("WRONG TRANSACTION BLOB, too big size " << tx_blob_it->size() << ", rejected"); m_p2p->drop_connection(context); return 1; } + + crypto::hash tx_hash = null_hash; + transaction tx; + if (!parse_and_validate_tx_from_blob(*tx_blob_it, tx, tx_hash)) + { + LOG_ERROR("WRONG TRANSACTION BLOB, Failed to parse, rejected"); + m_p2p->drop_connection(context); + return 1; + } + bvc.m_onboard_transactions[tx_hash] = tx; } m_core.pause_mine(); From da2b26f9fe316574501786836993a96ece705319 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 19 Nov 2019 02:28:15 +0100 Subject: [PATCH 06/21] fixed coretests regarding to new tx handling schema --- src/currency_core/blockchain_storage.cpp | 8 ++++++-- src/currency_core/currency_core.cpp | 1 - src/currency_core/tx_pool.h | 2 +- tests/core_tests/chaingen_main.cpp | 20 +++++++++++++++----- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 6743337b..11515ab3 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -6077,8 +6077,12 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha for (auto tx_id : b.tx_hashes) { std::shared_ptr tx_ptr; - CHECK_AND_ASSERT_MES(get_transaction_from_pool_or_db(tx_id, tx_ptr, split_height), false, "failed to get alt block tx " << tx_id << " with split_height == " << split_height); - transaction& tx = *tx_ptr; + auto it = abei.onboard_transactions.find(tx_id); + if (it == abei.onboard_transactions.end()) + { + CHECK_AND_ASSERT_MES(get_transaction_from_pool_or_db(tx_id, tx_ptr, split_height), false, "failed to get alt block tx " << tx_id << " with split_height == " << split_height); + } + const transaction& tx = it == abei.onboard_transactions.end() ? *tx_ptr : it->second; CHECK_AND_ASSERT_MES(tx.signatures.size() == tx.vin.size(), false, "invalid tx: tx.signatures.size() == " << tx.signatures.size() << ", tx.vin.size() == " << tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) { diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index 77052259..032c19ee 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -503,7 +503,6 @@ namespace currency //----------------------------------------------------------------------------------------------- bool core::handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate) { - bvc = AUTO_VAL_INIT_T(block_verification_context); block b = AUTO_VAL_INIT(b); if (!parse_block(block_blob, b, bvc)) { diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index b3031566..818ae50f 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -159,7 +159,7 @@ namespace currency typedef tools::db::cached_key_value_accessor transactions_container; typedef tools::db::cached_key_value_accessor hash_container; - typedef tools::db::cached_key_value_accessor key_images_container; + //typedef tools::db::cached_key_value_accessor key_images_container; typedef tools::db::cached_key_value_accessor solo_options_container; typedef tools::db::cached_key_value_accessor aliases_container; typedef tools::db::cached_key_value_accessor address_to_aliases_container; diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 761cc26a..3bce8726 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -359,6 +359,7 @@ private: size_t m_ev_index; test_core_listener* m_core_listener; + mutable std::unordered_map m_onboard_txs; bool m_txs_kept_by_block; bool m_skip_txs_blobsize_check; @@ -389,11 +390,19 @@ public: size_t pool_size = m_c.get_pool_transactions_count(); currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_c.handle_incoming_tx(tx_blob, tvc, m_txs_kept_by_block); - bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); - bool r = check_tx_verification_context(tvc, tx_added, m_ev_index, tx, m_validator); - LOCAL_ASSERT(r); - CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed"); + if (m_txs_kept_by_block) + { + m_onboard_txs[get_transaction_hash(tx)] = tx; + } + else + { + m_c.handle_incoming_tx(tx_blob, tvc, m_txs_kept_by_block); + bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); + bool r = check_tx_verification_context(tvc, tx_added, m_ev_index, tx, m_validator); + LOCAL_ASSERT(r); + CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed"); + } + return true; } @@ -403,6 +412,7 @@ public: m_core_listener->before_block_pushed_to_core(b, blob_blk, m_c); currency::block_verification_context bvc = AUTO_VAL_INIT(bvc); + bvc.m_onboard_transactions.swap(m_onboard_txs); m_c.handle_incoming_block(blob_blk, bvc); bool r = check_block_verification_context(bvc, m_ev_index, b, m_validator); CHECK_AND_NO_ASSERT_MES(r, false, "block verification context check failed"); From cf07085e6892a72aefe3bdaa50b2d0f6fa7e3e9a Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Wed, 20 Nov 2019 23:03:27 +0100 Subject: [PATCH 07/21] fixed two coretests --- src/currency_core/blockchain_storage.cpp | 2 +- tests/core_tests/multisig_wallet_tests.cpp | 2 +- tests/core_tests/tx_validation.cpp | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 11515ab3..fcab2571 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -4774,7 +4774,7 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise return 0; } //------------------------------------------------------------------ -bool get_tx_from_cache(const crypto::hash& tx_id, transactions_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t fee) +bool get_tx_from_cache(const crypto::hash& tx_id, transactions_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t& fee) { auto it = tx_cache.find(tx_id); if (it == tx_cache.end()) diff --git a/tests/core_tests/multisig_wallet_tests.cpp b/tests/core_tests/multisig_wallet_tests.cpp index e69cd871..0582d3ab 100644 --- a/tests/core_tests/multisig_wallet_tests.cpp +++ b/tests/core_tests/multisig_wallet_tests.cpp @@ -2361,7 +2361,7 @@ bool multisig_out_make_and_spent_in_altchain::generate(std::vector& events) events.push_back(tx_2); events.push_back(event_visitor_settings(event_visitor_settings::set_txs_kept_by_block, false)); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(3)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // make a block with tx_0 and put tx_0 to the blockchain MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, m_miner_acc, tx_0); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // tx_1 and tx_2 is still in the pool // it can never be added to any block as long as blk_1 is in the blockchain due to key image conflict @@ -1531,7 +1531,7 @@ bool tx_key_image_pool_conflict::generate(std::vector& events) MAKE_NEXT_BLOCK_TX1(events, blk_1a, blk_0r, m_miner_acc, tx_1); // however, it does not remove tx from the pool - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // // make sure stuck tx will be removed from the pool when it's too old @@ -1541,7 +1541,7 @@ bool tx_key_image_pool_conflict::generate(std::vector& events) // remove_stuck_txs should not remove anything, tx_1 and tx_2 should be in the pool DO_CALLBACK(events, "remove_stuck_txs"); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // shift time by CURRENCY_MEMPOOL_TX_LIVETIME events.push_back(event_core_time(CURRENCY_MEMPOOL_TX_LIVETIME + 1, true)); @@ -1562,11 +1562,11 @@ bool tx_key_image_pool_conflict::generate(std::vector& events) events.push_back(tx_2); events.push_back(event_visitor_settings(event_visitor_settings::set_txs_kept_by_block, false)); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // remove_stuck_txs should not remove anything, tx_1 and tx_2 should be in the pool DO_CALLBACK(events, "remove_stuck_txs"); - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // rewind 50 blocks so tx_0 spending its key image will be deep enough REWIND_BLOCKS_N_WITH_TIME(events, blk_3r, blk_3, m_miner_acc, 50); From 4eb23c19541a1c6314b2779721506595f78708fb Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Thu, 21 Nov 2019 00:46:05 +0100 Subject: [PATCH 08/21] fixed gen_double_spend_in_the_same_block --- tests/core_tests/chaingen_main.cpp | 2 +- tests/core_tests/double_spend.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 3bce8726..dc4ec806 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -250,7 +250,7 @@ bool gen_and_play_intermitted_by_blockchain_saveload(const char* const genclass_ #define GENERATE_AND_PLAY(genclass) \ - if(!postponed_tests.count(#genclass) && (run_single_test.empty() || run_single_test == #genclass)) \ + if(!postponed_tests.count(#genclass) && (run_single_test.empty() || std::string::npos != std::string(#genclass).find(run_single_test))) \ { \ TIME_MEASURE_START_MS(t); \ ++tests_count; \ diff --git a/tests/core_tests/double_spend.h b/tests/core_tests/double_spend.h index 3ca9ba9d..23096ce0 100644 --- a/tests/core_tests/double_spend.h +++ b/tests/core_tests/double_spend.h @@ -42,7 +42,7 @@ struct gen_double_spend_in_the_same_block : public gen_double_spend_base< gen_do { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 1 : 2; + static const size_t expected_pool_txs_count = 1; static const uint64_t expected_bob_balance = send_amount; static const uint64_t expected_alice_balance = 0; From c3066938062958fb321255bd42c1174e11bfb8b0 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 22 Nov 2019 00:47:21 +0100 Subject: [PATCH 09/21] all tests fixed --- tests/core_tests/double_spend.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/core_tests/double_spend.h b/tests/core_tests/double_spend.h index 23096ce0..901f1ac5 100644 --- a/tests/core_tests/double_spend.h +++ b/tests/core_tests/double_spend.h @@ -55,7 +55,7 @@ struct gen_double_spend_in_different_blocks : public gen_double_spend_base< gen_ { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 0 : 1; + static const size_t expected_pool_txs_count = 0; static const uint64_t expected_bob_balance = 0; static const uint64_t expected_alice_balance = send_amount - TESTS_DEFAULT_FEE; @@ -68,7 +68,7 @@ struct gen_double_spend_in_alt_chain_in_the_same_block : public gen_double_spend { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 1 : 2; + static const size_t expected_pool_txs_count = txs_kept_by_block ? 0 :1; static const uint64_t expected_bob_balance = send_amount; static const uint64_t expected_alice_balance = 0; @@ -81,14 +81,12 @@ struct gen_double_spend_in_alt_chain_in_different_blocks : public gen_double_spe { static const uint64_t send_amount = MK_TEST_COINS(17); static const bool has_invalid_tx = !txs_kept_by_block; - static const size_t expected_pool_txs_count = has_invalid_tx ? 1 : 2; + static const size_t expected_pool_txs_count = txs_kept_by_block ? 0:1; static const uint64_t expected_bob_balance = send_amount; - static const uint64_t expected_alice_balance = 0; - + static const uint64_t expected_alice_balance = 0; bool generate(std::vector& events) const; }; - class gen_double_spend_in_different_chains : public test_chain_unit_base { public: From cdfd32138739b6166bb6452279a37f2f63c155e8 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Wed, 27 Nov 2019 03:19:10 +0100 Subject: [PATCH 10/21] basic implementation of tx packing(in progress) --- src/wallet/wallet2.cpp | 43 +++++++++++++++++++++++++++++++++++++++++- src/wallet/wallet2.h | 7 ++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index da380010..eb90f017 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2289,6 +2289,17 @@ uint64_t wallet2::balance() const void wallet2::get_transfers(wallet2::transfer_container& incoming_transfers) const { incoming_transfers = m_transfers; +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::generate_packing_transaction_if_needed(transaction& tx) +{ + prepare_free_transfers_cache(0); + auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); + if (it == m_found_free_amounts.end() || it->second.size() < WALLET_POS_MINT_PACKING_SIZE) + return false; + + + } //---------------------------------------------------------------------------------------------------- std::string wallet2::get_transfers_str(bool include_spent /*= true*/, bool include_unspent /*= true*/) const @@ -3424,11 +3435,41 @@ void wallet2::send_escrow_proposal(const bc_services::contract_private_details& print_tx_sent_message(tx, "(from multisig)", fee); } //---------------------------------------------------------------------------------------------------- +bool wallet2::prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) +{ + prepare_free_transfers_cache(fake_outputs_count); + auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); + if (it == m_found_free_amounts.end() || it->second.size() < WALLET_POS_MINT_PACKING_SIZE) + return false; + + uint64_t found_money = 0; + for (auto set_it = it->second.begin(); set_it != it->second.end(); it++) + { + if (is_transfer_ready_to_go(m_transfers[*set_it], fake_outputs_count)) + { + found_money += it->first; + selected_indicies.push_back(*set_it); + WLT_LOG_L2("Selected index: " << *set_it << ", transfer_details: " << ENDL << epee::serialization::store_t_to_json(m_transfers[*set_it])); + } + it->second.erase(it->second.begin()); + if (!it->second.size()) + found_free_amounts.erase(it); + } + + + return prepare_tx_sources(fake_outputs_count, sources, selected_indicies, found_money); + +} +//---------------------------------------------------------------------------------------------------- bool wallet2::prepare_tx_sources(uint64_t needed_money, size_t fake_outputs_count, uint64_t dust_threshold, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) { found_money = select_transfers(needed_money, fake_outputs_count, dust_threshold, selected_indicies); THROW_IF_FALSE_WALLET_EX_MES(found_money >= needed_money, error::not_enough_money, "wallet_dump: " << ENDL << dump_trunsfers(false), found_money, needed_money, 0); - + return prepare_tx_sources(fake_outputs_count, sources, selected_indicies, found_money); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::prepare_tx_sources(size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) +{ typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; typedef currency::tx_source_entry::output_entry tx_output_entry; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index e1fd1d63..bb989655 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -42,6 +42,8 @@ #define WALLET_DEFAULT_TX_SPENDABLE_AGE 10 #define WALLET_POS_MINT_CHECK_HEIGHT_INTERVAL 1 +#define WALLET_POS_MINT_PACKING_SIZE 100 + #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "wallet" ENABLE_CHANNEL_BY_DEFAULT("wallet"); @@ -256,6 +258,7 @@ namespace tools currency::account_public_address crypt_address; uint8_t tx_outs_attr; bool shuffle; + bool perform_packing; }; struct finalize_tx_param @@ -784,7 +787,9 @@ private: void process_genesis_if_needed(const currency::block& genesis); bool build_escrow_proposal(bc_services::contract_private_details& ecrow_details, uint64_t fee, uint64_t unlock_time, currency::tx_service_attachment& att, std::vector& selected_indicies); bool prepare_tx_sources(uint64_t needed_money, size_t fake_outputs_count, uint64_t dust_threshold, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); + bool prepare_tx_sources(size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); bool prepare_tx_sources(crypto::hash multisig_id, std::vector& sources, uint64_t& found_money); + bool prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); uint64_t get_needed_money(uint64_t fee, const std::vector& dsts); void prepare_tx_destinations(uint64_t needed_money, uint64_t found_money, @@ -842,7 +847,7 @@ private: void exception_handler() const; uint64_t get_minimum_allowed_fee_for_contract(const crypto::hash& ms_id); void check_for_free_space_and_throw_if_it_lacks(const std::wstring& path, uint64_t exact_size_needed_if_known = UINT64_MAX); - + bool generate_packing_transaction_if_needed(transaction& tx); currency::account_base m_account; From d7c67b196caf0f1010419a96d1fb26f84b4e2b33 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Thu, 28 Nov 2019 05:28:36 +0100 Subject: [PATCH 11/21] added explicit transactions to blocktemplate --- src/currency_core/currency_core.cpp | 5 ++ src/currency_core/currency_core.h | 3 +- .../currency_format_utils_transactions.h | 2 + src/currency_core/miner.h | 1 + src/rpc/core_rpc_server.cpp | 43 +++++++++----- src/rpc/core_rpc_server_commands_defs.h | 3 +- src/wallet/wallet2.cpp | 58 +++++++++++++++++-- src/wallet/wallet2.h | 11 +++- 8 files changed, 104 insertions(+), 22 deletions(-) diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index 032c19ee..ceb88406 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -333,6 +333,11 @@ namespace currency return m_blockchain_storage.create_block_template(b, adr, stakeholder_address, diffic, height, ex_nonce, pos, pe); } //----------------------------------------------------------------------------------------------- + bool core::get_block_template(const create_block_template_params& params, create_block_template_response& resp) + { + return m_blockchain_storage.create_block_template(params, resp); + } + //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); diff --git a/src/currency_core/currency_core.h b/src/currency_core/currency_core.h index cdadf317..faeec262 100644 --- a/src/currency_core/currency_core.h +++ b/src/currency_core/currency_core.h @@ -53,7 +53,8 @@ namespace currency //-------------------- i_miner_handler ----------------------- virtual bool handle_block_found(const block& b, block_verification_context* p_verification_result = nullptr); - virtual bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()); + virtual bool get_block_template(const create_block_template_params& params, create_block_template_response& resp); + bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()); miner& get_miner(){ return m_miner; } static void init_options(boost::program_options::options_description& desc); diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index 603329e2..3c36798b 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -109,4 +109,6 @@ namespace currency blobdata tx_to_blob(const transaction& b); bool tx_to_blob(const transaction& b, blobdata& b_blob); bool read_keyimages_from_tx(const transaction& tx, std::list& kil); + + } diff --git a/src/currency_core/miner.h b/src/currency_core/miner.h index c1bd83f0..69a4bdc7 100644 --- a/src/currency_core/miner.h +++ b/src/currency_core/miner.h @@ -24,6 +24,7 @@ namespace currency struct i_miner_handler { virtual bool handle_block_found(const block& b, block_verification_context* p_verification_result = nullptr) = 0; + virtual bool get_block_template(const create_block_template_params& params, create_block_template_response& resp) = 0; virtual bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()) = 0; protected: ~i_miner_handler(){}; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 40ef799c..5a6e89b9 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -793,28 +793,45 @@ namespace currency return false; } - block b = AUTO_VAL_INIT(b); - wide_difficulty_type dt = 0; - currency::pos_entry pe = AUTO_VAL_INIT(pe); - pe.amount = req.pos_amount; - pe.index = req.pos_index; - pe.stake_unlock_time = req.stake_unlock_time; - //pe.keyimage key image will be set in the wallet - //pe.wallet_index is not included in serialization map, TODO: refactoring here - if (!m_core.get_block_template(b, miner_address, stakeholder_address, dt, res.height, req.extra_text, req.pos_block, pe)) + create_block_template_params params = AUTO_VAL_INIT(params); + params.miner_address = miner_address; + params.stakeholder_address = stakeholder_address; + params.ex_nonce = req.extra_text; + params.pos = req.pos_block; + params.pe.amount = req.pos_amount; + params.pe.index = req.pos_index; + params.pe.stake_unlock_time = req.stake_unlock_time; + //params.pe.keyimage key image will be set in the wallet + //params.pe.wallet_index is not included in serialization map, TODO: refactoring here + params.pcustom_fill_block_template_func = nullptr; + if (req.explicit_transaction.size()) + { + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(req.explicit_transaction, tx)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Wrong parameters: explicit_transaction is invalid"; + LOG_ERROR("Failed to parse explicit_transaction blob"); + return false; + } + params.explicit_txs.push_back(tx); + } + + create_block_template_response resp = AUTO_VAL_INIT(resp); + if (!m_core.get_block_template(params, resp)) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; LOG_ERROR("Failed to create block template"); return false; } - res.difficulty = dt.convert_to(); - blobdata block_blob = t_serializable_object_to_blob(b); + res.difficulty = resp.diffic.convert_to(); + blobdata block_blob = t_serializable_object_to_blob(resp.b); res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); - res.prev_hash = string_tools::pod_to_hex(b.prev_id); - + res.prev_hash = string_tools::pod_to_hex(resp.b.prev_id); + res.height = resp.height; //calculate epoch seed res.seed = currency::ethash_epoch_to_seed(currency::ethash_height_to_epoch(res.height)); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 4ff21dec..86fd7506 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -770,7 +770,7 @@ namespace currency { struct request { - //uint64_t reserve_size; //max 255 bytes + blobdata explicit_transaction; std::string extra_text; std::string wallet_address; std::string stakeholder_address; @@ -780,6 +780,7 @@ namespace currency uint64_t stake_unlock_time; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(explicit_transaction) KV_SERIALIZE(extra_text) KV_SERIALIZE(wallet_address) KV_SERIALIZE(stakeholder_address); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index eb90f017..f37b1ede 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2291,15 +2291,32 @@ void wallet2::get_transfers(wallet2::transfer_container& incoming_transfers) con incoming_transfers = m_transfers; } //---------------------------------------------------------------------------------------------------- -bool wallet2::generate_packing_transaction_if_needed(transaction& tx) +bool wallet2::generate_packing_transaction_if_needed(transaction& tx, uint64_t fake_outputs_number) { prepare_free_transfers_cache(0); auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); if (it == m_found_free_amounts.end() || it->second.size() < WALLET_POS_MINT_PACKING_SIZE) return false; + //let's check if we have at least WALLET_POS_MINT_PACKING_SIZE transactions which is ready to go + size_t count = 0; + for (auto it_ind = it->second.begin(); it_ind != it->second.end() && count < WALLET_POS_MINT_PACKING_SIZE; it_ind++) + { + if (is_transfer_ready_to_go(m_transfers[*it_ind], fake_outputs_number)) + ++count; + } + if (count < WALLET_POS_MINT_PACKING_SIZE) + return false; + construct_tx_param ctp = get_default_construct_tx_param(); + currency::tx_destination_entry de = AUTO_VAL_INIT(de); + de.addr.push_back(m_account.get_public_address()); + de.amount = WALLET_POS_MINT_PACKING_SIZE; + ctp.dsts.push_back(de); + ctp.perform_packing = true; + + transfer(ctp, tx, false, nullptr); - + return true; } //---------------------------------------------------------------------------------------------------- std::string wallet2::get_transfers_str(bool include_spent /*= true*/, bool include_unspent /*= true*/) const @@ -2769,6 +2786,12 @@ bool wallet2::build_minted_block(const currency::COMMAND_RPC_SCAN_POS::request& tmpl_req.pos_index = req.pos_entries[rsp.index].index; tmpl_req.extra_text = m_miner_text_info; tmpl_req.stake_unlock_time = req.pos_entries[rsp.index].stake_unlock_time; + //generate packing tx + transaction pack_tx = AUTO_VAL_INIT(pack_tx); + if (generate_packing_transaction_if_needed(pack_tx, 0)) + { + tx_to_blob(pack_tx, tmpl_req.explicit_transaction); + } m_core_proxy->call_COMMAND_RPC_GETBLOCKTEMPLATE(tmpl_req, tmpl_rsp); WLT_CHECK_AND_ASSERT_MES(tmpl_rsp.status == CORE_RPC_STATUS_OK, false, "Failed to create block template after kernel hash found!"); @@ -3453,12 +3476,10 @@ bool wallet2::prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake } it->second.erase(it->second.begin()); if (!it->second.size()) - found_free_amounts.erase(it); + m_found_free_amounts.erase(it); } - return prepare_tx_sources(fake_outputs_count, sources, selected_indicies, found_money); - } //---------------------------------------------------------------------------------------------------- bool wallet2::prepare_tx_sources(uint64_t needed_money, size_t fake_outputs_count, uint64_t dust_threshold, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) @@ -4110,6 +4131,8 @@ void wallet2::prepare_transaction(const construct_tx_param& ctp, finalize_tx_par TIME_MEASURE_START_MS(prepare_tx_sources_time); if (ctp.multisig_id == currency::null_hash) prepare_tx_sources(needed_money, ctp.fake_outputs_count, ctp.dust_policy.dust_threshold, ftp.sources, ftp.selected_transfers, found_money); + else if (ctp.perform_packing) + prepare_tx_sources_for_packing(WALLET_POS_MINT_PACKING_SIZE, 0, ftp.sources, ftp.selected_transfers, found_money); else prepare_tx_sources(ctp.multisig_id, ftp.sources, found_money); TIME_MEASURE_FINISH_MS(prepare_tx_sources_time); @@ -4239,7 +4262,32 @@ void wallet2::transfer(const std::vector& dsts, ctp.tx_outs_attr = tx_outs_attr; ctp.unlock_time = unlock_time; TIME_MEASURE_FINISH(precalculation_time); + transfer(ctp, tx, send_to_network, p_signed_tx_blob_str); +} +//---------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------- +construct_tx_param wallet2::get_default_construct_tx_param_inital() +{ + construct_tx_param ctp = AUTO_VAL_INIT(ctp); + ctp.fee = m_core_runtime_config.tx_default_fee; + ctp.dust_policy = tools::tx_dust_policy(DEFAULT_DUST_THRESHOLD); + ctp.split_strategy_id = tools::detail::ssi_digit; + ctp.tx_outs_attr = CURRENCY_TO_KEY_OUT_RELAXED; + ctp.shuffle = 0; + return ctp; +} +const construct_tx_param& wallet2::get_default_construct_tx_param() +{ + static construct_tx_param ctp = get_default_construct_tx_param_inital(); + return ctp; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::transfer(const construct_tx_param& ctp, + currency::transaction &tx, + bool send_to_network, + std::string* p_signed_tx_blob_str) +{ TIME_MEASURE_START(prepare_transaction_time); finalize_tx_param ftp = AUTO_VAL_INIT(ftp); prepare_transaction(ctp, ftp); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index bb989655..43d1b2fc 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -541,6 +541,11 @@ namespace tools const std::vector& attachments, currency::transaction& tx); + void transfer(const construct_tx_param& ctp, + currency::transaction &tx, + bool send_to_network, + std::string* p_signed_tx_blob_str); + template void transfer_from_contract( const std::list& owner_keys, @@ -805,7 +810,9 @@ private: void change_contract_state(wallet_public::escrow_contract_details_basic& contract, uint32_t new_state, const crypto::hash& contract_id, const wallet_public::wallet_transfer_info& wti) const; void change_contract_state(wallet_public::escrow_contract_details_basic& contract, uint32_t new_state, const crypto::hash& contract_id, const std::string& reason = "internal intention") const; - + + construct_tx_param get_default_construct_tx_param_inital(); + const construct_tx_param& get_default_construct_tx_param(); uint64_t get_tx_expiration_median() const; @@ -847,7 +854,7 @@ private: void exception_handler() const; uint64_t get_minimum_allowed_fee_for_contract(const crypto::hash& ms_id); void check_for_free_space_and_throw_if_it_lacks(const std::wstring& path, uint64_t exact_size_needed_if_known = UINT64_MAX); - bool generate_packing_transaction_if_needed(transaction& tx); + bool generate_packing_transaction_if_needed(transaction& tx, uint64_t fake_outputs_number); currency::account_base m_account; From 821776d1982a5833efacd56a664e0c609afc691d Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Thu, 28 Nov 2019 23:34:24 +0100 Subject: [PATCH 12/21] fixed wallet functions signatures --- src/wallet/wallet2.cpp | 9 ++++----- src/wallet/wallet2.h | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f37b1ede..6d8fd766 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2291,7 +2291,7 @@ void wallet2::get_transfers(wallet2::transfer_container& incoming_transfers) con incoming_transfers = m_transfers; } //---------------------------------------------------------------------------------------------------- -bool wallet2::generate_packing_transaction_if_needed(transaction& tx, uint64_t fake_outputs_number) +bool wallet2::generate_packing_transaction_if_needed(currency::transaction& tx, uint64_t fake_outputs_number) { prepare_free_transfers_cache(0); auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); @@ -3465,7 +3465,6 @@ bool wallet2::prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake if (it == m_found_free_amounts.end() || it->second.size() < WALLET_POS_MINT_PACKING_SIZE) return false; - uint64_t found_money = 0; for (auto set_it = it->second.begin(); set_it != it->second.end(); it++) { if (is_transfer_ready_to_go(m_transfers[*set_it], fake_outputs_count)) @@ -4312,7 +4311,7 @@ void wallet2::transfer(const construct_tx_param& ctp, TIME_MEASURE_FINISH(mark_transfers_as_spent_time); WLT_LOG_GREEN("[wallet::transfer]" - << " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) + //<< " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) << ", prepare_transaction_time: " << print_fixed_decimal_point(prepare_transaction_time, 3) << ", store_unsigned_tx_time: " << print_fixed_decimal_point(store_unsigned_tx_time, 3) << ", mark_transfers_as_spent_time: " << print_fixed_decimal_point(mark_transfers_as_spent_time, 3) @@ -4340,13 +4339,13 @@ void wallet2::transfer(const construct_tx_param& ctp, WLT_LOG_GREEN("[wallet::transfer]" - << " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) + //<< " precalculation_time: " << print_fixed_decimal_point(precalculation_time, 3) << ", prepare_transaction_time: " << print_fixed_decimal_point(prepare_transaction_time, 3) << ", finalize_transaction_time: " << print_fixed_decimal_point(finalize_transaction_time, 3) << ", mark_transfers_as_spent_time: " << print_fixed_decimal_point(mark_transfers_as_spent_time, 3) , LOG_LEVEL_0); - print_tx_sent_message(tx, std::string() + "(transfer)", fee); + print_tx_sent_message(tx, std::string() + "(transfer)", ctp.fee); } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 43d1b2fc..3035904a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -854,7 +854,7 @@ private: void exception_handler() const; uint64_t get_minimum_allowed_fee_for_contract(const crypto::hash& ms_id); void check_for_free_space_and_throw_if_it_lacks(const std::wstring& path, uint64_t exact_size_needed_if_known = UINT64_MAX); - bool generate_packing_transaction_if_needed(transaction& tx, uint64_t fake_outputs_number); + bool generate_packing_transaction_if_needed(currency::transaction& tx, uint64_t fake_outputs_number); currency::account_base m_account; From e82bd6b575a6cbe273d92ca1d6c2819ac84eec1c Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 29 Nov 2019 21:43:17 +0100 Subject: [PATCH 13/21] added new test, fixed multiple bugs --- .../include/serialization/keyvalue_helpers.h | 1 + .../serialization/keyvalue_hexemizer.h | 41 ++ src/currency_core/blockchain_storage.cpp | 4 +- src/currency_core/tx_pool.cpp | 1 + .../application/core_fast_rpc_proxy.h | 5 + src/rpc/core_rpc_server.cpp | 57 ++ src/rpc/core_rpc_server.h | 2 + src/rpc/core_rpc_server_commands_defs.h | 24 + src/wallet/core_default_rpc_proxy.cpp | 5 + src/wallet/core_default_rpc_proxy.h | 1 + src/wallet/core_rpc_proxy.h | 1 + src/wallet/wallet2.cpp | 43 +- src/wallet/wallet2.h | 10 +- tests/core_tests/chaingen_main.cpp | 1 + tests/core_tests/wallet_packing_tx.cpp | 571 ++++++++++++++++++ tests/core_tests/wallet_packing_tx.h | 17 + tests/core_tests/wallet_tests.cpp | 76 +++ tests/core_tests/wallet_tests.h | 8 + 18 files changed, 847 insertions(+), 21 deletions(-) create mode 100644 contrib/epee/include/serialization/keyvalue_hexemizer.h create mode 100644 tests/core_tests/wallet_packing_tx.cpp create mode 100644 tests/core_tests/wallet_packing_tx.h diff --git a/contrib/epee/include/serialization/keyvalue_helpers.h b/contrib/epee/include/serialization/keyvalue_helpers.h index 61882816..9d32d024 100644 --- a/contrib/epee/include/serialization/keyvalue_helpers.h +++ b/contrib/epee/include/serialization/keyvalue_helpers.h @@ -51,6 +51,7 @@ namespace epee } }; + //basic helpers for pod-to-hex serialization template std::string transform_t_pod_to_str(const t_pod_type & a) diff --git a/contrib/epee/include/serialization/keyvalue_hexemizer.h b/contrib/epee/include/serialization/keyvalue_hexemizer.h new file mode 100644 index 00000000..47e7e341 --- /dev/null +++ b/contrib/epee/include/serialization/keyvalue_hexemizer.h @@ -0,0 +1,41 @@ +// Copyright (c) 2006-2019, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#pragma once +#include "keyvalue_serialization.h" +namespace epee +{ + + struct hexemizer + { + std::string blob; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(blob) + END_KV_SERIALIZE_MAP() + }; + + +} \ No newline at end of file diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index fcab2571..7ecc4a3a 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -1358,8 +1358,8 @@ bool blockchain_storage::create_block_template(const create_block_template_param CRITICAL_REGION_END(); - size_t txs_size; - uint64_t fee; + size_t txs_size = 0; + uint64_t fee = 0; bool block_filled = false; if (pcustom_fill_block_template_func == nullptr) block_filled = m_tx_pool.fill_block_template(b, pos, median_size, already_generated_coins, txs_size, fee, height, params.explicit_txs); diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index 8a3045b1..09eb40b0 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -1138,6 +1138,7 @@ namespace currency // add explicit transactions for (const auto& tx : explicit_txs) { + fee += get_tx_fee(tx); bl.tx_hashes.push_back(get_transaction_hash(tx)); } return true; diff --git a/src/gui/qt-daemon/application/core_fast_rpc_proxy.h b/src/gui/qt-daemon/application/core_fast_rpc_proxy.h index ca189240..c26c1fb2 100644 --- a/src/gui/qt-daemon/application/core_fast_rpc_proxy.h +++ b/src/gui/qt-daemon/application/core_fast_rpc_proxy.h @@ -109,6 +109,11 @@ namespace tools return m_rpc.on_submitblock(req, rsp, m_err_stub, m_cntxt_stub); } //------------------------------------------------------------------------------------------------------------------------------ + bool call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) override + { + return m_rpc.on_submitblock2(req, rsp, m_err_stub, m_cntxt_stub); + } + //------------------------------------------------------------------------------------------------------------------------------ bool call_COMMAND_RPC_GET_POS_MINING_DETAILS(const currency::COMMAND_RPC_GET_POS_MINING_DETAILS::request& req, currency::COMMAND_RPC_GET_POS_MINING_DETAILS::response& rsp) override { return m_rpc.on_get_pos_mining_details(req, rsp, m_cntxt_stub); diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 5a6e89b9..28706e83 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -890,6 +890,63 @@ namespace currency // + res.status = "OK"; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_submitblock2(const COMMAND_RPC_SUBMITBLOCK2::request& req, COMMAND_RPC_SUBMITBLOCK2::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + { + CHECK_CORE_READY(); + + + block b = AUTO_VAL_INIT(b); + if (!parse_and_validate_block_from_blob(req.b, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + + block_verification_context bvc = AUTO_VAL_INIT(bvc); + for (const auto& txblob : req.explicit_txs) + { + + crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash); + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(txblob.blob, tx, tx_hash)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong explicit tx blob"; + return false; + } + bvc.m_onboard_transactions[tx_hash] = tx; + } + + + if (!m_core.handle_block_found(b, &bvc)) + { + if (bvc.m_added_to_altchain) + { + error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_ADDED_AS_ALTERNATIVE; + error_resp.message = "Block added as alternative"; + return false; + } + error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_NOT_ACCEPTED; + error_resp.message = "Block not accepted"; + return false; + } + //@#@ + //temporary double check timestamp + if (time(NULL) - get_actual_timestamp(b) > 5) + { + LOG_PRINT_RED_L0("Found block (" << get_block_hash(b) << ") timestamp (" << get_actual_timestamp(b) + << ") is suspiciously less (" << time(NULL) - get_actual_timestamp(b) << ") then curren time( " << time(NULL) << ")"); + //mark node to make it easier to find it via scanner + m_core.get_blockchain_storage().get_performnce_data().epic_failure_happend = true; + } + // + + res.status = "OK"; return true; } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 287d16d2..03851812 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -63,6 +63,7 @@ namespace currency bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); + bool on_submitblock2(const COMMAND_RPC_SUBMITBLOCK2::request& req, COMMAND_RPC_SUBMITBLOCK2::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); @@ -125,6 +126,7 @@ namespace currency MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) + MAP_JON_RPC_WE("submitblock2", on_submitblock2, COMMAND_RPC_SUBMITBLOCK2) MAP_JON_RPC_WE("getlastblockheader", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("getblockheaderbyhash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) MAP_JON_RPC_WE("getblockheaderbyheight", on_get_block_header_by_height, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 86fd7506..c0cb4990 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -5,6 +5,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include "serialization/keyvalue_hexemizer.h" #include "currency_protocol/currency_protocol_defs.h" #include "currency_core/currency_basic.h" #include "currency_core/difficulty.h" @@ -825,6 +826,29 @@ namespace currency }; }; + struct COMMAND_RPC_SUBMITBLOCK2 + { + struct request + { + std::string b; //hex encoded block blob + std::list explicit_txs; //hex encoded tx blobs + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(b) + KV_SERIALIZE(explicit_txs) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + struct block_header_response { uint8_t major_version; diff --git a/src/wallet/core_default_rpc_proxy.cpp b/src/wallet/core_default_rpc_proxy.cpp index 6140270d..ae6123bd 100644 --- a/src/wallet/core_default_rpc_proxy.cpp +++ b/src/wallet/core_default_rpc_proxy.cpp @@ -116,6 +116,11 @@ namespace tools return invoke_http_json_rpc_update_is_disconnect("submitblock", req, rsp); } //------------------------------------------------------------------------------------------------------------------------------ + bool default_http_core_proxy::call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) + { + return invoke_http_json_rpc_update_is_disconnect("submitblock2", req, rsp); + } + //------------------------------------------------------------------------------------------------------------------------------ bool default_http_core_proxy::check_connection() { CRITICAL_REGION_LOCAL(m_lock); diff --git a/src/wallet/core_default_rpc_proxy.h b/src/wallet/core_default_rpc_proxy.h index ec43ab5f..b514adbc 100644 --- a/src/wallet/core_default_rpc_proxy.h +++ b/src/wallet/core_default_rpc_proxy.h @@ -39,6 +39,7 @@ namespace tools bool call_COMMAND_RPC_SCAN_POS(const currency::COMMAND_RPC_SCAN_POS::request& req, currency::COMMAND_RPC_SCAN_POS::response& rsp) override; bool call_COMMAND_RPC_GETBLOCKTEMPLATE(const currency::COMMAND_RPC_GETBLOCKTEMPLATE::request& req, currency::COMMAND_RPC_GETBLOCKTEMPLATE::response& rsp) override; bool call_COMMAND_RPC_SUBMITBLOCK(const currency::COMMAND_RPC_SUBMITBLOCK::request& req, currency::COMMAND_RPC_SUBMITBLOCK::response& rsp) override; + bool call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) override; bool call_COMMAND_RPC_GET_POS_MINING_DETAILS(const currency::COMMAND_RPC_GET_POS_MINING_DETAILS::request& req, currency::COMMAND_RPC_GET_POS_MINING_DETAILS::response& rsp) override; bool call_COMMAND_RPC_GET_BLOCKS_DETAILS(const currency::COMMAND_RPC_GET_BLOCKS_DETAILS::request& req, currency::COMMAND_RPC_GET_BLOCKS_DETAILS::response& res) override; bool call_COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN(const currency::COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN::request& req, currency::COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN::response& res) override; diff --git a/src/wallet/core_rpc_proxy.h b/src/wallet/core_rpc_proxy.h index 6688e75a..828bd509 100644 --- a/src/wallet/core_rpc_proxy.h +++ b/src/wallet/core_rpc_proxy.h @@ -36,6 +36,7 @@ namespace tools virtual bool call_COMMAND_RPC_SCAN_POS(const currency::COMMAND_RPC_SCAN_POS::request& req, currency::COMMAND_RPC_SCAN_POS::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GETBLOCKTEMPLATE(const currency::COMMAND_RPC_GETBLOCKTEMPLATE::request& req, currency::COMMAND_RPC_GETBLOCKTEMPLATE::response& rsp){ return false; } virtual bool call_COMMAND_RPC_SUBMITBLOCK(const currency::COMMAND_RPC_SUBMITBLOCK::request& req, currency::COMMAND_RPC_SUBMITBLOCK::response& rsp){ return false; } + virtual bool call_COMMAND_RPC_SUBMITBLOCK2(const currency::COMMAND_RPC_SUBMITBLOCK2::request& req, currency::COMMAND_RPC_SUBMITBLOCK2::response& rsp) { return false; } virtual bool call_COMMAND_RPC_GET_POS_MINING_DETAILS(const currency::COMMAND_RPC_GET_POS_MINING_DETAILS::request& req, currency::COMMAND_RPC_GET_POS_MINING_DETAILS::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_BLOCKS_DETAILS(const currency::COMMAND_RPC_GET_BLOCKS_DETAILS::request& req, currency::COMMAND_RPC_GET_BLOCKS_DETAILS::response& res){ return false; } virtual bool call_COMMAND_RPC_GET_OFFERS_EX(const currency::COMMAND_RPC_GET_OFFERS_EX::request& req, currency::COMMAND_RPC_GET_OFFERS_EX::response& res){ return false; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 6d8fd766..cd17eb08 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -147,6 +147,11 @@ bool wallet2::set_core_proxy(const std::shared_ptr& proxy) return true; } //---------------------------------------------------------------------------------------------------- +void wallet2::set_pos_mint_packing_size(uint64_t new_size) +{ + m_pos_mint_packing_size = new_size; +} +//---------------------------------------------------------------------------------------------------- std::shared_ptr wallet2::get_core_proxy() { return m_core_proxy; @@ -2295,22 +2300,22 @@ bool wallet2::generate_packing_transaction_if_needed(currency::transaction& tx, { prepare_free_transfers_cache(0); auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); - if (it == m_found_free_amounts.end() || it->second.size() < WALLET_POS_MINT_PACKING_SIZE) + if (it == m_found_free_amounts.end() || it->second.size() < m_pos_mint_packing_size) return false; //let's check if we have at least WALLET_POS_MINT_PACKING_SIZE transactions which is ready to go size_t count = 0; - for (auto it_ind = it->second.begin(); it_ind != it->second.end() && count < WALLET_POS_MINT_PACKING_SIZE; it_ind++) + for (auto it_ind = it->second.begin(); it_ind != it->second.end() && count < m_pos_mint_packing_size; it_ind++) { if (is_transfer_ready_to_go(m_transfers[*it_ind], fake_outputs_number)) ++count; } - if (count < WALLET_POS_MINT_PACKING_SIZE) + if (count < m_pos_mint_packing_size) return false; construct_tx_param ctp = get_default_construct_tx_param(); currency::tx_destination_entry de = AUTO_VAL_INIT(de); de.addr.push_back(m_account.get_public_address()); - de.amount = WALLET_POS_MINT_PACKING_SIZE; + de.amount = m_pos_mint_packing_size*CURRENCY_BLOCK_REWARD; ctp.dsts.push_back(de); ctp.perform_packing = true; @@ -2835,10 +2840,13 @@ bool wallet2::build_minted_block(const currency::COMMAND_RPC_SCAN_POS::request& WLT_LOG_GREEN("Block constructed <" << get_block_hash(b) << ">, sending to core...", LOG_LEVEL_0); - currency::COMMAND_RPC_SUBMITBLOCK::request subm_req = AUTO_VAL_INIT(subm_req); - currency::COMMAND_RPC_SUBMITBLOCK::response subm_rsp = AUTO_VAL_INIT(subm_rsp); - subm_req.push_back(epee::string_tools::buff_to_hex_nodelimer(t_serializable_object_to_blob(b))); - m_core_proxy->call_COMMAND_RPC_SUBMITBLOCK(subm_req, subm_rsp); + currency::COMMAND_RPC_SUBMITBLOCK2::request subm_req = AUTO_VAL_INIT(subm_req); + currency::COMMAND_RPC_SUBMITBLOCK2::response subm_rsp = AUTO_VAL_INIT(subm_rsp); + subm_req.b = t_serializable_object_to_blob(b); + if (tmpl_req.explicit_transaction.size()) + subm_req.explicit_txs.push_back(hexemizer{ tmpl_req.explicit_transaction }); + + m_core_proxy->call_COMMAND_RPC_SUBMITBLOCK2(subm_req, subm_rsp); if (subm_rsp.status != CORE_RPC_STATUS_OK) { WLT_LOG_ERROR("Constructed block is not accepted by core, status: " << subm_rsp.status); @@ -3462,21 +3470,24 @@ bool wallet2::prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake { prepare_free_transfers_cache(fake_outputs_count); auto it = m_found_free_amounts.find(CURRENCY_BLOCK_REWARD); - if (it == m_found_free_amounts.end() || it->second.size() < WALLET_POS_MINT_PACKING_SIZE) + if (it == m_found_free_amounts.end() || it->second.size() < m_pos_mint_packing_size) return false; - for (auto set_it = it->second.begin(); set_it != it->second.end(); it++) + for (auto set_it = it->second.begin(); set_it != it->second.end() && selected_indicies.size() <= m_pos_mint_packing_size; ) { if (is_transfer_ready_to_go(m_transfers[*set_it], fake_outputs_count)) { found_money += it->first; selected_indicies.push_back(*set_it); WLT_LOG_L2("Selected index: " << *set_it << ", transfer_details: " << ENDL << epee::serialization::store_t_to_json(m_transfers[*set_it])); + + it->second.erase(set_it++); } - it->second.erase(it->second.begin()); - if (!it->second.size()) - m_found_free_amounts.erase(it); + else + set_it++; } + if (!it->second.size()) + m_found_free_amounts.erase(it); return prepare_tx_sources(fake_outputs_count, sources, selected_indicies, found_money); } @@ -4128,10 +4139,10 @@ void wallet2::prepare_transaction(const construct_tx_param& ctp, finalize_tx_par uint64_t found_money = 0; TIME_MEASURE_START_MS(prepare_tx_sources_time); - if (ctp.multisig_id == currency::null_hash) + if (ctp.perform_packing) + prepare_tx_sources_for_packing(WALLET_DEFAULT_POS_MINT_PACKING_SIZE, 0, ftp.sources, ftp.selected_transfers, found_money); + else if (ctp.multisig_id == currency::null_hash) prepare_tx_sources(needed_money, ctp.fake_outputs_count, ctp.dust_policy.dust_threshold, ftp.sources, ftp.selected_transfers, found_money); - else if (ctp.perform_packing) - prepare_tx_sources_for_packing(WALLET_POS_MINT_PACKING_SIZE, 0, ftp.sources, ftp.selected_transfers, found_money); else prepare_tx_sources(ctp.multisig_id, ftp.sources, found_money); TIME_MEASURE_FINISH_MS(prepare_tx_sources_time); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 3035904a..74c5e00f 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -42,7 +42,7 @@ #define WALLET_DEFAULT_TX_SPENDABLE_AGE 10 #define WALLET_POS_MINT_CHECK_HEIGHT_INTERVAL 1 -#define WALLET_POS_MINT_PACKING_SIZE 100 +#define WALLET_DEFAULT_POS_MINT_PACKING_SIZE 100 #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "wallet" @@ -316,7 +316,8 @@ namespace tools m_last_sync_percent(0), m_do_rise_transfer(false), m_watch_only(false), - m_last_pow_block_h(0) + m_last_pow_block_h(0), + m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE) {}; public: wallet2() : m_stop(false), @@ -329,7 +330,8 @@ namespace tools m_do_rise_transfer(false), m_log_prefix("???"), m_watch_only(false), - m_last_pow_block_h(0) + m_last_pow_block_h(0), + m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE) { m_core_runtime_config = currency::get_default_core_runtime_config(); }; @@ -494,6 +496,7 @@ namespace tools bool set_core_proxy(const std::shared_ptr& proxy); + void set_pos_mint_packing_size(uint64_t new_size); std::shared_ptr get_core_proxy(); uint64_t balance() const; uint64_t balance(uint64_t& unloked, uint64_t& awaiting_in, uint64_t& awaiting_out, uint64_t& mined) const; @@ -867,6 +870,7 @@ private: std::atomic m_local_bc_height; //temporary workaround std::atomic m_last_bc_timestamp; bool m_do_rise_transfer; + uint64_t m_pos_mint_packing_size; transfer_container m_transfers; multisig_transfer_container m_multisig_transfers; diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index dc4ec806..27bb39f9 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -856,6 +856,7 @@ int main(int argc, char* argv[]) GENERATE_AND_PLAY(wallet_outputs_with_same_key_image); GENERATE_AND_PLAY(wallet_unconfirmed_tx_expiration); GENERATE_AND_PLAY(wallet_unconfimed_tx_balance); + GENERATE_AND_PLAY(packing_outputs_on_pos_minting_wallet); GENERATE_AND_PLAY(wallet_rpc_integrated_address); GENERATE_AND_PLAY(wallet_rpc_integrated_address_transfer); diff --git a/tests/core_tests/wallet_packing_tx.cpp b/tests/core_tests/wallet_packing_tx.cpp new file mode 100644 index 00000000..031b56cf --- /dev/null +++ b/tests/core_tests/wallet_packing_tx.cpp @@ -0,0 +1,571 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "chaingen.h" +#include "wallet_tests.h" +#include "wallet_test_core_proxy.h" +#include "../../src/wallet/wallet_public_structs_defs.h" +#include "offers_helper.h" +#include "string_coding.h" +#include "random_helper.h" +#include "tx_builder.h" + +using namespace epee; +using namespace crypto; +using namespace currency; + +const uint64_t uint64_max = std::numeric_limits::max(); +const std::wstring g_wallet_filename = L"~coretests.wallet.file.tmp"; +const std::string g_wallet_password = "dofatibmzibeziyekigo"; +const currency::account_base null_account = AUTO_VAL_INIT(null_account); + + +POD_MAKE_COMPARABLE(currency, tx_out); + +// Determines which output is real and actually spent in tx inputs, when there are fake outputs. +bool determine_tx_real_inputs(currency::core& c, const currency::transaction& tx, const currency::account_keys& keys, std::vector& real_inputs) +{ + struct local_visitor + { + local_visitor(const currency::account_keys& keys, const crypto::key_image key_image) + : m_keys(keys) + , m_txin_key_image(key_image) + , m_output_in_input_index(0) + , m_found(false) + {} + + bool handle_output(const transaction& source_tx, const transaction& validated_tx, const tx_out& out, uint64_t out_i) + { + CHECK_AND_ASSERT_MES(!m_found, false, "Internal error: m_found is true but the visitor is still being applied"); + auto it = std::find(validated_tx.vout.begin(), validated_tx.vout.end(), out); + if (it == validated_tx.vout.end()) + return false; + size_t output_tx_index = it - validated_tx.vout.begin(); + + crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(validated_tx); + crypto::key_derivation derivation; + bool r = generate_key_derivation(tx_pub_key, m_keys.m_view_secret_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "generate_key_derivation failed"); + crypto::secret_key ephemeral_secret_key; + derive_secret_key(derivation, output_tx_index, m_keys.m_spend_secret_key, ephemeral_secret_key); + + crypto::public_key output_public_key = boost::get(out.target).key; + + /*crypto::public_key ephemeral_public_key; + derive_public_key(derivation, output_tx_index, m_keys.m_account_address.m_spend_public_key, ephemeral_public_key);*/ + + crypto::key_image ki; + generate_key_image(output_public_key, ephemeral_secret_key, ki); + + if (ki == m_txin_key_image) + { + m_found = true; + return false; // to break the loop in scan_outputkeys_for_indexes + } + + ++m_output_in_input_index; + return true; + } + + currency::account_keys m_keys; + crypto::key_image m_txin_key_image; + size_t m_output_in_input_index; + bool m_found; + }; + + for (auto& txin : tx.vin) + { + const txin_to_key& in = boost::get(txin); + if (in.key_offsets.size() == 1) + { + real_inputs.push_back(0); // trivial case when no mixin is used + continue; + } + local_visitor vis(keys, in.k_image); + bool r = c.get_blockchain_storage().scan_outputkeys_for_indexes(tx, in, vis); + CHECK_AND_ASSERT_MES(r || vis.m_found, false, "scan_outputkeys_for_indexes failed"); + if (!vis.m_found) + return false; + real_inputs.push_back(vis.m_output_in_input_index); + } + + return true; +} + +//------------------------------------------------------------------------------ + +//------------------------------------------------------------------------------ + +mined_balance_wallet_test::mined_balance_wallet_test() +{ + REGISTER_CALLBACK_METHOD(mined_balance_wallet_test, c1); + REGISTER_CALLBACK_METHOD(mined_balance_wallet_test, set_core_config); +} + +bool mined_balance_wallet_test::generate(std::vector& events) const +{ + GENERATE_ACCOUNT(preminer_acc); + GENERATE_ACCOUNT(miner_acc); + m_accounts.push_back(miner_acc); + GENERATE_ACCOUNT(alice_acc); + m_accounts.push_back(alice_acc); + + block blk_0 = AUTO_VAL_INIT(blk_0); + generator.construct_genesis_block(blk_0, preminer_acc, test_core_time::get_time()); + events.push_back(blk_0); + + DO_CALLBACK(events, "set_core_config"); + DO_CALLBACK(events, "c1"); + + return true; +} + +bool mined_balance_wallet_test::set_core_config(currency::core& c, size_t ev_index, const std::vector& events) +{ + core_runtime_config crc = c.get_blockchain_storage().get_core_runtime_config(); + crc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; + crc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; + c.get_blockchain_storage().set_core_runtime_config(crc); + return true; +} + +bool mined_balance_wallet_test::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + blockchain_storage& bcs = c.get_blockchain_storage(); + + core_runtime_config crc = bcs.get_core_runtime_config(); + crc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; + bcs.set_core_runtime_config(crc); + + std::shared_ptr miner_wlt = init_playtime_test_wallet(events, c, MINER_ACC_IDX); + std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); + + CHECK_AND_ASSERT_MES(check_balance_via_wallet(*miner_wlt.get(), "miner", 0), false, "wrong balance"); + CHECK_AND_ASSERT_MES(check_balance_via_wallet(*alice_wlt.get(), "alice", 0), false, "wrong balance"); + + uint64_t miner_mined_money = 0; + bool r = false; + std::list blocks; + + size_t n = CURRENCY_MINED_MONEY_UNLOCK_WINDOW; + r = mine_next_pow_blocks_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c, n); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_blocks_in_playtime failed"); + r = bcs.get_blocks(bcs.get_current_blockchain_size() - n, n, blocks); + CHECK_AND_ASSERT_MES(r, false, "get_blocks failed"); + + for (auto& b : blocks) + miner_mined_money += get_outs_money_amount(b.miner_tx); + + miner_wlt->refresh(); + CHECK_AND_ASSERT_MES(check_balance_via_wallet(*miner_wlt.get(), "miner", miner_mined_money, miner_mined_money), false, "wrong balance"); + + n = bcs.get_current_blockchain_size(); + r = miner_wlt->try_mint_pos(); + CHECK_AND_ASSERT_MES(r && bcs.get_current_blockchain_size() > n, false, "can't mint a PoS block"); + + block b = AUTO_VAL_INIT(b); + r = bcs.get_top_block(b); + CHECK_AND_ASSERT_MES(r, false, "get_top_block failed"); + CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 2, false, "Invalid PoS coinbase tx"); + + uint64_t coinbase_outs_amount = get_outs_money_amount(b.miner_tx); + uint64_t stake_amount = boost::get(b.miner_tx.vin[1]).amount; + CHECK_AND_ASSERT_MES(coinbase_outs_amount > stake_amount, false, "coinbase_outs_amount = " << coinbase_outs_amount << ", stake_amount = " << stake_amount << " : invalid condition"); + + miner_mined_money += coinbase_outs_amount - stake_amount; + + miner_wlt->refresh(); + + std::stringstream ss; + miner_wlt->dump_trunsfers(ss, false); + LOG_PRINT_CYAN("miner transfers: " << ENDL << ss.str(), LOG_LEVEL_0); + + CHECK_AND_ASSERT_MES(check_balance_via_wallet(*miner_wlt.get(), "miner", miner_mined_money, miner_mined_money), false, "wrong balance"); + + return true; +} + +//------------------------------------------------------------------------------ + +wallet_outputs_with_same_key_image::wallet_outputs_with_same_key_image() +{ + REGISTER_CALLBACK_METHOD(wallet_outputs_with_same_key_image, c1); +} + +bool wallet_outputs_with_same_key_image::generate(std::vector& events) const +{ + // Test idea: make sure wallet does not take into account valid outputs having the same key image + // Only one such output is spendable thus only one output should be taken into account. + + bool r = false; + m_accounts.resize(TOTAL_ACCS_COUNT); + account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); + account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); + + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 2); + + uint64_t tx_amount = MK_TEST_COINS(3); + + // tx_1 + std::vector sources_1; + r = fill_tx_sources(sources_1, events, blk_0r, miner_acc.get_keys(), tx_amount + TESTS_DEFAULT_FEE, 0); + CHECK_AND_ASSERT_MES(r, false, "fill_tx_sources failed"); + + std::vector destinations{ tx_destination_entry(tx_amount, alice_acc.get_public_address()) }; + + tx_builder builder; + builder.step1_init(); + builder.step2_fill_inputs(miner_acc.get_keys(), sources_1); + builder.step3_fill_outputs(destinations); + builder.step4_calc_hash(); + builder.step5_sign(sources_1); + + transaction tx_1 = builder.m_tx; + events.push_back(tx_1); + + // tx_2 with the same secret key + currency::keypair tmp_sec_key = builder.m_tx_key; + builder.step1_init(); + builder.m_tx_key = tmp_sec_key; + builder.m_tx.extra.clear(); + add_tx_pub_key_to_extra(builder.m_tx, builder.m_tx_key.pub); + + std::vector sources_2; + r = fill_tx_sources(sources_2, events, blk_0r, miner_acc.get_keys(), tx_amount + TESTS_DEFAULT_FEE, 0, sources_1); + CHECK_AND_ASSERT_MES(r, false, "fill_tx_sources failed"); + + // keep destinations the same + + builder.step2_fill_inputs(miner_acc.get_keys(), sources_2); + builder.step3_fill_outputs(destinations); + builder.step4_calc_hash(); + builder.step5_sign(sources_2); + + transaction tx_2 = builder.m_tx; + events.push_back(tx_2); + + // make sure tx_1 and tx_2 have been created with the same tx key + CHECK_AND_ASSERT_MES(get_tx_pub_key_from_extra(tx_1) == get_tx_pub_key_from_extra(tx_2), false, "internal error: tx_1 and tx_2 have different pub keys"); + + // now both txs are in the pool, make sure they are + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); + + DO_CALLBACK(events, "c1"); + + return true; +} + +bool wallet_outputs_with_same_key_image::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, m_accounts[ALICE_ACC_IDX]); + + // check Alice has no unlocked coins + bool r = refresh_wallet_and_check_balance("before tx_1 and tx_2 added", "Alice", alice_wlt, MK_TEST_COINS(3) * 2, true, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 2, 0); + CHECK_AND_ASSERT_MES(r, false, ""); + + r = mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime failed"); + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "there are txs in the pool!"); + + r = mine_next_pow_blocks_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_blocks_in_playtime failed"); + + // only one tx_1 output is counted as the tx_2 output has the very same key image + r = refresh_wallet_and_check_balance("after tx_1 and tx_2 added", "Alice", alice_wlt, MK_TEST_COINS(3), true, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 1, MK_TEST_COINS(3)); + CHECK_AND_ASSERT_MES(r, false, ""); + + // make sure Alice is able to transfer her coins to smbd + std::vector destinations{ tx_destination_entry(MK_TEST_COINS(3) - TESTS_DEFAULT_FEE, m_accounts[MINER_ACC_IDX].get_public_address()) }; + try + { + alice_wlt->transfer(destinations, 0, 0, TESTS_DEFAULT_FEE, empty_extra, empty_attachment); + } + catch (...) + { + CHECK_AND_ASSERT_MES(false, false, "Alice failed to transfer all her funds"); + } + + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "wrong tx count in the pool: " << c.get_pool_transactions_count()); + r = mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime failed"); + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "there are txs in the pool!"); + + return true; +} + +//------------------------------------------------------------------------------ + +wallet_unconfirmed_tx_expiration::wallet_unconfirmed_tx_expiration() +{ + REGISTER_CALLBACK_METHOD(wallet_unconfirmed_tx_expiration, c1); +} + +bool wallet_unconfirmed_tx_expiration::generate(std::vector& events) const +{ + // Test outline: + // 1. Alice sends tx with expiration. + // 2. Miner ignores Alice's tx, so tx expires in the pool. + // 3. Tx is being removed from the pool due to expiration. + // Make sure Alice eventually spent no coins and all her money is unlocked in the wallet. + + m_accounts.resize(TOTAL_ACCS_COUNT); + account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); + account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); + account_base& bob_acc = m_accounts[BOB_ACC_IDX]; bob_acc.generate(); + + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 1); + + bool r = false; + transaction tx_0 = AUTO_VAL_INIT(tx_0); + r = construct_tx_with_many_outputs(events, blk_0r, miner_acc.get_keys(), alice_acc.get_public_address(), TESTS_DEFAULT_FEE * 20, 10, TESTS_DEFAULT_FEE, tx_0); + CHECK_AND_ASSERT_MES(r, false, "construct_tx_with_many_outputs"); + events.push_back(tx_0); + transaction tx_1 = AUTO_VAL_INIT(tx_1); + r = construct_tx_with_many_outputs(events, blk_0r, miner_acc.get_keys(), bob_acc.get_public_address(), TESTS_DEFAULT_FEE * 20, 10, TESTS_DEFAULT_FEE, tx_1); + CHECK_AND_ASSERT_MES(r, false, "construct_tx_with_many_outputs"); + events.push_back(tx_1); + MAKE_NEXT_BLOCK_TX_LIST(events, blk_1, blk_0r, miner_acc, std::list({ tx_0, tx_1 })); + + REWIND_BLOCKS_N(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); + + DO_CALLBACK(events, "c1"); + + return true; +} + +bool wallet_unconfirmed_tx_expiration::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + bool r = false; + std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); + + uint64_t alice_start_balance = TESTS_DEFAULT_FEE * 20; + CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, alice_start_balance), false, ""); + + // Alice constructs and sends tx with expiration time + uint64_t expiration_time = c.get_blockchain_storage().get_tx_expiration_median() + TX_EXPIRATION_MEDIAN_SHIFT + 15; + etc_tx_details_expiration_time extra_entry = AUTO_VAL_INIT(extra_entry); + extra_entry.v = expiration_time; + std::vector extra({ extra_entry }); // extra with expiration time + std::vector destinations({ tx_destination_entry(TESTS_DEFAULT_FEE * 2, m_accounts[MINER_ACC_IDX].get_public_address()) }); + transaction tx = AUTO_VAL_INIT(tx); + try + { + alice_wlt->transfer(destinations, 0, 0, TESTS_DEFAULT_FEE, extra, empty_attachment, tools::detail::ssi_digit, tools::tx_dust_policy(DEFAULT_DUST_THRESHOLD), tx); + } + catch (std::exception &e) + { + CHECK_AND_ASSERT_MES(false, false, "alice_wlt->transfer() caused an exception: " << e.what()); + } + + CHECK_AND_ASSERT_MES(get_tx_expiration_time(tx) == expiration_time, false, "tx expiration time wasn't set"); + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Invalid txs count in the pool: " << c.get_pool_transactions_count()); + + CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("tx sent,", "Alice", alice_wlt, alice_start_balance - TESTS_DEFAULT_FEE * 2 - TESTS_DEFAULT_FEE, true, 0, UINT64_MAX, 0, 0, TESTS_DEFAULT_FEE * 2), false, ""); + + // mine a few block with no tx, so Alice's tx is expired in the pool + for (size_t i = 0; i < 5; ++i) + { + r = mine_next_pow_block_in_playtime_with_given_txs(m_accounts[MINER_ACC_IDX].get_public_address(), c, std::vector()); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime_with_given_txs failed"); + } + + // tx is still there + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Invalid txs count in the pool: " << c.get_pool_transactions_count()); + + // make sure expiration median was shifted enough + CHECK_AND_ASSERT_MES(c.get_blockchain_storage().is_tx_expired(tx), false, "wrong expiration time condition"); + + LOG_PRINT_CYAN("%%%%% tx_pool::on_idle()", LOG_LEVEL_0); + c.get_tx_pool().on_idle(); + + // make sure tx was removed by the pool + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "Invalid txs count in the pool: " << c.get_pool_transactions_count()); + + // mine one more block to trigger wallet's on_idle() and outdated tx clearing + r = mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime failed"); + + // make sure all Alice's money are unlocked and no coins were actually spent + CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("tx expired and removed from the pool,", "Alice", alice_wlt, alice_start_balance, true, 6, alice_start_balance, 0, 0, 0), false, ""); + + return true; +} + +//------------------------------------------------------------------------------ + +wallet_chain_switch_with_spending_the_same_ki::wallet_chain_switch_with_spending_the_same_ki() +{ + REGISTER_CALLBACK_METHOD(wallet_chain_switch_with_spending_the_same_ki, c1); +} + +bool wallet_chain_switch_with_spending_the_same_ki::generate(std::vector& events) const +{ + // Test outline + // 1. A wallet has one unspent output + // 2. wallet2::transfer() creates tx_0 that spends wallet's output + // 3. tx_0 is successfully put into the blockchain + // 4. Due to chain switch tx_0 is removed from the blockchain and get into the transaction pool + // 5. Make sure the wallet can't spend that output + // 6. After tx is expired make sure the wallet can spend that output + + + m_accounts.resize(TOTAL_ACCS_COUNT); + account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); + account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); + account_base& bob_acc = m_accounts[BOB_ACC_IDX]; bob_acc.generate(); + + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + + MAKE_TX(events, tx_0, miner_acc, alice_acc, MK_TEST_COINS(30), blk_0r); + MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_acc, tx_0); + + // rewind blocks to allow wallet be able to spend the coins + REWIND_BLOCKS_N_WITH_TIME(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); + + DO_CALLBACK(events, "c1"); + + return true; +} + +bool wallet_chain_switch_with_spending_the_same_ki::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + bool r = false; + std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); + + CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(30), true, UINT64_MAX, MK_TEST_COINS(30)), false, ""); + + std::vector destinations { tx_destination_entry(MK_TEST_COINS(30) - TESTS_DEFAULT_FEE, m_accounts[BOB_ACC_IDX].get_public_address()) }; + try + { + // create tx_1 + alice_wlt->transfer(destinations, 0, 0, TESTS_DEFAULT_FEE, empty_extra, empty_attachment); + } + catch (std::exception &e) + { + CHECK_AND_ASSERT_MES(false, false, "alice_wlt->transfer() caused an exception: " << e.what()); + } + + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); + + // mine blk_2 on height 22 + CHECK_AND_ASSERT_MES(mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c), false, ""); + + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); + + // refresh wallet + alice_wlt->refresh(); + // DO NOT scan_tx_pool here intentionally + CHECK_AND_ASSERT_MES(check_balance_via_wallet(*alice_wlt, "Alice", MK_TEST_COINS(0)), false, ""); + + uint64_t blk_1r_height = c.get_top_block_height() - 1; + crypto::hash blk_1r_id = c.get_block_id_by_height(blk_1r_height); + block blk_2a = AUTO_VAL_INIT(blk_2a); + r = mine_next_pow_block_in_playtime_with_given_txs(m_accounts[MINER_ACC_IDX].get_public_address(), c, std::vector(), blk_1r_id, blk_1r_height + 1, &blk_2a); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime_with_given_txs failed"); + + // one more to trigger chain switch + block blk_3a = AUTO_VAL_INIT(blk_3a); + r = mine_next_pow_block_in_playtime_with_given_txs(m_accounts[MINER_ACC_IDX].get_public_address(), c, std::vector(), get_block_hash(blk_2a), get_block_height(blk_2a) + 1, &blk_3a); + CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime_with_given_txs failed"); + + // make sure tx_1 has been moved back to the pool + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Incorrect txs count in the pool: " << c.get_pool_transactions_count()); + CHECK_AND_ASSERT_MES(c.get_alternative_blocks_count() == 1, false, "Incorrect alt blocks count: " << c.get_alternative_blocks_count()); + + //const transaction& tx_1 = boost::get(events[4 * CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 3]); + + // refresh wallet + alice_wlt->refresh(); + // DO NOT scan_tx_pool here intentionally + CHECK_AND_ASSERT_MES(check_balance_via_wallet(*alice_wlt, "Alice", MK_TEST_COINS(0)), false, ""); + + return true; +} + +//------------------------------------------------------------------------------ + +wallet_unconfimed_tx_balance::wallet_unconfimed_tx_balance() +{ + REGISTER_CALLBACK_METHOD(wallet_unconfimed_tx_balance, c1); +} + +bool wallet_unconfimed_tx_balance::generate(std::vector& events) const +{ + // Test outline: + // 1. Miner sends 100 coins to Alice (50 + 50) + // 2. Alice sends 30 back to Miner (tx is unconfirmed) + // 3. Make sure Alice's wallet has correct balance, when it is checked from wallet's callback + // 4. Few blocks are mined so the tx is get confirmed + // 5. Make sure Alice's balance has changed correctly + + m_accounts.resize(TOTAL_ACCS_COUNT); + account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); + account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); + + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + + MAKE_TX(events, tx_0, miner_acc, alice_acc, MK_TEST_COINS(50), blk_0r); + MAKE_TX(events, tx_1, miner_acc, alice_acc, MK_TEST_COINS(50), blk_0r); + MAKE_NEXT_BLOCK_TX_LIST(events, blk_1, blk_0r, miner_acc, std::list({ tx_0, tx_1 })); + + REWIND_BLOCKS_N(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); + + DO_CALLBACK(events, "c1"); + + return true; +} + +bool wallet_unconfimed_tx_balance::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + bool r = false; + std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); + + CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(100), false, UINT64_MAX, MK_TEST_COINS(100)), false, ""); + + bool callback_is_ok = false; + // this callback will ba called from within wallet2::transfer() below + std::shared_ptr l(new wlt_lambda_on_transfer2_wrapper( + [&callback_is_ok](const tools::wallet_public::wallet_transfer_info& wti, uint64_t balance, uint64_t unlocked_balance, uint64_t total_mined) -> bool + { + CHECK_AND_ASSERT_MES(balance == MK_TEST_COINS(70), false, "invalid balance: " << print_money_brief(balance)); + CHECK_AND_ASSERT_MES(unlocked_balance == MK_TEST_COINS(50), false, "invalid unlocked_balance: " << print_money_brief(unlocked_balance)); + CHECK_AND_ASSERT_MES(total_mined == 0, false, "invalid total_mined: " << print_money_brief(total_mined)); + callback_is_ok = true; + return true; + } + )); + alice_wlt->callback(l); + + uint64_t fee = TESTS_DEFAULT_FEE * 3; + std::vector destinations{ tx_destination_entry(MK_TEST_COINS(30) - fee, m_accounts[MINER_ACC_IDX].get_public_address()) }; + try + { + alice_wlt->transfer(destinations, 0, 0, fee, empty_extra, empty_attachment); + } + catch (std::exception &e) + { + CHECK_AND_ASSERT_MES(false, false, "alice_wlt->transfer() caused an exception: " << e.what()); + } + + CHECK_AND_NO_ASSERT_MES(callback_is_ok, false, "callback failed"); + + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); + + // 50 coins should be locked and 50 - unlocked + CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(70), false, UINT64_MAX, MK_TEST_COINS(50), 0, 0, MK_TEST_COINS(30) - fee), false, ""); + + // mine WALLET_DEFAULT_TX_SPENDABLE_AGE blocks so the tx get confirmed and coins get unlocked + CHECK_AND_ASSERT_MES(mine_next_pow_blocks_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c, WALLET_DEFAULT_TX_SPENDABLE_AGE), false, ""); + + CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); + + CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(70), false, UINT64_MAX, MK_TEST_COINS(70), 0, 0, 0), false, ""); + + return true; +} diff --git a/tests/core_tests/wallet_packing_tx.h b/tests/core_tests/wallet_packing_tx.h new file mode 100644 index 00000000..8e2fb4a6 --- /dev/null +++ b/tests/core_tests/wallet_packing_tx.h @@ -0,0 +1,17 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#include "chaingen.h" +#include "wallet_tests_basic.h" + +struct packing_for_pos_minting_wallet_test : public wallet_test +{ + packing_for_pos_minting_wallet_test(); + mined_balance_wallet_test(); + bool generate(std::vector& events) const; + bool set_core_config(currency::core& c, size_t ev_index, const std::vector& events); + bool c1(currency::core& c, size_t ev_index, const std::vector& events); +}; \ No newline at end of file diff --git a/tests/core_tests/wallet_tests.cpp b/tests/core_tests/wallet_tests.cpp index 05dd6f19..870f3099 100644 --- a/tests/core_tests/wallet_tests.cpp +++ b/tests/core_tests/wallet_tests.cpp @@ -3297,3 +3297,79 @@ bool wallet_unconfimed_tx_balance::c1(currency::core& c, size_t ev_index, const return true; } + +//------------------------------------------------------------------------------ + +packing_outputs_on_pos_minting_wallet::packing_outputs_on_pos_minting_wallet() +{ + REGISTER_CALLBACK_METHOD(packing_outputs_on_pos_minting_wallet, c1); + REGISTER_CALLBACK_METHOD(packing_outputs_on_pos_minting_wallet, set_core_config); +} +bool packing_outputs_on_pos_minting_wallet::generate(std::vector& events) const +{ + + // 0 10 11 21 22 <- blockchain height (assuming CURRENCY_MINED_MONEY_UNLOCK_WINDOW == 10) + // (0 )... (0r)- (1 )... (1r)- <- main chain + // tx_0 <- txs + + GENERATE_ACCOUNT(miner_acc); + m_accounts.push_back(miner_acc); + //GENERATE_ACCOUNT(alice_acc); + //m_accounts.push_back(alice_acc); + + // don't use MAKE_GENESIS_BLOCK here because it will mask 'generator' + currency::block blk_0 = AUTO_VAL_INIT(blk_0); + generator.construct_genesis_block(blk_0, miner_acc, test_core_time::get_time()); + events.push_back(blk_0); + + DO_CALLBACK(events, "set_core_config"); + + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW+5); + + //MAKE_TX_FEE(events, tx_0, miner_acc, alice_acc, MK_TEST_COINS(2000), TESTS_DEFAULT_FEE, blk_0r); + //MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_acc, tx_0); + //REWIND_BLOCKS_N_WITH_TIME(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); + + DO_CALLBACK(events, "c1"); + + return true; +} + +bool packing_outputs_on_pos_minting_wallet::set_core_config(currency::core& c, size_t ev_index, const std::vector& events) +{ + core_runtime_config crc = c.get_blockchain_storage().get_core_runtime_config(); + crc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; + crc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; + c.get_blockchain_storage().set_core_runtime_config(crc); + return true; +} + +bool packing_outputs_on_pos_minting_wallet::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + std::shared_ptr miner_wlt = init_playtime_test_wallet(events, c, MINER_ACC_IDX); + size_t blocks_fetched = 0; + bool received_money; + std::atomic atomic_false = ATOMIC_VAR_INIT(false); + miner_wlt->refresh(blocks_fetched, received_money, atomic_false); + CHECK_AND_ASSERT_MES(blocks_fetched == CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 5, false, "Incorrect numbers of blocks fetched"); + + miner_wlt->set_pos_mint_packing_size(4); + check_balance_via_wallet(*miner_wlt.get(), "miner_wlt", MK_TEST_COINS(2000), 0, MK_TEST_COINS(2000), 0, 0); + + miner_wlt->try_mint_pos(); + + CHECK_AND_ASSERT_MES(c.get_current_blockchain_size() == CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 7, false, "Incorrect blockchain height:" << c.get_current_blockchain_size()); + miner_wlt->refresh(blocks_fetched, received_money, atomic_false); + CHECK_AND_ASSERT_MES(blocks_fetched == 1, false, "Incorrect numbers of blocks fetched"); + + block top_block = AUTO_VAL_INIT(top_block); + bool r = c.get_blockchain_storage().get_top_block(top_block); + CHECK_AND_ASSERT_MES(r && is_pos_block(top_block), false, "get_top_block failed or smth goes wrong"); + uint64_t top_block_reward = get_outs_money_amount(top_block.miner_tx); + check_balance_via_wallet(*miner_wlt.get(), "miner_wlt", uint64_max, MK_TEST_COINS(2000) + top_block_reward, 0, 0, 0); + + miner_wlt->reset_password(g_wallet_password); + miner_wlt->store(g_wallet_filename); + + return true; +} \ No newline at end of file diff --git a/tests/core_tests/wallet_tests.h b/tests/core_tests/wallet_tests.h index 6fbd37bd..89dfb05f 100644 --- a/tests/core_tests/wallet_tests.h +++ b/tests/core_tests/wallet_tests.h @@ -251,3 +251,11 @@ struct wallet_unconfimed_tx_balance : public wallet_test bool generate(std::vector& events) const; bool c1(currency::core& c, size_t ev_index, const std::vector& events); }; + +struct packing_outputs_on_pos_minting_wallet : public wallet_test +{ + packing_outputs_on_pos_minting_wallet(); + bool generate(std::vector& events) const; + bool set_core_config(currency::core& c, size_t ev_index, const std::vector& events); + bool c1(currency::core& c, size_t ev_index, const std::vector& events); +}; From d337500b9c6c1b7f476675d9c0be0275a3809773 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 01:49:03 +0100 Subject: [PATCH 14/21] removed unused files --- tests/core_tests/wallet_packing_tx.cpp | 571 ------------------------- tests/core_tests/wallet_packing_tx.h | 17 - 2 files changed, 588 deletions(-) delete mode 100644 tests/core_tests/wallet_packing_tx.cpp delete mode 100644 tests/core_tests/wallet_packing_tx.h diff --git a/tests/core_tests/wallet_packing_tx.cpp b/tests/core_tests/wallet_packing_tx.cpp deleted file mode 100644 index 031b56cf..00000000 --- a/tests/core_tests/wallet_packing_tx.cpp +++ /dev/null @@ -1,571 +0,0 @@ -// Copyright (c) 2014-2018 Zano Project -// Copyright (c) 2014-2018 The Louisdor Project -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "chaingen.h" -#include "wallet_tests.h" -#include "wallet_test_core_proxy.h" -#include "../../src/wallet/wallet_public_structs_defs.h" -#include "offers_helper.h" -#include "string_coding.h" -#include "random_helper.h" -#include "tx_builder.h" - -using namespace epee; -using namespace crypto; -using namespace currency; - -const uint64_t uint64_max = std::numeric_limits::max(); -const std::wstring g_wallet_filename = L"~coretests.wallet.file.tmp"; -const std::string g_wallet_password = "dofatibmzibeziyekigo"; -const currency::account_base null_account = AUTO_VAL_INIT(null_account); - - -POD_MAKE_COMPARABLE(currency, tx_out); - -// Determines which output is real and actually spent in tx inputs, when there are fake outputs. -bool determine_tx_real_inputs(currency::core& c, const currency::transaction& tx, const currency::account_keys& keys, std::vector& real_inputs) -{ - struct local_visitor - { - local_visitor(const currency::account_keys& keys, const crypto::key_image key_image) - : m_keys(keys) - , m_txin_key_image(key_image) - , m_output_in_input_index(0) - , m_found(false) - {} - - bool handle_output(const transaction& source_tx, const transaction& validated_tx, const tx_out& out, uint64_t out_i) - { - CHECK_AND_ASSERT_MES(!m_found, false, "Internal error: m_found is true but the visitor is still being applied"); - auto it = std::find(validated_tx.vout.begin(), validated_tx.vout.end(), out); - if (it == validated_tx.vout.end()) - return false; - size_t output_tx_index = it - validated_tx.vout.begin(); - - crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(validated_tx); - crypto::key_derivation derivation; - bool r = generate_key_derivation(tx_pub_key, m_keys.m_view_secret_key, derivation); - CHECK_AND_ASSERT_MES(r, false, "generate_key_derivation failed"); - crypto::secret_key ephemeral_secret_key; - derive_secret_key(derivation, output_tx_index, m_keys.m_spend_secret_key, ephemeral_secret_key); - - crypto::public_key output_public_key = boost::get(out.target).key; - - /*crypto::public_key ephemeral_public_key; - derive_public_key(derivation, output_tx_index, m_keys.m_account_address.m_spend_public_key, ephemeral_public_key);*/ - - crypto::key_image ki; - generate_key_image(output_public_key, ephemeral_secret_key, ki); - - if (ki == m_txin_key_image) - { - m_found = true; - return false; // to break the loop in scan_outputkeys_for_indexes - } - - ++m_output_in_input_index; - return true; - } - - currency::account_keys m_keys; - crypto::key_image m_txin_key_image; - size_t m_output_in_input_index; - bool m_found; - }; - - for (auto& txin : tx.vin) - { - const txin_to_key& in = boost::get(txin); - if (in.key_offsets.size() == 1) - { - real_inputs.push_back(0); // trivial case when no mixin is used - continue; - } - local_visitor vis(keys, in.k_image); - bool r = c.get_blockchain_storage().scan_outputkeys_for_indexes(tx, in, vis); - CHECK_AND_ASSERT_MES(r || vis.m_found, false, "scan_outputkeys_for_indexes failed"); - if (!vis.m_found) - return false; - real_inputs.push_back(vis.m_output_in_input_index); - } - - return true; -} - -//------------------------------------------------------------------------------ - -//------------------------------------------------------------------------------ - -mined_balance_wallet_test::mined_balance_wallet_test() -{ - REGISTER_CALLBACK_METHOD(mined_balance_wallet_test, c1); - REGISTER_CALLBACK_METHOD(mined_balance_wallet_test, set_core_config); -} - -bool mined_balance_wallet_test::generate(std::vector& events) const -{ - GENERATE_ACCOUNT(preminer_acc); - GENERATE_ACCOUNT(miner_acc); - m_accounts.push_back(miner_acc); - GENERATE_ACCOUNT(alice_acc); - m_accounts.push_back(alice_acc); - - block blk_0 = AUTO_VAL_INIT(blk_0); - generator.construct_genesis_block(blk_0, preminer_acc, test_core_time::get_time()); - events.push_back(blk_0); - - DO_CALLBACK(events, "set_core_config"); - DO_CALLBACK(events, "c1"); - - return true; -} - -bool mined_balance_wallet_test::set_core_config(currency::core& c, size_t ev_index, const std::vector& events) -{ - core_runtime_config crc = c.get_blockchain_storage().get_core_runtime_config(); - crc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; - crc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; - c.get_blockchain_storage().set_core_runtime_config(crc); - return true; -} - -bool mined_balance_wallet_test::c1(currency::core& c, size_t ev_index, const std::vector& events) -{ - blockchain_storage& bcs = c.get_blockchain_storage(); - - core_runtime_config crc = bcs.get_core_runtime_config(); - crc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; - bcs.set_core_runtime_config(crc); - - std::shared_ptr miner_wlt = init_playtime_test_wallet(events, c, MINER_ACC_IDX); - std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); - - CHECK_AND_ASSERT_MES(check_balance_via_wallet(*miner_wlt.get(), "miner", 0), false, "wrong balance"); - CHECK_AND_ASSERT_MES(check_balance_via_wallet(*alice_wlt.get(), "alice", 0), false, "wrong balance"); - - uint64_t miner_mined_money = 0; - bool r = false; - std::list blocks; - - size_t n = CURRENCY_MINED_MONEY_UNLOCK_WINDOW; - r = mine_next_pow_blocks_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c, n); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_blocks_in_playtime failed"); - r = bcs.get_blocks(bcs.get_current_blockchain_size() - n, n, blocks); - CHECK_AND_ASSERT_MES(r, false, "get_blocks failed"); - - for (auto& b : blocks) - miner_mined_money += get_outs_money_amount(b.miner_tx); - - miner_wlt->refresh(); - CHECK_AND_ASSERT_MES(check_balance_via_wallet(*miner_wlt.get(), "miner", miner_mined_money, miner_mined_money), false, "wrong balance"); - - n = bcs.get_current_blockchain_size(); - r = miner_wlt->try_mint_pos(); - CHECK_AND_ASSERT_MES(r && bcs.get_current_blockchain_size() > n, false, "can't mint a PoS block"); - - block b = AUTO_VAL_INIT(b); - r = bcs.get_top_block(b); - CHECK_AND_ASSERT_MES(r, false, "get_top_block failed"); - CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 2, false, "Invalid PoS coinbase tx"); - - uint64_t coinbase_outs_amount = get_outs_money_amount(b.miner_tx); - uint64_t stake_amount = boost::get(b.miner_tx.vin[1]).amount; - CHECK_AND_ASSERT_MES(coinbase_outs_amount > stake_amount, false, "coinbase_outs_amount = " << coinbase_outs_amount << ", stake_amount = " << stake_amount << " : invalid condition"); - - miner_mined_money += coinbase_outs_amount - stake_amount; - - miner_wlt->refresh(); - - std::stringstream ss; - miner_wlt->dump_trunsfers(ss, false); - LOG_PRINT_CYAN("miner transfers: " << ENDL << ss.str(), LOG_LEVEL_0); - - CHECK_AND_ASSERT_MES(check_balance_via_wallet(*miner_wlt.get(), "miner", miner_mined_money, miner_mined_money), false, "wrong balance"); - - return true; -} - -//------------------------------------------------------------------------------ - -wallet_outputs_with_same_key_image::wallet_outputs_with_same_key_image() -{ - REGISTER_CALLBACK_METHOD(wallet_outputs_with_same_key_image, c1); -} - -bool wallet_outputs_with_same_key_image::generate(std::vector& events) const -{ - // Test idea: make sure wallet does not take into account valid outputs having the same key image - // Only one such output is spendable thus only one output should be taken into account. - - bool r = false; - m_accounts.resize(TOTAL_ACCS_COUNT); - account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); - account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); - - MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); - REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 2); - - uint64_t tx_amount = MK_TEST_COINS(3); - - // tx_1 - std::vector sources_1; - r = fill_tx_sources(sources_1, events, blk_0r, miner_acc.get_keys(), tx_amount + TESTS_DEFAULT_FEE, 0); - CHECK_AND_ASSERT_MES(r, false, "fill_tx_sources failed"); - - std::vector destinations{ tx_destination_entry(tx_amount, alice_acc.get_public_address()) }; - - tx_builder builder; - builder.step1_init(); - builder.step2_fill_inputs(miner_acc.get_keys(), sources_1); - builder.step3_fill_outputs(destinations); - builder.step4_calc_hash(); - builder.step5_sign(sources_1); - - transaction tx_1 = builder.m_tx; - events.push_back(tx_1); - - // tx_2 with the same secret key - currency::keypair tmp_sec_key = builder.m_tx_key; - builder.step1_init(); - builder.m_tx_key = tmp_sec_key; - builder.m_tx.extra.clear(); - add_tx_pub_key_to_extra(builder.m_tx, builder.m_tx_key.pub); - - std::vector sources_2; - r = fill_tx_sources(sources_2, events, blk_0r, miner_acc.get_keys(), tx_amount + TESTS_DEFAULT_FEE, 0, sources_1); - CHECK_AND_ASSERT_MES(r, false, "fill_tx_sources failed"); - - // keep destinations the same - - builder.step2_fill_inputs(miner_acc.get_keys(), sources_2); - builder.step3_fill_outputs(destinations); - builder.step4_calc_hash(); - builder.step5_sign(sources_2); - - transaction tx_2 = builder.m_tx; - events.push_back(tx_2); - - // make sure tx_1 and tx_2 have been created with the same tx key - CHECK_AND_ASSERT_MES(get_tx_pub_key_from_extra(tx_1) == get_tx_pub_key_from_extra(tx_2), false, "internal error: tx_1 and tx_2 have different pub keys"); - - // now both txs are in the pool, make sure they are - DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(2)); - - DO_CALLBACK(events, "c1"); - - return true; -} - -bool wallet_outputs_with_same_key_image::c1(currency::core& c, size_t ev_index, const std::vector& events) -{ - std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, m_accounts[ALICE_ACC_IDX]); - - // check Alice has no unlocked coins - bool r = refresh_wallet_and_check_balance("before tx_1 and tx_2 added", "Alice", alice_wlt, MK_TEST_COINS(3) * 2, true, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 2, 0); - CHECK_AND_ASSERT_MES(r, false, ""); - - r = mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime failed"); - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "there are txs in the pool!"); - - r = mine_next_pow_blocks_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_blocks_in_playtime failed"); - - // only one tx_1 output is counted as the tx_2 output has the very same key image - r = refresh_wallet_and_check_balance("after tx_1 and tx_2 added", "Alice", alice_wlt, MK_TEST_COINS(3), true, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 1, MK_TEST_COINS(3)); - CHECK_AND_ASSERT_MES(r, false, ""); - - // make sure Alice is able to transfer her coins to smbd - std::vector destinations{ tx_destination_entry(MK_TEST_COINS(3) - TESTS_DEFAULT_FEE, m_accounts[MINER_ACC_IDX].get_public_address()) }; - try - { - alice_wlt->transfer(destinations, 0, 0, TESTS_DEFAULT_FEE, empty_extra, empty_attachment); - } - catch (...) - { - CHECK_AND_ASSERT_MES(false, false, "Alice failed to transfer all her funds"); - } - - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "wrong tx count in the pool: " << c.get_pool_transactions_count()); - r = mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime failed"); - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "there are txs in the pool!"); - - return true; -} - -//------------------------------------------------------------------------------ - -wallet_unconfirmed_tx_expiration::wallet_unconfirmed_tx_expiration() -{ - REGISTER_CALLBACK_METHOD(wallet_unconfirmed_tx_expiration, c1); -} - -bool wallet_unconfirmed_tx_expiration::generate(std::vector& events) const -{ - // Test outline: - // 1. Alice sends tx with expiration. - // 2. Miner ignores Alice's tx, so tx expires in the pool. - // 3. Tx is being removed from the pool due to expiration. - // Make sure Alice eventually spent no coins and all her money is unlocked in the wallet. - - m_accounts.resize(TOTAL_ACCS_COUNT); - account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); - account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); - account_base& bob_acc = m_accounts[BOB_ACC_IDX]; bob_acc.generate(); - - MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); - REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 1); - - bool r = false; - transaction tx_0 = AUTO_VAL_INIT(tx_0); - r = construct_tx_with_many_outputs(events, blk_0r, miner_acc.get_keys(), alice_acc.get_public_address(), TESTS_DEFAULT_FEE * 20, 10, TESTS_DEFAULT_FEE, tx_0); - CHECK_AND_ASSERT_MES(r, false, "construct_tx_with_many_outputs"); - events.push_back(tx_0); - transaction tx_1 = AUTO_VAL_INIT(tx_1); - r = construct_tx_with_many_outputs(events, blk_0r, miner_acc.get_keys(), bob_acc.get_public_address(), TESTS_DEFAULT_FEE * 20, 10, TESTS_DEFAULT_FEE, tx_1); - CHECK_AND_ASSERT_MES(r, false, "construct_tx_with_many_outputs"); - events.push_back(tx_1); - MAKE_NEXT_BLOCK_TX_LIST(events, blk_1, blk_0r, miner_acc, std::list({ tx_0, tx_1 })); - - REWIND_BLOCKS_N(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); - - DO_CALLBACK(events, "c1"); - - return true; -} - -bool wallet_unconfirmed_tx_expiration::c1(currency::core& c, size_t ev_index, const std::vector& events) -{ - bool r = false; - std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); - - uint64_t alice_start_balance = TESTS_DEFAULT_FEE * 20; - CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, alice_start_balance), false, ""); - - // Alice constructs and sends tx with expiration time - uint64_t expiration_time = c.get_blockchain_storage().get_tx_expiration_median() + TX_EXPIRATION_MEDIAN_SHIFT + 15; - etc_tx_details_expiration_time extra_entry = AUTO_VAL_INIT(extra_entry); - extra_entry.v = expiration_time; - std::vector extra({ extra_entry }); // extra with expiration time - std::vector destinations({ tx_destination_entry(TESTS_DEFAULT_FEE * 2, m_accounts[MINER_ACC_IDX].get_public_address()) }); - transaction tx = AUTO_VAL_INIT(tx); - try - { - alice_wlt->transfer(destinations, 0, 0, TESTS_DEFAULT_FEE, extra, empty_attachment, tools::detail::ssi_digit, tools::tx_dust_policy(DEFAULT_DUST_THRESHOLD), tx); - } - catch (std::exception &e) - { - CHECK_AND_ASSERT_MES(false, false, "alice_wlt->transfer() caused an exception: " << e.what()); - } - - CHECK_AND_ASSERT_MES(get_tx_expiration_time(tx) == expiration_time, false, "tx expiration time wasn't set"); - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Invalid txs count in the pool: " << c.get_pool_transactions_count()); - - CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("tx sent,", "Alice", alice_wlt, alice_start_balance - TESTS_DEFAULT_FEE * 2 - TESTS_DEFAULT_FEE, true, 0, UINT64_MAX, 0, 0, TESTS_DEFAULT_FEE * 2), false, ""); - - // mine a few block with no tx, so Alice's tx is expired in the pool - for (size_t i = 0; i < 5; ++i) - { - r = mine_next_pow_block_in_playtime_with_given_txs(m_accounts[MINER_ACC_IDX].get_public_address(), c, std::vector()); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime_with_given_txs failed"); - } - - // tx is still there - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Invalid txs count in the pool: " << c.get_pool_transactions_count()); - - // make sure expiration median was shifted enough - CHECK_AND_ASSERT_MES(c.get_blockchain_storage().is_tx_expired(tx), false, "wrong expiration time condition"); - - LOG_PRINT_CYAN("%%%%% tx_pool::on_idle()", LOG_LEVEL_0); - c.get_tx_pool().on_idle(); - - // make sure tx was removed by the pool - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "Invalid txs count in the pool: " << c.get_pool_transactions_count()); - - // mine one more block to trigger wallet's on_idle() and outdated tx clearing - r = mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime failed"); - - // make sure all Alice's money are unlocked and no coins were actually spent - CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("tx expired and removed from the pool,", "Alice", alice_wlt, alice_start_balance, true, 6, alice_start_balance, 0, 0, 0), false, ""); - - return true; -} - -//------------------------------------------------------------------------------ - -wallet_chain_switch_with_spending_the_same_ki::wallet_chain_switch_with_spending_the_same_ki() -{ - REGISTER_CALLBACK_METHOD(wallet_chain_switch_with_spending_the_same_ki, c1); -} - -bool wallet_chain_switch_with_spending_the_same_ki::generate(std::vector& events) const -{ - // Test outline - // 1. A wallet has one unspent output - // 2. wallet2::transfer() creates tx_0 that spends wallet's output - // 3. tx_0 is successfully put into the blockchain - // 4. Due to chain switch tx_0 is removed from the blockchain and get into the transaction pool - // 5. Make sure the wallet can't spend that output - // 6. After tx is expired make sure the wallet can spend that output - - - m_accounts.resize(TOTAL_ACCS_COUNT); - account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); - account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); - account_base& bob_acc = m_accounts[BOB_ACC_IDX]; bob_acc.generate(); - - MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); - REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); - - MAKE_TX(events, tx_0, miner_acc, alice_acc, MK_TEST_COINS(30), blk_0r); - MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_acc, tx_0); - - // rewind blocks to allow wallet be able to spend the coins - REWIND_BLOCKS_N_WITH_TIME(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); - - DO_CALLBACK(events, "c1"); - - return true; -} - -bool wallet_chain_switch_with_spending_the_same_ki::c1(currency::core& c, size_t ev_index, const std::vector& events) -{ - bool r = false; - std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); - - CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(30), true, UINT64_MAX, MK_TEST_COINS(30)), false, ""); - - std::vector destinations { tx_destination_entry(MK_TEST_COINS(30) - TESTS_DEFAULT_FEE, m_accounts[BOB_ACC_IDX].get_public_address()) }; - try - { - // create tx_1 - alice_wlt->transfer(destinations, 0, 0, TESTS_DEFAULT_FEE, empty_extra, empty_attachment); - } - catch (std::exception &e) - { - CHECK_AND_ASSERT_MES(false, false, "alice_wlt->transfer() caused an exception: " << e.what()); - } - - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); - - // mine blk_2 on height 22 - CHECK_AND_ASSERT_MES(mine_next_pow_block_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c), false, ""); - - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); - - // refresh wallet - alice_wlt->refresh(); - // DO NOT scan_tx_pool here intentionally - CHECK_AND_ASSERT_MES(check_balance_via_wallet(*alice_wlt, "Alice", MK_TEST_COINS(0)), false, ""); - - uint64_t blk_1r_height = c.get_top_block_height() - 1; - crypto::hash blk_1r_id = c.get_block_id_by_height(blk_1r_height); - block blk_2a = AUTO_VAL_INIT(blk_2a); - r = mine_next_pow_block_in_playtime_with_given_txs(m_accounts[MINER_ACC_IDX].get_public_address(), c, std::vector(), blk_1r_id, blk_1r_height + 1, &blk_2a); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime_with_given_txs failed"); - - // one more to trigger chain switch - block blk_3a = AUTO_VAL_INIT(blk_3a); - r = mine_next_pow_block_in_playtime_with_given_txs(m_accounts[MINER_ACC_IDX].get_public_address(), c, std::vector(), get_block_hash(blk_2a), get_block_height(blk_2a) + 1, &blk_3a); - CHECK_AND_ASSERT_MES(r, false, "mine_next_pow_block_in_playtime_with_given_txs failed"); - - // make sure tx_1 has been moved back to the pool - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Incorrect txs count in the pool: " << c.get_pool_transactions_count()); - CHECK_AND_ASSERT_MES(c.get_alternative_blocks_count() == 1, false, "Incorrect alt blocks count: " << c.get_alternative_blocks_count()); - - //const transaction& tx_1 = boost::get(events[4 * CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 3]); - - // refresh wallet - alice_wlt->refresh(); - // DO NOT scan_tx_pool here intentionally - CHECK_AND_ASSERT_MES(check_balance_via_wallet(*alice_wlt, "Alice", MK_TEST_COINS(0)), false, ""); - - return true; -} - -//------------------------------------------------------------------------------ - -wallet_unconfimed_tx_balance::wallet_unconfimed_tx_balance() -{ - REGISTER_CALLBACK_METHOD(wallet_unconfimed_tx_balance, c1); -} - -bool wallet_unconfimed_tx_balance::generate(std::vector& events) const -{ - // Test outline: - // 1. Miner sends 100 coins to Alice (50 + 50) - // 2. Alice sends 30 back to Miner (tx is unconfirmed) - // 3. Make sure Alice's wallet has correct balance, when it is checked from wallet's callback - // 4. Few blocks are mined so the tx is get confirmed - // 5. Make sure Alice's balance has changed correctly - - m_accounts.resize(TOTAL_ACCS_COUNT); - account_base& miner_acc = m_accounts[MINER_ACC_IDX]; miner_acc.generate(); - account_base& alice_acc = m_accounts[ALICE_ACC_IDX]; alice_acc.generate(); - - MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); - REWIND_BLOCKS_N(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); - - MAKE_TX(events, tx_0, miner_acc, alice_acc, MK_TEST_COINS(50), blk_0r); - MAKE_TX(events, tx_1, miner_acc, alice_acc, MK_TEST_COINS(50), blk_0r); - MAKE_NEXT_BLOCK_TX_LIST(events, blk_1, blk_0r, miner_acc, std::list({ tx_0, tx_1 })); - - REWIND_BLOCKS_N(events, blk_1r, blk_1, miner_acc, WALLET_DEFAULT_TX_SPENDABLE_AGE); - - DO_CALLBACK(events, "c1"); - - return true; -} - -bool wallet_unconfimed_tx_balance::c1(currency::core& c, size_t ev_index, const std::vector& events) -{ - bool r = false; - std::shared_ptr alice_wlt = init_playtime_test_wallet(events, c, ALICE_ACC_IDX); - - CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(100), false, UINT64_MAX, MK_TEST_COINS(100)), false, ""); - - bool callback_is_ok = false; - // this callback will ba called from within wallet2::transfer() below - std::shared_ptr l(new wlt_lambda_on_transfer2_wrapper( - [&callback_is_ok](const tools::wallet_public::wallet_transfer_info& wti, uint64_t balance, uint64_t unlocked_balance, uint64_t total_mined) -> bool - { - CHECK_AND_ASSERT_MES(balance == MK_TEST_COINS(70), false, "invalid balance: " << print_money_brief(balance)); - CHECK_AND_ASSERT_MES(unlocked_balance == MK_TEST_COINS(50), false, "invalid unlocked_balance: " << print_money_brief(unlocked_balance)); - CHECK_AND_ASSERT_MES(total_mined == 0, false, "invalid total_mined: " << print_money_brief(total_mined)); - callback_is_ok = true; - return true; - } - )); - alice_wlt->callback(l); - - uint64_t fee = TESTS_DEFAULT_FEE * 3; - std::vector destinations{ tx_destination_entry(MK_TEST_COINS(30) - fee, m_accounts[MINER_ACC_IDX].get_public_address()) }; - try - { - alice_wlt->transfer(destinations, 0, 0, fee, empty_extra, empty_attachment); - } - catch (std::exception &e) - { - CHECK_AND_ASSERT_MES(false, false, "alice_wlt->transfer() caused an exception: " << e.what()); - } - - CHECK_AND_NO_ASSERT_MES(callback_is_ok, false, "callback failed"); - - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 1, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); - - // 50 coins should be locked and 50 - unlocked - CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(70), false, UINT64_MAX, MK_TEST_COINS(50), 0, 0, MK_TEST_COINS(30) - fee), false, ""); - - // mine WALLET_DEFAULT_TX_SPENDABLE_AGE blocks so the tx get confirmed and coins get unlocked - CHECK_AND_ASSERT_MES(mine_next_pow_blocks_in_playtime(m_accounts[MINER_ACC_IDX].get_public_address(), c, WALLET_DEFAULT_TX_SPENDABLE_AGE), false, ""); - - CHECK_AND_ASSERT_MES(c.get_pool_transactions_count() == 0, false, "Tx pool has incorrect number of txs: " << c.get_pool_transactions_count()); - - CHECK_AND_ASSERT_MES(refresh_wallet_and_check_balance("", "Alice", alice_wlt, MK_TEST_COINS(70), false, UINT64_MAX, MK_TEST_COINS(70), 0, 0, 0), false, ""); - - return true; -} diff --git a/tests/core_tests/wallet_packing_tx.h b/tests/core_tests/wallet_packing_tx.h deleted file mode 100644 index 8e2fb4a6..00000000 --- a/tests/core_tests/wallet_packing_tx.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2014-2018 Zano Project -// Copyright (c) 2014-2018 The Louisdor Project -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#pragma once -#include "chaingen.h" -#include "wallet_tests_basic.h" - -struct packing_for_pos_minting_wallet_test : public wallet_test -{ - packing_for_pos_minting_wallet_test(); - mined_balance_wallet_test(); - bool generate(std::vector& events) const; - bool set_core_config(currency::core& c, size_t ev_index, const std::vector& events); - bool c1(currency::core& c, size_t ev_index, const std::vector& events); -}; \ No newline at end of file From 3eb68cb8fb72707d24a1e6539c399eaf55830090 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 01:54:08 +0100 Subject: [PATCH 15/21] added log message --- src/wallet/wallet2.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index cd17eb08..3e18a9ea 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2794,8 +2794,9 @@ bool wallet2::build_minted_block(const currency::COMMAND_RPC_SCAN_POS::request& //generate packing tx transaction pack_tx = AUTO_VAL_INIT(pack_tx); if (generate_packing_transaction_if_needed(pack_tx, 0)) - { + { tx_to_blob(pack_tx, tmpl_req.explicit_transaction); + WLT_LOG_GREEN("Pacling inputs: " << pack_tx.vin.size() << " inputs consolidated", LOG_LEVEL_0); } m_core_proxy->call_COMMAND_RPC_GETBLOCKTEMPLATE(tmpl_req, tmpl_rsp); WLT_CHECK_AND_ASSERT_MES(tmpl_rsp.status == CORE_RPC_STATUS_OK, false, "Failed to create block template after kernel hash found!"); From 91be33581a946d3eeb5d26d56af8319488a09e9d Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 02:18:45 +0100 Subject: [PATCH 16/21] added command to simplewallet for printing output distribution --- src/simplewallet/simplewallet.cpp | 13 +++++++++++++ src/simplewallet/simplewallet.h | 1 + src/wallet/wallet2.cpp | 10 ++++++++++ src/wallet/wallet2.h | 1 + 4 files changed, 25 insertions(+) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 98cd5ba6..87532a6a 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -207,6 +207,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("fix_collisions", boost::bind(&simple_wallet::fix_collisions, this, _1), "Rescan transfers for key image collisions"); m_cmd_binder.set_handler("scan_transfers_for_id", boost::bind(&simple_wallet::scan_transfers_for_id, this, _1), "Rescan transfers for tx_id"); m_cmd_binder.set_handler("scan_transfers_for_ki", boost::bind(&simple_wallet::scan_transfers_for_ki, this, _1), "Rescan transfers for key image"); + m_cmd_binder.set_handler("print_utxo_distribution", boost::bind(&simple_wallet::print_utxo_distribution, this, _1), "Prints utxo distribution"); m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), "Show current wallet public address"); m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::integrated_address, this, _1), "integrated_address [| &args) print_td_list(td); return true; } +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::print_utxo_distribution(const std::vector &args) +{ + std::map distribution; + m_wallet->get_utxo_distribution(distribution); + for (auto& e : distribution) + { + message_writer() << std::left << setw(25) << print_money(e.first) << "|" << e.second; + } + return true; +} + //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_transfer_info(const std::vector &args) { diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 34ba13b2..91fccfd3 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -67,6 +67,7 @@ namespace currency bool fix_collisions(const std::vector &args ); bool scan_transfers_for_id(const std::vector &args); bool scan_transfers_for_ki(const std::vector &args); + bool print_utxo_distribution(const std::vector &args); bool show_blockchain_height(const std::vector &args); bool show_wallet_bcheight(const std::vector &args); bool transfer(const std::vector &args); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 3e18a9ea..996b50fa 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2442,6 +2442,16 @@ void wallet2::sign_transfer_files(const std::string& tx_sources_file, const std: THROW_IF_FALSE_WALLET_CMN_ERR_EX(r, "failed to store signed tx to file " << signed_tx_file); } //---------------------------------------------------------------------------------------------------- +bool wallet2::get_utxo_distribution(std::map& distribution) +{ + prepare_free_transfers_cache(0); + for (auto ent : m_found_free_amounts) + { + distribution[ent.first] = ent.second.size(); + } + return true; +} +//---------------------------------------------------------------------------------------------------- void wallet2::submit_transfer(const std::string& signed_tx_blob, currency::transaction& tx) { // decrypt sources diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 74c5e00f..382bac78 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -740,6 +740,7 @@ namespace tools std::string get_log_prefix() const { return m_log_prefix; } static uint64_t get_max_unlock_time_from_receive_indices(const currency::transaction& tx, const money_transfer2_details& td); + bool get_utxo_distribution(std::map& distribution); private: void add_transfers_to_expiration_list(const std::vector& selected_transfers, uint64_t expiration, uint64_t change_amount, const crypto::hash& related_tx_id); From eebac8eca4b3ea2fcb46422cbf2d198f60adf835 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 02:25:02 +0100 Subject: [PATCH 17/21] disabled unused macro --- src/wallet/wallet2.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 996b50fa..2d411e68 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -4266,7 +4266,7 @@ void wallet2::transfer(const std::vector& dsts, bool send_to_network, std::string* p_signed_tx_blob_str) { - TIME_MEASURE_START(precalculation_time); + //TIME_MEASURE_START(precalculation_time); construct_tx_param ctp = AUTO_VAL_INIT(ctp); ctp.attachments = attachments; ctp.crypt_address = currency::get_crypt_address_from_destinations(m_account.get_keys(), dsts); @@ -4282,7 +4282,7 @@ void wallet2::transfer(const std::vector& dsts, ctp.split_strategy_id = destination_split_strategy_id; ctp.tx_outs_attr = tx_outs_attr; ctp.unlock_time = unlock_time; - TIME_MEASURE_FINISH(precalculation_time); + //TIME_MEASURE_FINISH(precalculation_time); transfer(ctp, tx, send_to_network, p_signed_tx_blob_str); } //---------------------------------------------------------------------------------------------------- From 86d98d3e6771c307d94116df2bc5c97b3dda349f Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 03:16:54 +0100 Subject: [PATCH 18/21] fixed linux compilation issue --- contrib/epee/include/syncobj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index b5907538..3af9b28d 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -699,7 +699,7 @@ namespace epee #define CRITICAL_REGION_END() } -#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj##.##op;obj##_lock.unlock();} +#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj## . ##op;obj##_lock.unlock();} #define SHARED_CRITICAL_REGION_LOCAL(x) boost::shared_lock< boost::shared_mutex > critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_LOCAL(x) boost::unique_lock< boost::shared_mutex > critical_region_var(x) From 7d4f95ca535adeea67485d010b591eb6960e791f Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 04:16:17 +0100 Subject: [PATCH 19/21] fixed problem with seed nodes + transactions loaded to onboard container instead of prevalidation in tx pool --- .../currency_protocol_handler.inl | 21 +++++++++++++------ src/p2p/net_node.inl | 16 +++++++------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/currency_protocol/currency_protocol_handler.inl b/src/currency_protocol/currency_protocol_handler.inl index c3c224d2..db0dc3f6 100644 --- a/src/currency_protocol/currency_protocol_handler.inl +++ b/src/currency_protocol/currency_protocol_handler.inl @@ -529,27 +529,36 @@ namespace currency { CHECK_STOP_FLAG__DROP_AND_RETURN_IF_SET(1, "Blocks processing interrupted, connection dropped"); + block_verification_context bvc = boost::value_initialized(); //process transactions TIME_MEASURE_START(transactions_process_time); for (const auto& tx_blob : block_entry.txs) { CHECK_STOP_FLAG__DROP_AND_RETURN_IF_SET(1, "Block txs processing interrupted, connection dropped"); - - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(tx_blob, tvc, true); - if(tvc.m_verification_failed) + crypto::hash tx_id = null_hash; + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_id)) { - LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " + LOG_ERROR_CCONTEXT("failed to parse tx: " << string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); m_p2p->drop_connection(context); return 1; } + bvc.m_onboard_transactions[tx_id] = tx; +// tx_verification_context tvc = AUTO_VAL_INIT(tvc); +// m_core.handle_incoming_tx(tx_blob, tvc, true); +// if(tvc.m_verification_failed) +// { +// LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " +// << string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); +// m_p2p->drop_connection(context); +// return 1; +// } } TIME_MEASURE_FINISH(transactions_process_time); //process block TIME_MEASURE_START(block_process_time); - block_verification_context bvc = boost::value_initialized(); m_core.handle_incoming_block(block_entry.block, bvc, false); if (count > 2 && bvc.m_already_exists) diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 8557f89e..c70b45d7 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -276,14 +276,14 @@ namespace nodetool #ifndef TESTNET //TODO: //ADD_HARDCODED_SEED_NODE(std::string("0.0.0.0:") + std::to_string(P2P_DEFAULT_PORT)); - ADD_HARDCODED_SEED_NODE("95.217.43.225:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("94.130.137.230:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("95.217.42.247:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("94.130.160.115:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("195.201.107.230:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("95.217.46.49:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("159.69.76.144:", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("144.76.183.143:", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.43.225", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.137.230", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.42.247", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.160.115", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("195.201.107.230", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.46.49", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("159.69.76.144", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("144.76.183.143", P2P_DEFAULT_PORT); #else //TODO: ADD_HARDCODED_SEED_NODE("95.217.43.225", P2P_DEFAULT_PORT); From 98636c52a13f796b3ec85dbbaca17bfd3e1af2c2 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 04:30:40 +0100 Subject: [PATCH 20/21] CIRITCAL_OPERATION macro compatible with *nix --- contrib/epee/include/syncobj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index 3af9b28d..457c5f37 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -699,7 +699,7 @@ namespace epee #define CRITICAL_REGION_END() } -#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj## . ##op;obj##_lock.unlock();} +#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj . op;obj##_lock.unlock();} #define SHARED_CRITICAL_REGION_LOCAL(x) boost::shared_lock< boost::shared_mutex > critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_LOCAL(x) boost::unique_lock< boost::shared_mutex > critical_region_var(x) From c863094c11bd0241767ba3082406cf1de03b8fc4 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 3 Dec 2019 23:55:02 +0100 Subject: [PATCH 21/21] fixed misprint --- src/gui/qt-daemon/application/view_iface.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gui/qt-daemon/application/view_iface.h b/src/gui/qt-daemon/application/view_iface.h index 0dbfd962..85c1b08e 100644 --- a/src/gui/qt-daemon/application/view_iface.h +++ b/src/gui/qt-daemon/application/view_iface.h @@ -383,12 +383,12 @@ public: struct get_recent_transfers_request { uint64_t wallet_id; - uint64_t offest; + uint64_t offset; uint64_t count; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(wallet_id) - KV_SERIALIZE(offest) + KV_SERIALIZE(offset) KV_SERIALIZE(count) END_KV_SERIALIZE_MAP() };