From 267053964d1af0e175dc5f80c0416f811fe8ee3d Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Wed, 10 Jun 2020 23:56:14 +0200 Subject: [PATCH 1/8] extended api for fetching blocks/txs, started work on implementing deferred fetching of global output indexes --- src/currency_core/basic_kv_structs.h | 27 +++++++++++++++++++ src/currency_core/blockchain_storage.cpp | 2 +- src/currency_core/blockchain_storage.h | 2 +- .../currency_protocol_defs.h | 6 ++++- src/rpc/core_rpc_server.cpp | 21 +++++++++++---- src/rpc/core_rpc_server_commands_defs.h | 2 ++ src/wallet/view_iface.h | 12 ++------- src/wallet/wallet2.cpp | 7 +++++ src/wallet/wallet2.h | 13 +++++---- 9 files changed, 69 insertions(+), 23 deletions(-) create mode 100644 src/currency_core/basic_kv_structs.h diff --git a/src/currency_core/basic_kv_structs.h b/src/currency_core/basic_kv_structs.h new file mode 100644 index 00000000..1ff520c6 --- /dev/null +++ b/src/currency_core/basic_kv_structs.h @@ -0,0 +1,27 @@ +// Copyright (c) 2014-2018 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "warnings.h" + +PUSH_VS_WARNINGS +DISABLE_VS_WARNINGS(4100) +DISABLE_VS_WARNINGS(4503) +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage_template_helper.h" +POP_VS_WARNINGS + +namespace currency +{ + template + struct struct_with_one_t_type + { + t_type v; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(v) + END_KV_SERIALIZE_MAP() + }; +} diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index b196fbee..e59a8f9d 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -3093,7 +3093,7 @@ bool blockchain_storage::get_est_height_from_date(uint64_t date, uint64_t& res_h return true; } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height)const +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height, bool need_global_indexes)const { CRITICAL_REGION_LOCAL(m_read_lock); blocks_direct_container blocks_direct; diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 48352edf..73c9af2b 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -254,7 +254,7 @@ namespace currency bool get_short_chain_history(std::list& ids)const; bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const; bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset)const; - bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0)const; + bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0, bool need_global_indexes = false)const; bool find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0)const; //bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp)const; diff --git a/src/currency_protocol/currency_protocol_defs.h b/src/currency_protocol/currency_protocol_defs.h index 73b85e17..0bb974e0 100644 --- a/src/currency_protocol/currency_protocol_defs.h +++ b/src/currency_protocol/currency_protocol_defs.h @@ -13,6 +13,7 @@ #include "currency_core/connection_context.h" #include "currency_core/blockchain_storage_basic.h" #include "currency_protocol/blobdatatype.h" +#include "currency_core/basic_kv_structs.h" namespace currency { @@ -20,7 +21,7 @@ namespace currency #define BC_COMMANDS_POOL_BASE 2000 - + /************************************************************************/ /* */ /************************************************************************/ @@ -28,9 +29,12 @@ namespace currency { blobdata block; std::list txs; + std::vector > > tx_global_outs; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block) KV_SERIALIZE(txs) + KV_SERIALIZE(tx_global_outs) END_KV_SERIALIZE_MAP() }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index a527a484..ae353bc2 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -314,20 +314,31 @@ namespace currency return true; } - std::list > > bs; - if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height)) + blockchain_storage::blocks_direct_container bs; + if (!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height)) { res.status = API_RETURN_CODE_FAIL; return false; } - BOOST_FOREACH(auto& b, bs) + for (auto& b : bs) { res.blocks.resize(res.blocks.size()+1); - res.blocks.back().block = block_to_blob(b.first); + res.blocks.back().block = block_to_blob(b.first->bl); + if (req.need_global_indexes) + { + res.blocks.back().tx_global_outs.resize(b.second.size()); + } + size_t i = 0; + BOOST_FOREACH(auto& t, b.second) { - res.blocks.back().txs.push_back(tx_to_blob(t)); + res.blocks.back().txs.push_back(tx_to_blob(t->tx)); + if (req.need_global_indexes) + { + res.blocks.back().tx_global_outs[i].v = t->m_global_output_indexes; + } + i++; } } diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 1a0044b5..c2e22bf6 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -109,10 +109,12 @@ namespace currency struct request { + bool need_global_indexes; uint64_t minimum_height; std::list block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(need_global_indexes) KV_SERIALIZE(minimum_height) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) END_KV_SERIALIZE_MAP() diff --git a/src/wallet/view_iface.h b/src/wallet/view_iface.h index 7320b3ed..b46c5ad5 100644 --- a/src/wallet/view_iface.h +++ b/src/wallet/view_iface.h @@ -19,8 +19,10 @@ DISABLE_VS_WARNINGS(4503) #include "rpc/core_rpc_server_commands_defs.h" #include "wallet/wallet_public_structs_defs.h" #include "currency_core/offers_services_helpers.h" +#include "currency_core/basic_kv_structs.h" #include "currency_core/basic_api_response_codes.h" #include "common/error_codes.h" + POP_VS_WARNINGS //#endif @@ -769,16 +771,6 @@ public: END_KV_SERIALIZE_MAP() }; - template - struct struct_with_one_t_type - { - t_type v; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(v) - END_KV_SERIALIZE_MAP() - }; - #define API_MAX_ALIASES_COUNT 10000 struct i_view diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 93070875..687839e0 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -365,6 +365,7 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t pwallet_info->m_block_height = height; pwallet_info->m_block_timestamp = b.timestamp; +#ifndef MOBILE_WALLET_BUILD //good news - got money! take care about it //usually we have only one transfer for user in transaction currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request req = AUTO_VAL_INIT(req); @@ -377,6 +378,7 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t THROW_IF_TRUE_WALLET_EX(res.o_indexes.size() != tx.vout.size(), error::wallet_internal_error, "transactions outputs size=" + std::to_string(tx.vout.size()) + " not match with COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES response size=" + std::to_string(res.o_indexes.size())); +#endif for (size_t i_in_outs = 0; i_in_outs != outs.size(); i_in_outs++) { @@ -455,7 +457,12 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t td.m_ptx_wallet_info = pwallet_info; td.m_internal_output_index = o; td.m_key_image = ki; +#ifdef MOBILE_WALLET_BUILD + td.m_global_output_index = WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED; +#else td.m_global_output_index = res.o_indexes[o]; +#endif + if (coin_base_tx) { //last out in coinbase tx supposed to be change from coinstake diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index c3e893d1..e02f7138 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -48,7 +48,15 @@ #define WALLET_DEFAULT_POS_MINT_PACKING_SIZE 100 +#define WALLET_TRANSFER_DETAIL_FLAG_SPENT uint32_t(1 << 0) +#define WALLET_TRANSFER_DETAIL_FLAG_BLOCKED uint32_t(1 << 1) +#define WALLET_TRANSFER_DETAIL_FLAG_ESCROW_PROPOSAL_RESERVATION uint32_t(1 << 2) +#define WALLET_TRANSFER_DETAIL_FLAG_MINED_TRANSFER uint32_t(1 << 3) +#define WALLET_TRANSFER_DETAIL_FLAG_COLD_SIG_RESERVATION uint32_t(1 << 4) // transfer is reserved for cold-signing (unsigned tx was created and passed for signing) + + const uint64_t WALLET_MINIMUM_HEIGHT_UNSET_CONST = std::numeric_limits::max(); +const uint64_t WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED = std::numeric_limits::max(); #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "wallet" @@ -346,11 +354,6 @@ namespace tools m_core_runtime_config = currency::get_default_core_runtime_config(); }; -#define WALLET_TRANSFER_DETAIL_FLAG_SPENT uint32_t(1 << 0) -#define WALLET_TRANSFER_DETAIL_FLAG_BLOCKED uint32_t(1 << 1) -#define WALLET_TRANSFER_DETAIL_FLAG_ESCROW_PROPOSAL_RESERVATION uint32_t(1 << 2) -#define WALLET_TRANSFER_DETAIL_FLAG_MINED_TRANSFER uint32_t(1 << 3) -#define WALLET_TRANSFER_DETAIL_FLAG_COLD_SIG_RESERVATION uint32_t(1 << 4) // transfer is reserved for cold-signing (unsigned tx was created and passed for signing) static std::string transfer_flags_to_str(uint32_t flags); static std::string transform_tx_to_str(const currency::transaction& tx); From 8f9f9f1698fbbebff382632cf25500cebe666c73 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 12 Jun 2020 00:32:08 +0200 Subject: [PATCH 2/8] implemented deferred fetching of global output indexes on receiving phase(preparation phase is todo) --- contrib/epee/include/misc_language.h | 26 +++++++ src/currency_core/blockchain_storage.cpp | 4 +- src/currency_core/blockchain_storage.h | 4 +- src/currency_core/currency_format_utils.cpp | 12 +++ .../currency_protocol_defs.h | 1 + src/rpc/core_rpc_server.cpp | 3 +- src/wallet/wallet2.cpp | 78 ++++++++++++++----- src/wallet/wallet2.h | 3 +- 8 files changed, 106 insertions(+), 25 deletions(-) diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index 6c2a287d..1a02c7c8 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -77,6 +77,32 @@ namespace epee namespace misc_utils { + template + struct triple + { // store a pair of values + typedef _Ty1 first_type; + typedef _Ty2 second_type; + typedef _Ty3 third_type; + + triple() + : first(), second(), third() + { // default construct + } + + triple(const _Ty1& _Val1, const _Ty2& _Val2, const _Ty3& _Val3) + : first(_Val1), second(_Val2), third(_Val3) + { // construct from specified values + } + + _Ty1 first; // the first stored value + _Ty2 second; // the second stored value + _Ty3 third; // the second stored value + }; + + + template t_type get_max_t_val(t_type t) { diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index e59a8f9d..0ae47e21 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -3112,7 +3112,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height)const +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height, bool request_coinbase_info)const { CRITICAL_REGION_LOCAL(m_read_lock); if (!find_blockchain_supplement(qblock_ids, start_height)) @@ -3129,6 +3129,8 @@ bool blockchain_storage::find_blockchain_supplement(const std::list mis; get_transactions_direct(m_db_blocks[i]->bl.tx_hashes, blocks.back().second, mis); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, block " << get_block_hash(m_db_blocks[i]->bl) << " [" << i << "] contains missing transactions: " << mis); + if(request_coinbase_info) + blocks.back().third = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx)); } return true; } diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 73c9af2b..5f83c509 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -168,7 +168,7 @@ namespace currency typedef tools::db::basic_key_to_array_accessor outputs_container; // out_amount => ['global_output', ...] typedef tools::db::cached_key_value_accessor key_images_container; - typedef std::list, std::list > > > blocks_direct_container; + typedef std::list, std::list >, std::shared_ptr > > blocks_direct_container; friend struct add_transaction_input_visitor; //--------------------------------------------------------------------------------- @@ -255,7 +255,7 @@ namespace currency bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const; bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset)const; bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0, bool need_global_indexes = false)const; - bool find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0)const; + bool find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0, bool request_coinbase_info = false)const; //bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp)const; bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res)const; diff --git a/src/currency_core/currency_format_utils.cpp b/src/currency_core/currency_format_utils.cpp index d51b7821..db87f721 100644 --- a/src/currency_core/currency_format_utils.cpp +++ b/src/currency_core/currency_format_utils.cpp @@ -1739,12 +1739,24 @@ namespace currency bool r = currency::parse_and_validate_block_from_blob(bl_entry.block, blextin_ptr->bl); bdde.block_ptr = blextin_ptr; CHECK_AND_ASSERT_MES(r, false, "failed to parse block from blob: " << string_tools::buff_to_hex_nodelimer(bl_entry.block)); + size_t i = 0; + if (bl_entry.tx_global_outs.size()) + { + CHECK_AND_ASSERT_MES(bl_entry.tx_global_outs.size() == bl_entry.txs.size(), false, "tx_global_outs count " << bl_entry.tx_global_outs.size() << " count missmatch with bl_entry.txs count " << bl_entry.txs.size()); + } + for (const auto& tx_blob : bl_entry.txs) { std::shared_ptr tche_ptr(new currency::transaction_chain_entry()); r = parse_and_validate_tx_from_blob(tx_blob, tche_ptr->tx); CHECK_AND_ASSERT_MES(r, false, "failed to parse tx from blob: " << string_tools::buff_to_hex_nodelimer(tx_blob)); bdde.txs_ptr.push_back(tche_ptr); + if (bl_entry.tx_global_outs.size()) + { + CHECK_AND_ASSERT_MES(bl_entry.tx_global_outs[i].v.size() == tche_ptr->tx.vout.size(), false, "tx_global_outs for tx" << bl_entry.tx_global_outs[i].v.size() << " count missmatch with tche_ptr->tx.vout.size() count " << tche_ptr->tx.vout.size()); + tche_ptr->m_global_output_indexes = bl_entry.tx_global_outs[i].v; + } + i++; } } return true; diff --git a/src/currency_protocol/currency_protocol_defs.h b/src/currency_protocol/currency_protocol_defs.h index 0bb974e0..65e4a5b7 100644 --- a/src/currency_protocol/currency_protocol_defs.h +++ b/src/currency_protocol/currency_protocol_defs.h @@ -41,6 +41,7 @@ namespace currency struct block_direct_data_entry { std::shared_ptr block_ptr; + std::shared_ptr coinbase_ptr; std::list > txs_ptr; }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index ae353bc2..e98918a4 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -279,7 +279,7 @@ namespace currency } blockchain_storage::blocks_direct_container bs; - if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height)) + if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height, req.need_global_indexes)) { res.status = API_RETURN_CODE_FAIL; return false; @@ -290,6 +290,7 @@ namespace currency res.blocks.resize(res.blocks.size()+1); res.blocks.back().block_ptr = b.first; res.blocks.back().txs_ptr = std::move(b.second); + res.blocks.back().coinbase_ptr = b.third; } res.status = API_RETURN_CODE_OK; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 687839e0..00fa9f7c 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -239,7 +239,24 @@ size_t wallet2::scan_for_transaction_entries(const crypto::hash& tx_id, const cr return details.size(); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t height, const currency::block& b) +void wallet2::fetch_tx_global_indixes(const currency::transaction& tx, std::vector& goutputs_indexes) +{ + currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request req = AUTO_VAL_INIT(req); + currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response res = AUTO_VAL_INIT(res); + req.txid = get_transaction_hash(tx); + bool r = m_core_proxy->call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(req, res); + THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "get_o_indexes.bin"); + THROW_IF_TRUE_WALLET_EX(res.status == API_RETURN_CODE_BUSY, error::daemon_busy, "get_o_indexes.bin"); + THROW_IF_TRUE_WALLET_EX(res.status != API_RETURN_CODE_OK, error::get_out_indices_error, res.status); + THROW_IF_TRUE_WALLET_EX(res.o_indexes.size() != tx.vout.size(), error::wallet_internal_error, + "transactions outputs size=" + std::to_string(tx.vout.size()) + + " not match with COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES response size=" + std::to_string(res.o_indexes.size())); + + goutputs_indexes = res.o_indexes; +} + +//---------------------------------------------------------------------------------------------------- +void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t height, const currency::block& b, const std::vector* pglobal_indexes) { std::vector recipients, recipients_aliases; process_unconfirmed(tx, recipients, recipients_aliases); @@ -365,20 +382,23 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t pwallet_info->m_block_height = height; pwallet_info->m_block_timestamp = b.timestamp; + if (is_auditable()) + { + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(pglobal_indexes && pglobal_indexes->size() == tx.vout.size(), "wrong pglobal_indexes = " << pglobal_indexes << ""); + } + std::vector outputs_index_local; + + if (!pglobal_indexes) + { #ifndef MOBILE_WALLET_BUILD - //good news - got money! take care about it - //usually we have only one transfer for user in transaction - currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request req = AUTO_VAL_INIT(req); - currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response res = AUTO_VAL_INIT(res); - req.txid = get_transaction_hash(tx); - bool r = m_core_proxy->call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(req, res); - THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "get_o_indexes.bin"); - THROW_IF_TRUE_WALLET_EX(res.status == API_RETURN_CODE_BUSY, error::daemon_busy, "get_o_indexes.bin"); - THROW_IF_TRUE_WALLET_EX(res.status != API_RETURN_CODE_OK, error::get_out_indices_error, res.status); - THROW_IF_TRUE_WALLET_EX(res.o_indexes.size() != tx.vout.size(), error::wallet_internal_error, - "transactions outputs size=" + std::to_string(tx.vout.size()) + - " not match with COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES response size=" + std::to_string(res.o_indexes.size())); + //good news - got money! take care about it + //usually we have only one transfer for user in transaction + fetch_tx_global_indixes(tx, outputs_index_local); + pglobal_indexes = &outputs_index_local; #endif + } + + for (size_t i_in_outs = 0; i_in_outs != outs.size(); i_in_outs++) { @@ -458,9 +478,13 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t td.m_internal_output_index = o; td.m_key_image = ki; #ifdef MOBILE_WALLET_BUILD - td.m_global_output_index = WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED; + if (pglobal_indexes && pglobal_indexes->size() > o) + td.m_global_output_index = (*pglobal_indexes)[o]; + else + td.m_global_output_index = WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED; #else - td.m_global_output_index = res.o_indexes[o]; + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(pglobal_indexes, "pglobal_indexes IS NULL in non mobile wallet"); + td.m_global_output_index = (*pglobal_indexes)[o]; #endif if (coin_base_tx) @@ -477,9 +501,13 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t add_transfer_to_transfers_cache(tx.vout[o].amount, transfer_index); uint64_t amount = tx.vout[o].amount; - auto amount_gindex_pair = std::make_pair(amount, td.m_global_output_index); - WLT_CHECK_AND_ASSERT_MES_NO_RET(m_amount_gindex_to_transfer_id.count(amount_gindex_pair) == 0, "update m_amount_gindex_to_transfer_id: amount " << amount << ", gindex " << td.m_global_output_index << " already exists"); - m_amount_gindex_to_transfer_id[amount_gindex_pair] = transfer_index; + if (is_watch_only() && is_auditable()) + { + WLT_CHECK_AND_ASSERT_MES_NO_RET(td.m_global_output_index != WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED, "td.m_global_output_index != WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED validation failed"); + auto amount_gindex_pair = std::make_pair(amount, td.m_global_output_index); + WLT_CHECK_AND_ASSERT_MES_NO_RET(m_amount_gindex_to_transfer_id.count(amount_gindex_pair) == 0, "update m_amount_gindex_to_transfer_id: amount " << amount << ", gindex " << td.m_global_output_index << " already exists"); + m_amount_gindex_to_transfer_id[amount_gindex_pair] = transfer_index; + } if (max_out_unlock_time < get_tx_unlock_time(tx, o)) max_out_unlock_time = get_tx_unlock_time(tx, o); @@ -1161,16 +1189,22 @@ void wallet2::process_new_blockchain_entry(const currency::block& b, const curre "current_index=" + std::to_string(height) + ", get_blockchain_current_height()=" + std::to_string(get_blockchain_current_size())); //optimization: seeking only for blocks that are not older then the wallet creation time plus 1 day. 1 day is for possible user incorrect time setup + const std::vector* pglobal_index = nullptr; if (b.timestamp + 60 * 60 * 24 > m_account.get_createtime()) { + pglobal_index = nullptr; + if (bche.coinbase_ptr.get()) + { + pglobal_index = &(bche.coinbase_ptr->m_global_output_indexes); + } TIME_MEASURE_START(miner_tx_handle_time); - process_new_transaction(b.miner_tx, height, b); + process_new_transaction(b.miner_tx, height, b, pglobal_index); TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); for(const auto& tx_entry: bche.txs_ptr) { - process_new_transaction(tx_entry->tx, height, b); + process_new_transaction(tx_entry->tx, height, b, &(tx_entry->m_global_output_indexes)); } TIME_MEASURE_FINISH(txs_handle_time); WLT_LOG_L3("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); @@ -1245,6 +1279,9 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response res = AUTO_VAL_INIT(res); req.minimum_height = get_wallet_minimum_height(); + if (is_auditable()) + req.need_global_indexes = true; + m_chain.get_short_chain_history(req.block_ids); bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); if (!r) @@ -2865,6 +2902,7 @@ bool wallet2::get_pos_entries(currency::COMMAND_RPC_SCAN_POS::request& req) { for (size_t i = 0; i != m_transfers.size(); i++) { + WLT_CHECK_AND_ASSERT_MES(tr.m_global_output_index != WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED, false, "Wrong output input in transaction"); auto& tr = m_transfers[i]; uint64_t stake_unlock_time = 0; if (!is_transfer_okay_for_pos(tr, stake_unlock_time)) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index e02f7138..98306f11 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -830,7 +830,8 @@ private: void add_transfers_to_expiration_list(const std::vector& selected_transfers, uint64_t expiration, uint64_t change_amount, const crypto::hash& related_tx_id); void remove_transfer_from_expiration_list(uint64_t transfer_index); void load_keys(const std::string& keys_file_name, const std::string& password, uint64_t file_signature); - void process_new_transaction(const currency::transaction& tx, uint64_t height, const currency::block& b); + void process_new_transaction(const currency::transaction& tx, uint64_t height, const currency::block& b, const std::vector* pglobal_indexes); + void fetch_tx_global_indixes(const currency::transaction& tx, std::vector& goutputs_indexes); void detach_blockchain(uint64_t including_height); bool extract_offers_from_transfer_entry(size_t i, std::unordered_map& offers_local); bool select_my_offers(std::list& offers); From 73c42b78929ee0a16317358b177fde8e32d69dba Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 12 Jun 2020 23:32:06 +0200 Subject: [PATCH 3/8] implemented prefetch of outputs before transfer --- src/rpc/core_rpc_server.cpp | 15 ++++-- src/rpc/core_rpc_server_commands_defs.h | 10 ++-- src/wallet/wallet2.cpp | 66 ++++++++++++++++++++++--- src/wallet/wallet2.h | 2 + 4 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index e98918a4..e65af49c 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -379,14 +379,19 @@ namespace currency bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, connection_context& cntx) { CHECK_CORE_READY(); - bool r = m_core.get_tx_outputs_gindexs(req.txid, res.o_indexes); - if(!r) + res.tx_global_outs.resize(req.txids.size()); + size_t i = 0; + for (auto& txid : req.txids) { - res.status = "Failed"; - return true; + bool r = m_core.get_tx_outputs_gindexs(txid, res.tx_global_outs[i].v); + if (!r) + { + res.status = API_RETURN_CODE_FAIL; + return true; + } + i++; } res.status = API_RETURN_CODE_OK; - LOG_PRINT_L2("COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES: [" << res.o_indexes.size() << "]"); return true; } //------------------------------------------------------------------------------------------------------------------------------ diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index c2e22bf6..a362a15c 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -238,19 +238,21 @@ namespace currency { struct request { - crypto::hash txid; + std::list txids; + BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_VAL_POD_AS_BLOB(txid) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(txids) END_KV_SERIALIZE_MAP() }; struct response { - std::vector o_indexes; + std::vector > > tx_global_outs; + //std::vector o_indexes; std::string status; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(o_indexes) + KV_SERIALIZE(tx_global_outs) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 00fa9f7c..0cb042d5 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -240,19 +240,43 @@ size_t wallet2::scan_for_transaction_entries(const crypto::hash& tx_id, const cr } //---------------------------------------------------------------------------------------------------- void wallet2::fetch_tx_global_indixes(const currency::transaction& tx, std::vector& goutputs_indexes) +{ + std::list> txs; + txs.push_back(tx); + std::vector > res; + fetch_tx_global_indixes(txs, res); + THROW_IF_FALSE_WALLET_INT_ERR_EX(res.size() == 1, "fetch_tx_global_indixes for single entry returned wrong result: res.size()=" << res.size()); + goutputs_indexes = res[0]; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::fetch_tx_global_indixes(const std::list>& txs, std::vector>& goutputs_indexes) { currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request req = AUTO_VAL_INIT(req); currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response res = AUTO_VAL_INIT(res); - req.txid = get_transaction_hash(tx); + for (auto& tx : txs) + { + req.txids.push_back(get_transaction_hash(tx)); + } + bool r = m_core_proxy->call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(req, res); THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "get_o_indexes.bin"); THROW_IF_TRUE_WALLET_EX(res.status == API_RETURN_CODE_BUSY, error::daemon_busy, "get_o_indexes.bin"); THROW_IF_TRUE_WALLET_EX(res.status != API_RETURN_CODE_OK, error::get_out_indices_error, res.status); - THROW_IF_TRUE_WALLET_EX(res.o_indexes.size() != tx.vout.size(), error::wallet_internal_error, - "transactions outputs size=" + std::to_string(tx.vout.size()) + - " not match with COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES response size=" + std::to_string(res.o_indexes.size())); + THROW_IF_FALSE_WALLET_INT_ERR_EX(res.tx_global_outs.size() == txs.size(), "res.tx_global_outs.size()(" << res.tx_global_outs.size() + << ") == txs.size()(" << txs.size() << ")"); + goutputs_indexes.resize(txs.size()); + auto it_resp = res.tx_global_outs.begin(); + auto it_txs = txs.begin(); + size_t i = 0; + for (; it_resp != res.tx_global_outs.end();) + { + THROW_IF_FALSE_WALLET_INT_ERR_EX(it_resp->v.size() == it_txs->get().vout.size(), + "transactions outputs size=" << it_txs->get().vout.size() << + " not match with COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES response[i] size=" << it_resp->v.size()); - goutputs_indexes = res.o_indexes; + goutputs_indexes[i] = it_resp->v; + it_resp++; it_txs++; i++; + } } //---------------------------------------------------------------------------------------------------- @@ -2902,8 +2926,8 @@ bool wallet2::get_pos_entries(currency::COMMAND_RPC_SCAN_POS::request& req) { for (size_t i = 0; i != m_transfers.size(); i++) { - WLT_CHECK_AND_ASSERT_MES(tr.m_global_output_index != WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED, false, "Wrong output input in transaction"); auto& tr = m_transfers[i]; + WLT_CHECK_AND_ASSERT_MES(tr.m_global_output_index != WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED, false, "Wrong output input in transaction"); uint64_t stake_unlock_time = 0; if (!is_transfer_okay_for_pos(tr, stake_unlock_time)) continue; @@ -3788,6 +3812,31 @@ bool wallet2::prepare_tx_sources(uint64_t needed_money, size_t fake_outputs_coun return prepare_tx_sources(fake_outputs_count, sources, selected_indicies, found_money); } //---------------------------------------------------------------------------------------------------- +void wallet2::prefetch_global_indicies_if_needed(std::vector& selected_indicies) +{ + std::list> txs; + std::list indices_that_requested_global_indicies; + for (uint64_t i : selected_indicies) + { + if (m_transfers[i].m_global_output_index == WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED) + { + indices_that_requested_global_indicies.push_back(i); + txs.push_back(m_transfers[i].m_ptx_wallet_info->m_tx); + } + } + + std::vector > outputs_for_all_txs; + fetch_tx_global_indixes(txs, outputs_for_all_txs); + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(txs.size() == outputs_for_all_txs.size(), "missmatch sizes txs.size() == outputs_for_all_txs.size()"); + auto it_indices = indices_that_requested_global_indicies.begin(); + auto it_ooutputs = outputs_for_all_txs.begin(); + for (; it_ooutputs != outputs_for_all_txs.end();) + { + transfer_details& td = m_transfers[*it_indices]; + td.m_global_output_index = (*it_ooutputs)[td.m_internal_output_index]; + } +} +//---------------------------------------------------------------------------------------------------- bool wallet2::prepare_tx_sources(size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money) { typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; @@ -3826,6 +3875,9 @@ bool wallet2::prepare_tx_sources(size_t fake_outputs_count, std::vector* pglobal_indexes); void fetch_tx_global_indixes(const currency::transaction& tx, std::vector& goutputs_indexes); + void fetch_tx_global_indixes(const std::list>& txs, std::vector>& goutputs_indexes); void detach_blockchain(uint64_t including_height); bool extract_offers_from_transfer_entry(size_t i, std::unordered_map& offers_local); bool select_my_offers(std::list& offers); @@ -883,6 +884,7 @@ private: bool prepare_tx_sources(size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); bool prepare_tx_sources(crypto::hash multisig_id, std::vector& sources, uint64_t& found_money); bool prepare_tx_sources_for_packing(uint64_t items_to_pack, size_t fake_outputs_count, std::vector& sources, std::vector& selected_indicies, uint64_t& found_money); + void prefetch_global_indicies_if_needed(std::vector& selected_indicies); uint64_t get_needed_money(uint64_t fee, const std::vector& dsts); void prepare_tx_destinations(uint64_t needed_money, uint64_t found_money, From 53bdb6417e2b716e013782264c17a7bd82ec4f57 Mon Sep 17 00:00:00 2001 From: sowle Date: Sat, 13 Jun 2020 12:13:41 +0300 Subject: [PATCH 4/8] marking place for prefetch --- src/wallet/wallet2.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 0cb042d5..ecbb04e1 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -4789,6 +4789,10 @@ void wallet2::sweep_below(size_t fake_outs_count, const currency::account_public if (selected_transfers.size() > tx_sources_for_querying_random_outs_max) selected_transfers.erase(selected_transfers.begin() + tx_sources_for_querying_random_outs_max, selected_transfers.end()); + // + // TODO: prefetch gindexes here for each element of selected_transfers + // + typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; typedef currency::tx_source_entry::output_entry tx_output_entry; From 0f98b6d9906a075d47915736d6f807526ea18bf5 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 13 Jun 2020 21:04:21 +0200 Subject: [PATCH 5/8] fixed few bugs related to prefetch --- src/wallet/wallet2.cpp | 39 +++++++++++---------- src/wallet/wallet2.h | 5 ++- tests/core_tests/chaingen.cpp | 12 +++++-- tests/core_tests/wallet_test_core_proxy.cpp | 16 ++++++--- 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index ecbb04e1..ac6c256e 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -400,6 +400,9 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t if(!outs.empty() /*&& tx_money_got_in_outs*/) { + //good news - got money! take care about it + //usually we have only one transfer for user in transaction + //create once instance of tx for all entries std::shared_ptr pwallet_info(new transaction_wallet_info()); pwallet_info->m_tx = tx; @@ -414,16 +417,13 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t if (!pglobal_indexes) { -#ifndef MOBILE_WALLET_BUILD - //good news - got money! take care about it - //usually we have only one transfer for user in transaction - fetch_tx_global_indixes(tx, outputs_index_local); - pglobal_indexes = &outputs_index_local; -#endif + if (!m_use_deffered_global_outputs) + { + fetch_tx_global_indixes(tx, outputs_index_local); + pglobal_indexes = &outputs_index_local; + } } - - - + for (size_t i_in_outs = 0; i_in_outs != outs.size(); i_in_outs++) { size_t o = outs[i_in_outs]; @@ -501,16 +501,18 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t td.m_ptx_wallet_info = pwallet_info; td.m_internal_output_index = o; td.m_key_image = ki; -#ifdef MOBILE_WALLET_BUILD - if (pglobal_indexes && pglobal_indexes->size() > o) - td.m_global_output_index = (*pglobal_indexes)[o]; + if (m_use_deffered_global_outputs) + { + if (pglobal_indexes && pglobal_indexes->size() > o) + td.m_global_output_index = (*pglobal_indexes)[o]; + else + td.m_global_output_index = WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED; + } else - td.m_global_output_index = WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED; -#else - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(pglobal_indexes, "pglobal_indexes IS NULL in non mobile wallet"); - td.m_global_output_index = (*pglobal_indexes)[o]; -#endif - + { + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(pglobal_indexes, "pglobal_indexes IS NULL in non mobile wallet"); + td.m_global_output_index = (*pglobal_indexes)[o]; + } if (coin_base_tx) { //last out in coinbase tx supposed to be change from coinstake @@ -3834,6 +3836,7 @@ void wallet2::prefetch_global_indicies_if_needed(std::vector& selected { transfer_details& td = m_transfers[*it_indices]; td.m_global_output_index = (*it_ooutputs)[td.m_internal_output_index]; + it_ooutputs++; it_indices++; } } //---------------------------------------------------------------------------------------------------- diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 2bd2e38b..4b0461d4 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -333,7 +333,8 @@ namespace tools m_last_pow_block_h(0), m_minimum_height(WALLET_MINIMUM_HEIGHT_UNSET_CONST), m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE), - m_current_wallet_file_size(0) + m_current_wallet_file_size(0), + m_use_deffered_global_outputs(false) {}; public: wallet2() : m_stop(false), @@ -824,6 +825,7 @@ namespace tools bool get_utxo_distribution(std::map& distribution); uint64_t get_sync_progress(); uint64_t get_wallet_file_size()const; + void set_use_deffered_global_outputs(bool use) { m_use_deffered_global_outputs = use;} private: @@ -1001,6 +1003,7 @@ private: std::string m_miner_text_info; mutable uint64_t m_current_wallet_file_size; + bool m_use_deffered_global_outputs; //this needed to access wallets state in coretests, for creating abnormal blocks and tranmsactions friend class test_generator; diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index b5921344..7270a4af 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -384,10 +384,16 @@ bool test_generator::build_wallets(const blockchain_vector& blocks, {} virtual bool call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(const currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& rqt, currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& rsp) { - auto it = m_txs_outs.find(rqt.txid); - CHECK_AND_ASSERT_MES(it != m_txs_outs.end(), false, "tx " << rqt.txid << " was not found in tx global outout indexes"); + rsp.tx_global_outs.resize(rqt.txids.size()); + size_t i = 0; + for (auto& txid : rqt.txids) + { + auto it = m_txs_outs.find(txid); + CHECK_AND_ASSERT_MES(it != m_txs_outs.end(), false, "tx " << txid << " was not found in tx global outout indexes"); + rsp.tx_global_outs[i].v = it->second; + i++; + } rsp.status = API_RETURN_CODE_OK; - rsp.o_indexes = it->second; return true; } }; diff --git a/tests/core_tests/wallet_test_core_proxy.cpp b/tests/core_tests/wallet_test_core_proxy.cpp index 0ebe76fa..aeb7727d 100644 --- a/tests/core_tests/wallet_test_core_proxy.cpp +++ b/tests/core_tests/wallet_test_core_proxy.cpp @@ -45,12 +45,18 @@ bool wallet_test_core_proxy::update_blockchain(const std::vectorsecond; + res.tx_global_outs.resize(req.txids.size()); + size_t i = 0; + for (auto& txid : req.txids) + { + auto it = m_txs_outs.find(txid); + CHECK_AND_ASSERT_MES(it != m_txs_outs.end(), false, "tx " << txid << " was not found in tx global outout indexes"); + res.tx_global_outs[i].v = it->second; + i++; + } + res.status = API_RETURN_CODE_OK; return true; } From a4cb3e6a327d1b9ff2eb3eed05e3a6c63a0bd696 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sun, 14 Jun 2020 02:17:10 +0200 Subject: [PATCH 6/8] fixed conditions for deffered outs fetch --- src/wallet/wallet2.cpp | 43 +++++++++++++++------ src/wallet/wallet2.h | 32 +-------------- tests/core_tests/wallet_test_core_proxy.cpp | 3 +- 3 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index ac6c256e..983fc872 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -35,7 +35,24 @@ using namespace currency; ENABLE_CHANNEL_BY_DEFAULT("wallet") namespace tools { - + wallet2::wallet2() : m_stop(false), + m_wcallback(new i_wallet2_callback()), //stub + m_core_proxy(new default_http_core_proxy()), + m_upper_transaction_size_limit(0), + m_height_of_start_sync(0), + m_last_sync_percent(0), + m_fake_outputs_count(0), + m_do_rise_transfer(false), + m_log_prefix("???"), + m_watch_only(false), + m_last_pow_block_h(0), + m_minimum_height(WALLET_MINIMUM_HEIGHT_UNSET_CONST), + m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE), + m_current_wallet_file_size(0), + m_use_deffered_global_outputs(false) + { + m_core_runtime_config = currency::get_default_core_runtime_config(); + } //--------------------------------------------------------------- uint64_t wallet2::get_max_unlock_time_from_receive_indices(const currency::transaction& tx, const money_transfer2_details& td) { @@ -415,9 +432,13 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t } std::vector outputs_index_local; - if (!pglobal_indexes) + if (!pglobal_indexes || (pglobal_indexes->size() == 0 && tx.vout.size() != 0)) { - if (!m_use_deffered_global_outputs) + if (m_use_deffered_global_outputs) + { + pglobal_indexes = nullptr; + } + else { fetch_tx_global_indixes(tx, outputs_index_local); pglobal_indexes = &outputs_index_local; @@ -501,6 +522,7 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t td.m_ptx_wallet_info = pwallet_info; td.m_internal_output_index = o; td.m_key_image = ki; + LOG_PRINT_L0("pglobal_indexes = " << pglobal_indexes); if (m_use_deffered_global_outputs) { if (pglobal_indexes && pglobal_indexes->size() > o) @@ -511,6 +533,7 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t else { WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(pglobal_indexes, "pglobal_indexes IS NULL in non mobile wallet"); + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(pglobal_indexes->size() > o, "pglobal_indexes size()(" << pglobal_indexes->size() << ") <= o " << o ); td.m_global_output_index = (*pglobal_indexes)[o]; } if (coin_base_tx) @@ -4768,9 +4791,9 @@ void wallet2::sweep_below(size_t fake_outs_count, const currency::account_public amount_total = 0; outs_swept = 0; - std::vector selected_transfers; + std::vector selected_transfers; selected_transfers.reserve(m_transfers.size()); - for (size_t i = 0; i < m_transfers.size(); ++i) + for (uint64_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; uint64_t amount = td.amount(); @@ -4786,15 +4809,13 @@ void wallet2::sweep_below(size_t fake_outs_count, const currency::account_public WLT_THROW_IF_FALSE_WALLET_CMN_ERR_EX(!selected_transfers.empty(), "No spendable outputs meet the criterion"); // sort by amount descending in order to spend bigger outputs first - std::sort(selected_transfers.begin(), selected_transfers.end(), [this](size_t a, size_t b) { return m_transfers[b].amount() < m_transfers[a].amount(); }); + std::sort(selected_transfers.begin(), selected_transfers.end(), [this](uint64_t a, uint64_t b) { return m_transfers[b].amount() < m_transfers[a].amount(); }); // limit RPC request with reasonable number of sources if (selected_transfers.size() > tx_sources_for_querying_random_outs_max) selected_transfers.erase(selected_transfers.begin() + tx_sources_for_querying_random_outs_max, selected_transfers.end()); - // - // TODO: prefetch gindexes here for each element of selected_transfers - // + prefetch_global_indicies_if_needed(selected_transfers); typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; typedef currency::tx_source_entry::output_entry tx_output_entry; @@ -4805,7 +4826,7 @@ void wallet2::sweep_below(size_t fake_outs_count, const currency::account_public COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req = AUTO_VAL_INIT(req); req.use_forced_mix_outs = false; req.outs_count = fake_outs_count + 1; - for (size_t i : selected_transfers) + for (uint64_t i : selected_transfers) req.amounts.push_back(m_transfers[i].amount()); r = m_core_proxy->call_COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS(req, rpc_get_random_outs_resp); @@ -4858,7 +4879,7 @@ void wallet2::sweep_below(size_t fake_outs_count, const currency::account_public for (size_t st_index = 0; st_index < st_index_upper_boundary; ++st_index) { currency::tx_source_entry& src = ftp.sources[st_index]; - size_t tr_index = selected_transfers[st_index]; + uint64_t tr_index = selected_transfers[st_index]; transfer_details& td = m_transfers[tr_index]; src.transfer_index = tr_index; src.amount = td.amount(); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 4b0461d4..4c2fb43f 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -324,37 +324,9 @@ namespace tools class wallet2 { - wallet2(const wallet2&) : m_stop(false), - m_wcallback(new i_wallet2_callback()), - m_height_of_start_sync(0), - m_last_sync_percent(0), - m_do_rise_transfer(false), - m_watch_only(false), - m_last_pow_block_h(0), - m_minimum_height(WALLET_MINIMUM_HEIGHT_UNSET_CONST), - m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE), - m_current_wallet_file_size(0), - m_use_deffered_global_outputs(false) - {}; + wallet2(const wallet2&) = delete; public: - wallet2() : m_stop(false), - m_wcallback(new i_wallet2_callback()), //stub - m_core_proxy(new default_http_core_proxy()), - m_upper_transaction_size_limit(0), - m_height_of_start_sync(0), - m_last_sync_percent(0), - m_fake_outputs_count(0), - m_do_rise_transfer(false), - m_log_prefix("???"), - m_watch_only(false), - m_last_pow_block_h(0), - m_minimum_height(WALLET_MINIMUM_HEIGHT_UNSET_CONST), - m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE), - m_current_wallet_file_size(0) - { - m_core_runtime_config = currency::get_default_core_runtime_config(); - }; - + wallet2(); static std::string transfer_flags_to_str(uint32_t flags); static std::string transform_tx_to_str(const currency::transaction& tx); diff --git a/tests/core_tests/wallet_test_core_proxy.cpp b/tests/core_tests/wallet_test_core_proxy.cpp index aeb7727d..ee9d1363 100644 --- a/tests/core_tests/wallet_test_core_proxy.cpp +++ b/tests/core_tests/wallet_test_core_proxy.cpp @@ -104,6 +104,8 @@ bool wallet_test_core_proxy::call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency:: { currency::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); req.block_ids = rqt.block_ids; + req.minimum_height = rqt.minimum_height; + req.need_global_indexes = rqt.need_global_indexes; currency::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); bool r = this->call_COMMAND_RPC_GET_BLOCKS_FAST(req, res); rsp.status = res.status; @@ -114,7 +116,6 @@ bool wallet_test_core_proxy::call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency:: r = unserialize_block_complete_entry(res, rsp); } return r; - } bool wallet_test_core_proxy::call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(const currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& rqt, currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& rsp) From 925db8ceebe20d2d8ce82e73d14160ed161bee28 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sun, 14 Jun 2020 19:31:10 +0200 Subject: [PATCH 7/8] fixed coretests(all working in both deffered and normla mode) --- src/wallet/wallet2.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 983fc872..a2efb6cc 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -49,7 +49,7 @@ namespace tools m_minimum_height(WALLET_MINIMUM_HEIGHT_UNSET_CONST), m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE), m_current_wallet_file_size(0), - m_use_deffered_global_outputs(false) + m_use_deffered_global_outputs(true) { m_core_runtime_config = currency::get_default_core_runtime_config(); } @@ -2952,10 +2952,20 @@ bool wallet2::get_pos_entries(currency::COMMAND_RPC_SCAN_POS::request& req) for (size_t i = 0; i != m_transfers.size(); i++) { auto& tr = m_transfers[i]; - WLT_CHECK_AND_ASSERT_MES(tr.m_global_output_index != WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED, false, "Wrong output input in transaction"); + uint64_t stake_unlock_time = 0; if (!is_transfer_okay_for_pos(tr, stake_unlock_time)) continue; + + if (tr.m_global_output_index == WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED) + { + //TODO: this code needed mostly for coretests, since in real life cases only mobile wallet supposed to + // have WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED, and mobile wallet is not supposed to do PoS mining + std::vector indicies; indicies.push_back(i); + prefetch_global_indicies_if_needed(indicies); + } + //WLT_CHECK_AND_ASSERT_MES(tr.m_global_output_index != WALLET_GLOBAL_OUTPUT_INDEX_UNDEFINED, false, "Wrong output input in transaction"); + currency::pos_entry pe = AUTO_VAL_INIT(pe); pe.amount = tr.amount(); pe.index = tr.m_global_output_index; From 6a967bf6184e111c9773338963d761ecb7906ee1 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sun, 14 Jun 2020 19:46:06 +0200 Subject: [PATCH 8/8] configured mobile native library for using deffered loading of global outputs indicies --- src/wallet/plain_wallet_api.cpp | 2 ++ src/wallet/wallet2.cpp | 2 +- src/wallet/wallets_manager.cpp | 6 +++++- src/wallet/wallets_manager.h | 4 +++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/wallet/plain_wallet_api.cpp b/src/wallet/plain_wallet_api.cpp index 1073919a..f8d0fdbe 100644 --- a/src/wallet/plain_wallet_api.cpp +++ b/src/wallet/plain_wallet_api.cpp @@ -207,6 +207,8 @@ namespace plain_wallet return GENERAL_INTERNAL_ERRROR_INIT; } + ptr->gwm.set_use_deffered_global_outputs(true); + if(!ptr->gwm.start()) { LOG_ERROR("Failed to start wallets_manager"); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index a2efb6cc..e2eed54c 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -49,7 +49,7 @@ namespace tools m_minimum_height(WALLET_MINIMUM_HEIGHT_UNSET_CONST), m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE), m_current_wallet_file_size(0), - m_use_deffered_global_outputs(true) + m_use_deffered_global_outputs(false) { m_core_runtime_config = currency::get_default_core_runtime_config(); } diff --git a/src/wallet/wallets_manager.cpp b/src/wallet/wallets_manager.cpp index 39a3398c..f900f1ce 100644 --- a/src/wallet/wallets_manager.cpp +++ b/src/wallet/wallets_manager.cpp @@ -63,7 +63,8 @@ wallets_manager::wallets_manager():m_pview(&m_view_stub), m_remote_node_mode(false), m_is_pos_allowed(false), m_qt_logs_enbaled(false), - m_dont_save_wallet_at_stop(false) + m_dont_save_wallet_at_stop(false), + m_use_deffered_global_outputs(false) { #ifndef MOBILE_WALLET_BUILD m_offers_service.set_disabled(true); @@ -814,6 +815,7 @@ std::string wallets_manager::open_wallet(const std::wstring& path, const std::st std::shared_ptr w(new tools::wallet2()); + w->set_use_deffered_global_outputs(m_use_deffered_global_outputs); owr.wallet_id = m_wallet_id_counter++; w->callback(std::shared_ptr(new i_wallet_to_i_backend_adapter(this, owr.wallet_id))); @@ -927,6 +929,7 @@ std::string wallets_manager::get_recent_transfers(size_t wallet_id, uint64_t off std::string wallets_manager::generate_wallet(const std::wstring& path, const std::string& password, view::open_wallet_response& owr) { std::shared_ptr w(new tools::wallet2()); + w->set_use_deffered_global_outputs(m_use_deffered_global_outputs); owr.wallet_id = m_wallet_id_counter++; w->callback(std::shared_ptr(new i_wallet_to_i_backend_adapter(this, owr.wallet_id))); if (m_remote_node_mode) @@ -1012,6 +1015,7 @@ void wallets_manager::get_gui_options(view::gui_options& opt) std::string wallets_manager::restore_wallet(const std::wstring& path, const std::string& password, const std::string& restore_key, bool auditable_watch_only, view::open_wallet_response& owr) { std::shared_ptr w(new tools::wallet2()); + w->set_use_deffered_global_outputs(m_use_deffered_global_outputs); owr.wallet_id = m_wallet_id_counter++; w->callback(std::shared_ptr(new i_wallet_to_i_backend_adapter(this, owr.wallet_id))); if (m_remote_node_mode) diff --git a/src/wallet/wallets_manager.h b/src/wallet/wallets_manager.h index af037621..d2b93d73 100644 --- a/src/wallet/wallets_manager.h +++ b/src/wallet/wallets_manager.h @@ -63,6 +63,7 @@ public: std::atomic stop_for_refresh; //use separate var for passing to "refresh" member function, //because it can be changed there due to internal interruption logis + std::atomic break_mining_loop; std::atomic wallet_state; std::atomic last_wallet_synch_height; @@ -154,7 +155,7 @@ public: void get_gui_options(view::gui_options& opt); std::string get_wallet_log_prefix(size_t wallet_id) const; bool is_qt_logs_enabled() const { return m_qt_logs_enbaled; } - + void set_use_deffered_global_outputs(bool use) { m_use_deffered_global_outputs = use; } private: void main_worker(const po::variables_map& vm); bool init_local_daemon(); @@ -186,6 +187,7 @@ private: std::shared_ptr m_rpc_proxy; po::variables_map m_vm; + bool m_use_deffered_global_outputs; std::atomic m_last_daemon_height; std::atomic m_last_daemon_network_state; std::atomic m_last_daemon_is_disconnected;