From 4825b7c01262b65007bf111ae386fe6090eb29b9 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 24 Apr 2020 01:05:36 +0200 Subject: [PATCH 01/15] initial code for upgrading walle sync protocol --- src/currency_core/currency_config.h | 3 +- src/wallet/wallet2.cpp | 85 +++++++++++++++++++++++------ src/wallet/wallet2.h | 34 +++++++++++- 3 files changed, 100 insertions(+), 22 deletions(-) diff --git a/src/currency_core/currency_config.h b/src/currency_core/currency_config.h index 8de1ee32..0fb51938 100644 --- a/src/currency_core/currency_config.h +++ b/src/currency_core/currency_config.h @@ -215,7 +215,8 @@ #define BC_OFFERS_CURRENCY_MARKET_FILENAME "market.bin" -#define WALLET_FILE_SERIALIZATION_VERSION (CURRENCY_FORMATION_VERSION+65) +#define WALLET_FILE_SERIALIZATION_VERSION (CURRENCY_FORMATION_VERSION+66) + #define CURRENT_MEMPOOL_ARCHIVE_VER (CURRENCY_FORMATION_VERSION+31) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 5aaf395f..a418be65 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1070,8 +1070,8 @@ void wallet2::process_unconfirmed(const currency::transaction& tx, std::vectoron_new_block(height, b); } //---------------------------------------------------------------------------------------------------- +void wallet2::push_new_block_id(const crypto::hash& id, uint64_t height) +{ + //primary 10 + //self check + if (!m_last_10_blocks.empty()) + { + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(get_blockchain_current_height() == height, "Inernal error: get_blockchain_current_height(){" << get_blockchain_current_height() << "} == height{" << height << "} is not equal"); + } + + m_last_10_blocks[height] = id; + if (m_last_10_blocks.size() > WALLET_EVERYBLOCK_SIZE) + { + m_last_10_blocks.erase(m_last_10_blocks.begin()); + } + + //every 10-th + if (height % 10 == 0) + { + //self check + if (!m_last_144_blocks_every_10.empty()) + { + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_10.end())->first + 10 == height, "Inernal error: (--m_last_144_blocks_every_10.end())->first + 10{" << (--m_last_144_blocks_every_10.end())->first + 10 << "} == height{" << height << "} is not equal"); + } + m_last_144_blocks_every_10[height] = id; + } + //every 100-th + if (height % 100 == 0) + { + //self check + if (!m_last_144_blocks_every_100.empty()) + { + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_100.end())->first + 100 == height, "Inernal error: (--m_last_144_blocks_every_100.end())->first + 100{" << (--m_last_144_blocks_every_100.end())->first + 100 << "} == height{" << height << "} is not equal"); + } + m_last_144_blocks_every_100[height] = id; + } + //every 1000-th + //every 100-th + if (height % 1000 == 0) + { + //self check + if (!m_last_144_blocks_every_1000.empty()) + { + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_1000.end())->first + 1000 == height, "Inernal error: (--m_last_144_blocks_every_1000.end())->first + 1000{" << (--m_last_144_blocks_every_1000.end())->first + 1000 << "} == height{" << height << "} is not equal"); + } + m_last_144_blocks_every_1000[height] = id; + } + +} +//---------------------------------------------------------------------------------------------------- void wallet2::get_short_chain_history(std::list& ids) { ids.clear(); size_t i = 0; size_t current_multiplier = 1; - size_t sz = m_blockchain.size(); + size_t sz = get_blockchain_current_height(); if(!sz) return; size_t current_back_offset = 1; @@ -1168,9 +1217,9 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) return; } THROW_IF_TRUE_WALLET_EX(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, res.status); - THROW_IF_TRUE_WALLET_EX(m_blockchain.size() && m_blockchain.size() <= res.start_height, error::wallet_internal_error, + THROW_IF_TRUE_WALLET_EX(get_blockchain_current_height() && get_blockchain_current_height() <= res.start_height, error::wallet_internal_error, "wrong daemon response: m_start_height=" + std::to_string(res.start_height) + - " not less than local blockchain size=" + std::to_string(m_blockchain.size())); + " not less than local blockchain size=" + std::to_string(get_blockchain_current_height())); handle_pulled_blocks(blocks_added, stop, res); } @@ -1180,7 +1229,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop { size_t current_index = res.start_height; - if (res.start_height == 0 && m_blockchain.size() == 1 && !res.blocks.empty()) + if (res.start_height == 0 && get_blockchain_current_height() == 1 && !res.blocks.empty()) { const currency::block& genesis = res.blocks.front().block_ptr->bl; THROW_IF_TRUE_WALLET_EX(get_block_height(genesis) != 0, error::wallet_internal_error, "first block expected to be genesis"); @@ -1199,7 +1248,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop //TODO: get_block_hash is slow crypto::hash bl_id = get_block_hash(bl); - if (current_index >= m_blockchain.size()) + if (current_index >= get_blockchain_current_height()) { process_new_blockchain_entry(bl, bl_entry, bl_id, current_index); ++blocks_added; @@ -1233,7 +1282,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop } } - WLT_LOG_L1("[PULL BLOCKS] " << res.start_height << " --> " << m_blockchain.size()); + WLT_LOG_L1("[PULL BLOCKS] " << res.start_height << " --> " << get_blockchain_current_height()); } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_sync_progress() @@ -1610,7 +1659,7 @@ void wallet2::refresh(size_t & blocks_fetched, bool& received_money, std::atomic size_t added_blocks = 0; size_t try_count = 0; crypto::hash last_tx_hash_id = m_transfers.size() ? get_transaction_hash(m_transfers.back().m_ptx_wallet_info->m_tx) : null_hash; - m_height_of_start_sync = m_blockchain.size(); + m_height_of_start_sync = get_blockchain_current_height(); m_last_sync_percent = 0; while (!stop.load(std::memory_order_relaxed)) { @@ -2075,7 +2124,7 @@ void wallet2::load(const std::wstring& wallet_, const std::string& password) { reset_history(); } - m_local_bc_height = m_blockchain.size(); + m_local_bc_height = get_blockchain_current_height(); THROW_IF_TRUE_WALLET_EX(need_to_resync, error::wallet_load_notice_wallet_restored, epee::string_encoding::convert_to_ansii(m_wallet_file)); WLT_LOG_L0("Loaded wallet file" << (m_watch_only ? " (WATCH ONLY) " : " ") << string_encoding::convert_to_ansii(m_wallet_file) << " with public address: " << m_account.get_public_address_str()); @@ -2623,7 +2672,7 @@ bool wallet2::is_transfer_okay_for_pos(const transfer_details& tr, uint64_t& sta return false; //prevent staking of after-last-pow-coins - if (m_blockchain.size() - tr.m_ptx_wallet_info->m_block_height <= m_core_runtime_config.min_coinstake_age) + if (get_blockchain_current_height() - tr.m_ptx_wallet_info->m_block_height <= m_core_runtime_config.min_coinstake_age) return false; if (tr.m_ptx_wallet_info->m_block_height > m_last_pow_block_h) @@ -2945,20 +2994,20 @@ bool wallet2::is_transfer_unlocked(const transfer_details& td, bool for_pos_mini if (td.m_flags&WALLET_TRANSFER_DETAIL_FLAG_BLOCKED) return false; - if (td.m_ptx_wallet_info->m_block_height + WALLET_DEFAULT_TX_SPENDABLE_AGE > m_blockchain.size()) + if (td.m_ptx_wallet_info->m_block_height + WALLET_DEFAULT_TX_SPENDABLE_AGE > get_blockchain_current_height()) return false; uint64_t unlock_time = get_tx_unlock_time(td.m_ptx_wallet_info->m_tx, td.m_internal_output_index); - if (for_pos_mining && m_blockchain.size() > m_core_runtime_config.hard_fork1_starts_after_height) + if (for_pos_mining && get_blockchain_current_height() > m_core_runtime_config.hard_fork1_starts_after_height) { //allowed of staking locked coins with stake_lock_time = unlock_time; } else { - if (!currency::is_tx_spendtime_unlocked(unlock_time, m_blockchain.size(), m_core_runtime_config.get_core_time())) + if (!currency::is_tx_spendtime_unlocked(unlock_time, get_blockchain_current_height(), m_core_runtime_config.get_core_time())) return false; } return true; @@ -4047,10 +4096,10 @@ void wallet2::process_genesis_if_needed(const currency::block& genesis) if (!m_transfers.empty() || !m_key_images.empty()) return; - THROW_IF_TRUE_WALLET_EX(m_blockchain.size() > 1, error::wallet_internal_error, "Can't change wallet genesis block once the blockchain has been populated"); + THROW_IF_TRUE_WALLET_EX(get_blockchain_current_height() > 1, error::wallet_internal_error, "Can't change wallet genesis block once the blockchain has been populated"); crypto::hash genesis_hash = get_block_hash(genesis); - if (m_blockchain.size() == 1 && m_blockchain[0] != genesis_hash) + if (get_blockchain_current_height() == 1 && m_blockchain[0] != genesis_hash) WLT_LOG_L0("Changing genesis block for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_blockchain[0] << " -> " << genesis_hash); m_blockchain.clear(); @@ -4065,7 +4114,7 @@ void wallet2::process_genesis_if_needed(const currency::block& genesis) void wallet2::set_genesis(const crypto::hash& genesis_hash) { - THROW_IF_TRUE_WALLET_EX(m_blockchain.size() != 1, error::wallet_internal_error, "Can't change wallet genesis hash once the blockchain has been populated"); + THROW_IF_TRUE_WALLET_EX(get_blockchain_current_height() != 1, error::wallet_internal_error, "Can't change wallet genesis hash once the blockchain has been populated"); WLT_LOG_L0("Changing genesis hash for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_blockchain[0] << " -> " << genesis_hash); m_blockchain[0] = genesis_hash; } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index aa866008..00ff344a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -44,6 +44,12 @@ #define WALLET_DEFAULT_POS_MINT_PACKING_SIZE 100 +#define WALLET_EVERYBLOCK_SIZE 10 +#define WALLET_EVERY_10_BLOCKS_SIZE 144 +#define WALLET_EVERY_100_BLOCKS_SIZE 144 +#define WALLET_EVERY_1000_BLOCKS_SIZE 144 + + #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "wallet" @@ -632,7 +638,9 @@ namespace tools uint64_t fee, size_t& outs_total, uint64_t& amount_total, size_t& outs_swept, currency::transaction* p_result_tx = nullptr, std::string* p_filename_or_unsigned_tx_blob_str = nullptr); bool get_transfer_address(const std::string& adr_str, currency::account_public_address& addr, std::string& payment_id); - uint64_t get_blockchain_current_height() const { return m_blockchain.size(); } + uint64_t get_blockchain_current_height() const { + return m_last_10_blocks.empty() ? 0 : (--m_last_10_blocks.end())->first + 1; + } template inline void serialize(t_archive &a, const unsigned int ver) @@ -665,8 +673,21 @@ namespace tools return; } } + //convert from old version + if (ver < CURRENCY_FORMATION_VERSION + 65) + { + //TODO: a & m_blockchain; + } + else + { + a & m_genesis; + a & m_last_10_blocks; + a & m_last_144_blocks_every_10; + a & m_last_144_blocks_every_100; + a & m_last_144_blocks_every_1000; + } + - a & m_blockchain; a & m_transfers; a & m_multisig_transfers; a & m_key_images; @@ -870,6 +891,7 @@ private: bool generate_packing_transaction_if_needed(currency::transaction& tx, uint64_t fake_outputs_number); bool store_unsigned_tx_to_file_and_reserve_transfers(const finalize_tx_param& ftp, const std::string& filename, std::string* p_unsigned_tx_blob_str = nullptr); void check_and_throw_if_self_directed_tx_with_payment_id_requested(const construct_tx_param& ctp); + void push_new_block_id(const crypto::hash& id, uint64_t height); currency::account_base m_account; bool m_watch_only; @@ -877,7 +899,13 @@ private: std::wstring m_wallet_file; std::wstring m_pending_ki_file; std::string m_password; - std::vector m_blockchain; + //std::vector m_blockchain; + crypto::hash m_genesis; + std::map m_last_10_blocks; + std::map m_last_144_blocks_every_10; //1 day + std::map m_last_144_blocks_every_100; //10 days + std::map m_last_144_blocks_every_1000; //100 days + std::atomic m_local_bc_height; //temporary workaround std::atomic m_last_bc_timestamp; bool m_do_rise_transfer; From 3b975d899e9d075a8ee8d13fa4c0ede0bdf91c2e Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 24 Apr 2020 19:18:19 +0200 Subject: [PATCH 02/15] implemented new items lookup and get_short_chain_history --- .../include/serialization/keyvalue_helpers.h | 10 +- src/rpc/core_rpc_server_commands_defs.h | 30 +++++ src/wallet/wallet2.cpp | 118 ++++++++++++++---- src/wallet/wallet2.h | 5 +- 4 files changed, 136 insertions(+), 27 deletions(-) diff --git a/contrib/epee/include/serialization/keyvalue_helpers.h b/contrib/epee/include/serialization/keyvalue_helpers.h index 9d32d024..0ae911b0 100644 --- a/contrib/epee/include/serialization/keyvalue_helpers.h +++ b/contrib/epee/include/serialization/keyvalue_helpers.h @@ -84,6 +84,12 @@ namespace epee return res; } //------------------------------------------------------------------------------------------------------------------- - - +#pragma pack(push, 1) + template + struct pod_pair + { + first_t first; + second_t second; + }; +#pragma pack(pop) } \ No newline at end of file diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 2d7130bc..275e917e 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -141,6 +141,36 @@ namespace currency typedef COMMAND_RPC_GET_BLOCKS_FAST_T COMMAND_RPC_GET_BLOCKS_FAST; typedef COMMAND_RPC_GET_BLOCKS_FAST_T COMMAND_RPC_GET_BLOCKS_DIRECT; + template + struct COMMAND_RPC_GET_BLOCKS_FUZZY_FAST_T + { + + struct request + { + std::list > block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list blocks; + uint64_t start_height; + uint64_t current_height; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(blocks) + KV_SERIALIZE(start_height) + KV_SERIALIZE(current_height) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + typedef COMMAND_RPC_GET_BLOCKS_FUZZY_FAST_T COMMAND_RPC_GET_BLOCKS_FUZZY_FAST; + typedef COMMAND_RPC_GET_BLOCKS_FUZZY_FAST_T COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT; //----------------------------------------------- diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index a418be65..728e3011 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1149,39 +1149,111 @@ void wallet2::push_new_block_id(const crypto::hash& id, uint64_t height) } //---------------------------------------------------------------------------------------------------- -void wallet2::get_short_chain_history(std::list& ids) +// void wallet2::get_short_chain_history(std::list& ids) +// { +// ids.clear(); +// size_t i = 0; +// size_t current_multiplier = 1; +// size_t sz = get_blockchain_current_height(); +// if(!sz) +// return; +// size_t current_back_offset = 1; +// bool genesis_included = false; +// while(current_back_offset < sz) +// { +// ids.push_back(m_blockchain[sz-current_back_offset]); +// if(sz-current_back_offset == 0) +// genesis_included = true; +// if(i < 10) +// { +// ++current_back_offset; +// }else +// { +// current_back_offset += current_multiplier *= 2; +// } +// ++i; +// } +// if(!genesis_included) +// ids.push_back(m_blockchain[0]); +// } +//---------------------------------------------------------------------------------------------------- +bool wallet2::lookup_item_around(uint64_t i, std::pair& result) +{ + //in which container we are looking for? + uint64_t devider = 0; + std::map* pcontainer; + if (m_last_144_blocks_every_10.size() && i < m_last_144_blocks_every_10.begin()->first) + { + devider = 10; + pcontainer = &m_last_144_blocks_every_10; + } + else if (m_last_144_blocks_every_100.size() && i < m_last_144_blocks_every_100.begin()->first) + { + devider = 100; + pcontainer = &m_last_144_blocks_every_100; + } + else if (m_last_144_blocks_every_1000.size() && i < m_last_144_blocks_every_1000.begin()->first) + { + devider = 1000; + pcontainer = &m_last_144_blocks_every_1000; + } + else + return false; + + //look in every 10'th + i = i - i % devider; + auto it = pcontainer->find(i); + //self check + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it != pcontainer->end(), + "Inernal error: amount " << i << " not found for devider " << devider + << " pcontainer={" << pcontainer->begin()->first << ":"<< (--pcontainer->end())->first <<"}"); + result = *it; + return true; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::get_short_chain_history(std::list >& ids) { ids.clear(); - size_t i = 0; - size_t current_multiplier = 1; - size_t sz = get_blockchain_current_height(); - if(!sz) + uint64_t i = 0; + uint64_t sz = get_blockchain_current_height(); + if (!sz) return; - size_t current_back_offset = 1; - bool genesis_included = false; - while(current_back_offset < sz) + + //first put last 10 + for (auto it = m_last_10_blocks.rbegin(); it != m_last_10_blocks.rend(); it++) { - ids.push_back(m_blockchain[sz-current_back_offset]); - if(sz-current_back_offset == 0) - genesis_included = true; - if(i < 10) - { - ++current_back_offset; - }else - { - current_back_offset += current_multiplier *= 2; - } - ++i; + ids.push_back({ it->first, it->second }); + i = it->first; } - if(!genesis_included) - ids.push_back(m_blockchain[0]); + + uint64_t current_back_offset = m_last_10_blocks.size(); + //self check + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(current_back_offset == sz-i, "Inernal error: current_back_offset{" << current_back_offset << "} == sz-i{" << sz << " - " << i << "} is not equal"); + + uint64_t current_offset_distance = 10; + current_back_offset += 10; + while (current_back_offset < sz) + { + uint64_t get_item_around = sz - current_back_offset; + std::pair item = AUTO_VAL_INIT(item); + if (!lookup_item_around(get_item_around, item)) + break; + + //readjust item current_back_offset + current_back_offset = sz - item.first; + + ids.push_back(item); + current_offset_distance *= 2; + current_back_offset += current_offset_distance; + } + ids.push_back({ 0, m_genesis }); } //---------------------------------------------------------------------------------------------------- void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) { blocks_added = 0; - currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request req = AUTO_VAL_INIT(req); - currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response res = AUTO_VAL_INIT(res); + currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request req = AUTO_VAL_INIT(req); + currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response res = AUTO_VAL_INIT(res); get_short_chain_history(req.block_ids); bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); if (!r) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 00ff344a..f01a12cb 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -638,7 +638,7 @@ namespace tools uint64_t fee, size_t& outs_total, uint64_t& amount_total, size_t& outs_swept, currency::transaction* p_result_tx = nullptr, std::string* p_filename_or_unsigned_tx_blob_str = nullptr); bool get_transfer_address(const std::string& adr_str, currency::account_public_address& addr, std::string& payment_id); - uint64_t get_blockchain_current_height() const { + inline uint64_t get_blockchain_current_height() const { return m_last_10_blocks.empty() ? 0 : (--m_last_10_blocks.end())->first + 1; } @@ -778,7 +778,6 @@ private: void load_keys(const std::string& keys_file_name, const std::string& password); void process_new_transaction(const currency::transaction& tx, uint64_t height, const currency::block& b); void detach_blockchain(uint64_t height); - void get_short_chain_history(std::list& ids); bool extract_offers_from_transfer_entry(size_t i, std::unordered_map& offers_local); bool select_my_offers(std::list& offers); bool clear(); @@ -892,6 +891,8 @@ private: bool store_unsigned_tx_to_file_and_reserve_transfers(const finalize_tx_param& ftp, const std::string& filename, std::string* p_unsigned_tx_blob_str = nullptr); void check_and_throw_if_self_directed_tx_with_payment_id_requested(const construct_tx_param& ctp); void push_new_block_id(const crypto::hash& id, uint64_t height); + bool lookup_item_around(uint64_t i, std::pair& result); + void get_short_chain_history(std::list >& ids); currency::account_base m_account; bool m_watch_only; From 93d07e607fd09d5c1f827f1b58da3e5331dade9f Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 24 Apr 2020 23:54:30 +0200 Subject: [PATCH 03/15] re-implemented handle_pulled_blocks --- src/wallet/core_rpc_proxy.h | 1 + src/wallet/wallet2.cpp | 125 ++++++++++++++++++++++++++++++------ src/wallet/wallet2.h | 5 +- 3 files changed, 111 insertions(+), 20 deletions(-) diff --git a/src/wallet/core_rpc_proxy.h b/src/wallet/core_rpc_proxy.h index 0b6e0c92..a64c6350 100644 --- a/src/wallet/core_rpc_proxy.h +++ b/src/wallet/core_rpc_proxy.h @@ -22,6 +22,7 @@ namespace tools virtual bool call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(const currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& rqt, currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_BLOCKS_FAST(const currency::COMMAND_RPC_GET_BLOCKS_FAST::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FAST::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& rsp) { return false; } + virtual bool call_COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& rsp) { return false; } virtual bool call_COMMAND_RPC_GET_INFO(const currency::COMMAND_RPC_GET_INFO::request& rqt, currency::COMMAND_RPC_GET_INFO::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_TX_POOL(const currency::COMMAND_RPC_GET_TX_POOL::request& rqt, currency::COMMAND_RPC_GET_TX_POOL::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_ALIASES_BY_ADDRESS(const currency::COMMAND_RPC_GET_ALIASES_BY_ADDRESS::request& rqt, currency::COMMAND_RPC_GET_ALIASES_BY_ADDRESS::response& rsp){ return false; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 728e3011..76e62404 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1205,7 +1205,7 @@ bool wallet2::lookup_item_around(uint64_t i, std::pair& auto it = pcontainer->find(i); //self check WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it != pcontainer->end(), - "Inernal error: amount " << i << " not found for devider " << devider + "Inernal error: index " << i << " not found for devider " << devider << " pcontainer={" << pcontainer->begin()->first << ":"<< (--pcontainer->end())->first <<"}"); result = *it; return true; @@ -1255,7 +1255,7 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request req = AUTO_VAL_INIT(req); currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response res = AUTO_VAL_INIT(res); get_short_chain_history(req.block_ids); - bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); + bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT(req, res); if (!r) throw error::no_connection_to_daemon(LOCATION_STR, "getblocks.bin"); @@ -1275,11 +1275,11 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) r = string_tools::parse_tpod_from_hex_string(gbd_res.blocks.back().id, new_genesis_id); THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "get_blocks_details"); reset_all(); - m_blockchain.push_back(new_genesis_id); + m_genesis = new_genesis_id; WLT_LOG_MAGENTA("New genesis set for wallet: " << new_genesis_id, LOG_LEVEL_0); get_short_chain_history(req.block_ids); //req.block_ids.push_back(new_genesis_id); - bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); + bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT(req, res); THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "getblocks.bin"); } if (res.status == CORE_RPC_STATUS_BUSY) @@ -1296,8 +1296,64 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) handle_pulled_blocks(blocks_added, stop, res); } //---------------------------------------------------------------------------------------------------- +void wallet2::check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed) +{ + if (!m_last_10_blocks.empty() && i > m_last_10_blocks.begin()->first) + { + //must be in short sequence (m_last_10_blocks) + //self check + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_10_blocks.end())->first >= i, + "Inernal error: index " << i << " is not located in expected range of m_last_10_blocks={" + << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); + + auto it = m_last_10_blocks.find(i); + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it != m_last_10_blocks.end(), + "Inernal error: filde to find index " << i << " in m_last_10_blocks={" + << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); + + block_found = true; + if (id == it->second) + block_matched = true; + else + block_matched = false; + } + else + { + //lazy lookup + std::pair result = AUTO_VAL_INIT(result); + bool r = lookup_item_around(i, result); + if (!r) + { + WLT_LOG_L0("Wallet is getting fully resynced due to unmatched block " << id << " at " << i ); + block_matched = block_found = false; + full_reset_needed = true; + return; + } + else + { + if (result.first == i) + { + block_found = true; + if (result.second == id) + { + block_matched = true; + } + else + { + block_matched = false; + } + } + else + { + block_found = false; + block_matched = false; + } + } + } +} +//---------------------------------------------------------------------------------------------------- void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop, - currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& res) + currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& res) { size_t current_index = res.start_height; @@ -1310,38 +1366,71 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop ++current_index; } + uint64_t last_matched_index = 0; + bool been_matched_block = false; for(const auto& bl_entry: res.blocks) { if (stop) break; const currency::block& bl = bl_entry.block_ptr->bl; + uint64_t height = get_block_height(bl); + uint64_t processed_blocks_count = get_blockchain_current_height(); //TODO: get_block_hash is slow crypto::hash bl_id = get_block_hash(bl); - if (current_index >= get_blockchain_current_height()) - { - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index); - ++blocks_added; + if (height > processed_blocks_count) + {//internal error: + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(false, + "height{" << height <<"} > processed_blocks_count{" << processed_blocks_count << "}"); } - else if(bl_id != m_blockchain[current_index]) + else if (height == processed_blocks_count) { - //split detected here !!! - THROW_IF_TRUE_WALLET_EX(current_index == res.start_height, error::wallet_internal_error, - "wrong daemon response: split starts from the first block in response " + string_tools::pod_to_hex(bl_id) + - " (height " + std::to_string(res.start_height) + "), local block id at this height: " + - string_tools::pod_to_hex(m_blockchain[current_index])); + //regular block handling + //self check + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(been_matched_block, + "internal error: been_matched_block == false on process_new_blockchain_entry, bl_id" << bl_id << "h=" << height + << " (start_height=" + std::to_string(res.start_height) + ")"); - detach_blockchain(current_index); process_new_blockchain_entry(bl, bl_entry, bl_id, current_index); ++blocks_added; } else - { - WLT_LOG_L2("Block " << bl_id << " @ " << current_index << " is already in wallet's blockchain"); + { + //checking if we need reorganize (might be just first matched block) + bool block_found = false; + bool block_matched = false; + bool full_reset_needed = false; + check_if_block_matched(height, bl_id, block_found, block_matched, full_reset_needed); + if (block_found && block_matched) + { + //block matched in that number + last_matched_index = height; + been_matched_block = true; + WLT_LOG_L2("Block " << bl_id << " @ " << height << " is already in wallet's blockchain"); + } + else + { + //this should happen ONLY after block been matched, if not then is internal error + if (full_reset_needed) + { + last_matched_index = 0; + } + else + { + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(been_matched_block, + "unmatched block while never been mathced block"); + } + + //reorganize + detach_blockchain(last_matched_index); + process_new_blockchain_entry(bl, bl_entry, bl_id, height); + ++blocks_added; + } } + ++current_index; if (res.current_height > m_height_of_start_sync) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index f01a12cb..11a3a0ff 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -814,7 +814,7 @@ private: const std::vector& recipients, const std::vector& recipients_aliases); void handle_pulled_blocks(size_t& blocks_added, std::atomic& stop, - currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& blocks); + currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& blocks); std::string get_alias_for_address(const std::string& addr); static bool build_kernel(const currency::pos_entry& pe, const currency::stake_modifier_type& stake_modifier, currency::stake_kernel& kernel, uint64_t& coindays_weight, uint64_t timestamp); bool is_connected_to_net(); @@ -892,7 +892,8 @@ private: void check_and_throw_if_self_directed_tx_with_payment_id_requested(const construct_tx_param& ctp); void push_new_block_id(const crypto::hash& id, uint64_t height); bool lookup_item_around(uint64_t i, std::pair& result); - void get_short_chain_history(std::list >& ids); + void get_short_chain_history(std::list >& ids); + void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed); currency::account_base m_account; bool m_watch_only; From caaca44f1e0d39cf7a1203f6826fd94bdf038b96 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 25 Apr 2020 00:30:55 +0200 Subject: [PATCH 04/15] re-implemented detach blocks --- src/rpc/core_rpc_server_commands_defs.h | 2 +- src/simplewallet/simplewallet.cpp | 2 +- src/wallet/wallet2.cpp | 68 ++++++++++++++++--------- src/wallet/wallet2.h | 12 +++-- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 275e917e..725f5860 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -147,7 +147,7 @@ namespace currency struct request { - std::list > block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ + std::list > block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 36d23a7f..c1978d57 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1116,7 +1116,7 @@ bool simple_wallet::show_blockchain_height(const std::vector& args) bool simple_wallet::show_wallet_bcheight(const std::vector& args) { - uint64_t bc_height = m_wallet->get_blockchain_current_height(); + uint64_t bc_height = m_wallet->get_blockchain_current_size(); success_msg_writer() << bc_height; return true; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 76e62404..2f596686 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1070,8 +1070,8 @@ void wallet2::process_unconfirmed(const currency::transaction& tx, std::vector& stop) return; } THROW_IF_TRUE_WALLET_EX(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, res.status); - THROW_IF_TRUE_WALLET_EX(get_blockchain_current_height() && get_blockchain_current_height() <= res.start_height, error::wallet_internal_error, + THROW_IF_TRUE_WALLET_EX(get_blockchain_current_size() && get_blockchain_current_size() <= res.start_height, error::wallet_internal_error, "wrong daemon response: m_start_height=" + std::to_string(res.start_height) + - " not less than local blockchain size=" + std::to_string(get_blockchain_current_height())); + " not less than local blockchain size=" + std::to_string(get_blockchain_current_size())); handle_pulled_blocks(blocks_added, stop, res); } @@ -1357,7 +1357,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop { size_t current_index = res.start_height; - if (res.start_height == 0 && get_blockchain_current_height() == 1 && !res.blocks.empty()) + if (res.start_height == 0 && get_blockchain_current_size() == 1 && !res.blocks.empty()) { const currency::block& genesis = res.blocks.front().block_ptr->bl; THROW_IF_TRUE_WALLET_EX(get_block_height(genesis) != 0, error::wallet_internal_error, "first block expected to be genesis"); @@ -1375,7 +1375,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop const currency::block& bl = bl_entry.block_ptr->bl; uint64_t height = get_block_height(bl); - uint64_t processed_blocks_count = get_blockchain_current_height(); + uint64_t processed_blocks_count = get_blockchain_current_size(); //TODO: get_block_hash is slow crypto::hash bl_id = get_block_hash(bl); @@ -1422,7 +1422,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(been_matched_block, "unmatched block while never been mathced block"); } - + //TODO: take into account date of wallet creation //reorganize detach_blockchain(last_matched_index); process_new_blockchain_entry(bl, bl_entry, bl_id, height); @@ -1443,7 +1443,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop } } - WLT_LOG_L1("[PULL BLOCKS] " << res.start_height << " --> " << get_blockchain_current_height()); + WLT_LOG_L1("[PULL BLOCKS] " << res.start_height << " --> " << get_blockchain_current_size()); } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_sync_progress() @@ -1820,7 +1820,7 @@ void wallet2::refresh(size_t & blocks_fetched, bool& received_money, std::atomic size_t added_blocks = 0; size_t try_count = 0; crypto::hash last_tx_hash_id = m_transfers.size() ? get_transaction_hash(m_transfers.back().m_ptx_wallet_info->m_tx) : null_hash; - m_height_of_start_sync = get_blockchain_current_height(); + m_height_of_start_sync = get_blockchain_current_size(); m_last_sync_percent = 0; while (!stop.load(std::memory_order_relaxed)) { @@ -1929,6 +1929,26 @@ bool wallet2::refresh(size_t & blocks_fetched, bool& received_money, bool& ok, s return ok; } //---------------------------------------------------------------------------------------------------- +void clean_map_from_items_above(std::map& container, uint64_t height) +{ + while (container.size() && (--container.end())->first > height) + { + container.erase(--container.end()); + } +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::detach_from_block_ids(uint64_t height) +{ + //calculate number of erased blocks + uint64_t blocks_detached = (get_blockchain_current_size() -1 ) - height; + //id at height should be kept, the rest - erased + clean_map_from_items_above(m_last_10_blocks, height); + clean_map_from_items_above(m_last_10_blocks, height); + clean_map_from_items_above(m_last_10_blocks, height); + clean_map_from_items_above(m_last_10_blocks, height); + return blocks_detached; +} +//---------------------------------------------------------------------------------------------------- void wallet2::detach_blockchain(uint64_t height) { WLT_LOG_L0("Detaching blockchain on height " << height); @@ -1953,9 +1973,8 @@ void wallet2::detach_blockchain(uint64_t height) } } - size_t blocks_detached = m_blockchain.end() - (m_blockchain.begin()+height); - m_blockchain.erase(m_blockchain.begin()+height, m_blockchain.end()); - m_local_bc_height -= blocks_detached; + size_t blocks_detached = detach_from_block_ids(height); + m_local_bc_size -= height+1; //rollback spends // do not clear spent flag in spent transfers as corresponding txs are most likely in the pool @@ -2043,7 +2062,7 @@ bool wallet2::reset_all() m_transfer_history.clear(); //m_account = AUTO_VAL_INIT(m_account); - m_local_bc_height = 1; + m_local_bc_size = 1; //including genesis m_last_bc_timestamp = 0; m_height_of_start_sync = 0; m_last_sync_percent = 0; @@ -2285,7 +2304,6 @@ void wallet2::load(const std::wstring& wallet_, const std::string& password) { reset_history(); } - m_local_bc_height = get_blockchain_current_height(); THROW_IF_TRUE_WALLET_EX(need_to_resync, error::wallet_load_notice_wallet_restored, epee::string_encoding::convert_to_ansii(m_wallet_file)); WLT_LOG_L0("Loaded wallet file" << (m_watch_only ? " (WATCH ONLY) " : " ") << string_encoding::convert_to_ansii(m_wallet_file) << " with public address: " << m_account.get_public_address_str()); @@ -2833,7 +2851,7 @@ bool wallet2::is_transfer_okay_for_pos(const transfer_details& tr, uint64_t& sta return false; //prevent staking of after-last-pow-coins - if (get_blockchain_current_height() - tr.m_ptx_wallet_info->m_block_height <= m_core_runtime_config.min_coinstake_age) + if (get_blockchain_current_size() - tr.m_ptx_wallet_info->m_block_height <= m_core_runtime_config.min_coinstake_age) return false; if (tr.m_ptx_wallet_info->m_block_height > m_last_pow_block_h) @@ -3155,20 +3173,20 @@ bool wallet2::is_transfer_unlocked(const transfer_details& td, bool for_pos_mini if (td.m_flags&WALLET_TRANSFER_DETAIL_FLAG_BLOCKED) return false; - if (td.m_ptx_wallet_info->m_block_height + WALLET_DEFAULT_TX_SPENDABLE_AGE > get_blockchain_current_height()) + if (td.m_ptx_wallet_info->m_block_height + WALLET_DEFAULT_TX_SPENDABLE_AGE > get_blockchain_current_size()) return false; uint64_t unlock_time = get_tx_unlock_time(td.m_ptx_wallet_info->m_tx, td.m_internal_output_index); - if (for_pos_mining && get_blockchain_current_height() > m_core_runtime_config.hard_fork1_starts_after_height) + if (for_pos_mining && get_blockchain_current_size() > m_core_runtime_config.hard_fork1_starts_after_height) { //allowed of staking locked coins with stake_lock_time = unlock_time; } else { - if (!currency::is_tx_spendtime_unlocked(unlock_time, get_blockchain_current_height(), m_core_runtime_config.get_core_time())) + if (!currency::is_tx_spendtime_unlocked(unlock_time, get_blockchain_current_size(), m_core_runtime_config.get_core_time())) return false; } return true; @@ -4257,16 +4275,16 @@ void wallet2::process_genesis_if_needed(const currency::block& genesis) if (!m_transfers.empty() || !m_key_images.empty()) return; - THROW_IF_TRUE_WALLET_EX(get_blockchain_current_height() > 1, error::wallet_internal_error, "Can't change wallet genesis block once the blockchain has been populated"); + THROW_IF_TRUE_WALLET_EX(get_blockchain_current_size() > 1, error::wallet_internal_error, "Can't change wallet genesis block once the blockchain has been populated"); crypto::hash genesis_hash = get_block_hash(genesis); - if (get_blockchain_current_height() == 1 && m_blockchain[0] != genesis_hash) + if (get_blockchain_current_size() == 1 && m_blockchain[0] != genesis_hash) WLT_LOG_L0("Changing genesis block for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_blockchain[0] << " -> " << genesis_hash); m_blockchain.clear(); m_blockchain.push_back(genesis_hash); - m_local_bc_height = 1; + m_local_bc_size = 1; m_last_bc_timestamp = genesis.timestamp; WLT_LOG_L2("Processing genesis block: " << genesis_hash); @@ -4275,7 +4293,7 @@ void wallet2::process_genesis_if_needed(const currency::block& genesis) void wallet2::set_genesis(const crypto::hash& genesis_hash) { - THROW_IF_TRUE_WALLET_EX(get_blockchain_current_height() != 1, error::wallet_internal_error, "Can't change wallet genesis hash once the blockchain has been populated"); + THROW_IF_TRUE_WALLET_EX(get_blockchain_current_size() != 1, error::wallet_internal_error, "Can't change wallet genesis hash once the blockchain has been populated"); WLT_LOG_L0("Changing genesis hash for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_blockchain[0] << " -> " << genesis_hash); m_blockchain[0] = genesis_hash; } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 11a3a0ff..d0d71597 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -638,8 +638,8 @@ namespace tools uint64_t fee, size_t& outs_total, uint64_t& amount_total, size_t& outs_swept, currency::transaction* p_result_tx = nullptr, std::string* p_filename_or_unsigned_tx_blob_str = nullptr); bool get_transfer_address(const std::string& adr_str, currency::account_public_address& addr, std::string& payment_id); - inline uint64_t get_blockchain_current_height() const { - return m_last_10_blocks.empty() ? 0 : (--m_last_10_blocks.end())->first + 1; + inline uint64_t get_blockchain_current_size() const { + return m_local_bc_size; } template @@ -676,10 +676,11 @@ namespace tools //convert from old version if (ver < CURRENCY_FORMATION_VERSION + 65) { - //TODO: a & m_blockchain; + //TODO: export from a & m_blockchain; } else { + a & m_local_bc_size; a & m_genesis; a & m_last_10_blocks; a & m_last_144_blocks_every_10; @@ -715,7 +716,7 @@ namespace tools //for unit tests friend class ::test_generator; - //next functions in public area only because of test_generator + //next functions in public area only becausce of test_generator //TODO: Need refactoring - remove it back to private zone void set_genesis(const crypto::hash& genesis_hash); bool prepare_and_sign_pos_block(currency::block& b, @@ -894,6 +895,7 @@ private: bool lookup_item_around(uint64_t i, std::pair& result); void get_short_chain_history(std::list >& ids); void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed); + uint64_t detach_from_block_ids(uint64_t height); currency::account_base m_account; bool m_watch_only; @@ -908,7 +910,7 @@ private: std::map m_last_144_blocks_every_100; //10 days std::map m_last_144_blocks_every_1000; //100 days - std::atomic m_local_bc_height; //temporary workaround + std::atomic m_local_bc_size; //temporary workaround std::atomic m_last_bc_timestamp; bool m_do_rise_transfer; uint64_t m_pos_mint_packing_size; From 7b0eaf7ae42547e397d2e4fb401a353ce9205a77 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sun, 26 Apr 2020 01:49:57 +0200 Subject: [PATCH 05/15] fixed cleaning and reseting genesis --- src/common/atomics_boost_serialization.h | 32 ++++++++++++++++++++++ src/wallet/wallet2.cpp | 34 +++++++++++++++--------- src/wallet/wallet2.h | 2 ++ 3 files changed, 55 insertions(+), 13 deletions(-) create mode 100644 src/common/atomics_boost_serialization.h diff --git a/src/common/atomics_boost_serialization.h b/src/common/atomics_boost_serialization.h new file mode 100644 index 00000000..e9aa9798 --- /dev/null +++ b/src/common/atomics_boost_serialization.h @@ -0,0 +1,32 @@ +// Copyright (c) 2014-2018 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace boost +{ + namespace serialization + { + template + inline void save(Archive &a, const std::atomic &x, const boost::serialization::version_type ver) + { + a << x.load(); + } + + template + inline void load(Archive &a, std::atomic &x, const boost::serialization::version_type ver) + { + value_t s = AUTO_VAL_INIT(s); + a >> s; + x.store(s); + } + template + inline void serialize(Archive &a, std::atomic &x, const boost::serialization::version_type ver) + { + split_free(a, x, ver); + } + } +} diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 2f596686..3d3c8933 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1242,7 +1242,7 @@ void wallet2::get_short_chain_history(std::list 1, error::wallet_internal_error, "Can't change wallet genesis block once the blockchain has been populated"); crypto::hash genesis_hash = get_block_hash(genesis); - if (get_blockchain_current_size() == 1 && m_blockchain[0] != genesis_hash) - WLT_LOG_L0("Changing genesis block for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_blockchain[0] << " -> " << genesis_hash); + if (get_blockchain_current_size() == 1 && m_genesis != genesis_hash) + WLT_LOG_L0("Changing genesis block for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_genesis << " -> " << genesis_hash); - m_blockchain.clear(); + //m_blockchain.clear(); - m_blockchain.push_back(genesis_hash); + //m_blockchain.push_back(genesis_hash); + m_genesis = genesis_hash; m_local_bc_size = 1; m_last_bc_timestamp = genesis.timestamp; @@ -4294,8 +4302,8 @@ void wallet2::process_genesis_if_needed(const currency::block& genesis) void wallet2::set_genesis(const crypto::hash& genesis_hash) { THROW_IF_TRUE_WALLET_EX(get_blockchain_current_size() != 1, error::wallet_internal_error, "Can't change wallet genesis hash once the blockchain has been populated"); - WLT_LOG_L0("Changing genesis hash for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_blockchain[0] << " -> " << genesis_hash); - m_blockchain[0] = genesis_hash; + WLT_LOG_L0("Changing genesis hash for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_genesis << " -> " << genesis_hash); + m_genesis = genesis_hash; } //---------------------------------------------------------------------------------------------------- void wallet2::print_tx_sent_message(const currency::transaction& tx, const std::string& description, uint64_t fee /* = UINT64_MAX */) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index d0d71597..352abe3c 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -14,6 +14,7 @@ #include #include + #include "include_base_utils.h" #include "profile_tools.h" #include "sync_locked_object.h" @@ -26,6 +27,7 @@ #include "wallet_public_structs_defs.h" #include "currency_core/currency_format_utils.h" #include "common/unordered_containers_boost_serialization.h" +#include "common/atomics_boost_serialization.h" #include "storages/portable_storage_template_helper.h" #include "crypto/chacha8.h" #include "crypto/hash.h" From bf222f18fe6e617cbaa3d33bc27c1f3c1a37ca5d Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Mon, 27 Apr 2020 00:02:58 +0200 Subject: [PATCH 06/15] on_get_blocks_fuzzy_direct/find_blockchain_supplement_fuzzy draft impl --- src/currency_core/blockchain_storage.cpp | 25 +++++++++++++++++++ src/currency_core/blockchain_storage.h | 1 + src/rpc/core_rpc_server.cpp | 31 +++++++++++++++++++++++- src/rpc/core_rpc_server.h | 2 ++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 54b45be9..3c24566a 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -2938,6 +2938,31 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const +{ + CRITICAL_REGION_LOCAL(m_read_lock); +// if (!find_blockchain_supplement(qblock_ids, resp.start_height)) +// return false; +// +// resp.total_height = get_current_blockchain_size(); +// size_t count = 0; +// +// block_context_info* pprevinfo = nullptr; +// size_t i = 0; +// for (i = resp.start_height; i != m_db_blocks.size() && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) +// { +// resp.m_block_ids.push_back(block_context_info()); +// +// if (pprevinfo) +// pprevinfo->h = m_db_blocks[i]->bl.prev_id; +// resp.m_block_ids.back().cumul_size = m_db_blocks[i]->block_cumulative_size; +// pprevinfo = &resp.m_block_ids.back(); +// } +// if (pprevinfo) +// pprevinfo->h = get_block_hash(m_db_blocks[--i]->bl); + + return true; +} //------------------------------------------------------------------ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const { diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index bc595256..2a3bba4e 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -252,6 +252,7 @@ namespace currency size_t get_total_transactions()const; bool get_outs(uint64_t amount, std::list& pkeys)const; bool get_short_chain_history(std::list& ids)const; + bool find_blockchain_supplement_fuzzy(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const; bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const; bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset)const; bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index b8a8db05..4bb19be5 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -288,7 +288,36 @@ namespace currency res.status = CORE_RPC_STATUS_OK; return true; } - //------------------------------------------------------------------------------------------------------------------------------ + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_blocks_fuzzy_direct(const COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request& req, COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& res, connection_context& cntx) + { + CHECK_CORE_READY(); + + if (req.block_ids.back() != m_core.get_blockchain_storage().get_block_id_by_height(0)) + { + //genesis mismatch, return specific + res.status = CORE_RPC_STATUS_GENESIS_MISMATCH; + return true; + } + + blockchain_storage::blocks_direct_container bs; + if (!m_core.get_blockchain_storage().find_blockchain_supplement2(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + { + res.status = CORE_RPC_STATUS_FAILED; + return false; + } + + for (auto& b : bs) + { + res.blocks.resize(res.blocks.size() + 1); + res.blocks.back().block_ptr = b.first; + res.blocks.back().txs_ptr = std::move(b.second); + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx) { CHECK_CORE_READY(); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index b58373dc..ac1a21ff 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -39,6 +39,8 @@ namespace currency bool init(const boost::program_options::variables_map& vm); bool on_get_blocks_direct(const COMMAND_RPC_GET_BLOCKS_DIRECT::request& req, COMMAND_RPC_GET_BLOCKS_DIRECT::response& res, connection_context& cntx); + bool on_get_blocks_fuzzy_direct(const COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request& req, COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& res, connection_context& cntx); + bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx); bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx); From 4e0fa49748b9d330fd927173a77380ed60e60b5f Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Tue, 28 Apr 2020 00:40:31 +0200 Subject: [PATCH 07/15] reverted API to original due to universal implementation, adapted server side to new protocol, implemented one extra API --- src/currency_core/blockchain_storage.cpp | 84 ++++++++++++++++++------ src/currency_core/blockchain_storage.h | 5 +- src/rpc/core_rpc_server.cpp | 42 ++++-------- src/rpc/core_rpc_server.h | 3 + src/rpc/core_rpc_server_commands_defs.h | 58 ++++++++-------- src/wallet/core_default_rpc_proxy.cpp | 5 ++ src/wallet/core_default_rpc_proxy.h | 1 + src/wallet/core_fast_rpc_proxy.h | 4 ++ src/wallet/core_rpc_proxy.h | 2 +- src/wallet/wallet2.cpp | 37 ++++++++--- src/wallet/wallet2.h | 7 +- src/wallet/wallets_manager.cpp | 1 + 12 files changed, 153 insertions(+), 96 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 3c24566a..49ad0ac0 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -70,6 +70,8 @@ using namespace currency; #endif #define BLOCK_POS_STRICT_SEQUENCE_LIMIT 20 +#define BLOCKCHAIN_FIRST_BLOCK_TIMESTAMP 1557342384 + DISABLE_VS_WARNINGS(4267) @@ -2938,29 +2940,67 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const +//------------------------------------------------------------------ +bool blockchain_storage::get_est_height_from_date(uint64_t date, uint64_t& res_h)const { CRITICAL_REGION_LOCAL(m_read_lock); -// if (!find_blockchain_supplement(qblock_ids, resp.start_height)) -// return false; -// -// resp.total_height = get_current_blockchain_size(); -// size_t count = 0; -// -// block_context_info* pprevinfo = nullptr; -// size_t i = 0; -// for (i = resp.start_height; i != m_db_blocks.size() && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) -// { -// resp.m_block_ids.push_back(block_context_info()); -// -// if (pprevinfo) -// pprevinfo->h = m_db_blocks[i]->bl.prev_id; -// resp.m_block_ids.back().cumul_size = m_db_blocks[i]->block_cumulative_size; -// pprevinfo = &resp.m_block_ids.back(); -// } -// if (pprevinfo) -// pprevinfo->h = get_block_hash(m_db_blocks[--i]->bl); +#define GET_EST_HEIGHT_FROM_DATE_THRESHOLD 1440 + if (date < BLOCKCHAIN_FIRST_BLOCK_TIMESTAMP) + return false; + + + uint64_t calculated_estimated_height = (date - BLOCKCHAIN_FIRST_BLOCK_TIMESTAMP) / DIFFICULTY_TOTAL_TARGET; + + if (date > m_db_blocks[m_db_blocks.size() - 1]->bl.timestamp) + { + //that suspicious but also could be(in case someone just created wallet offline in + //console and then got it synchronyzing and last block had a little timestamp shift) + //let's just return 1 day behind for safety reasons. + if (m_db_blocks.size() > 1440) + { + res_h = m_db_blocks.size() - 1440; + return true; + } + else + { + //likely impossible, but just in case + res_h = 1; + } + + } + if (calculated_estimated_height > m_db_blocks.size() - 1) + calculated_estimated_height = m_db_blocks.size() - 1; + + //goal is to get timestamp in window in between 1day+1hour and 1 hour before target(1 hour is just to be sure that + //we didn't miss actual wallet start because of timestamp and difficulty fluctuations) + uint64_t low_boundary = date - 90000; //1 day + 1 hour + uint64_t aim = date - 46800; + uint64_t high_boundary = date - 3600; //1 hour + + uint64_t iteration_coun = 0; + while (true) + { + iteration_coun++; + uint64_t correction = 0; + uint64_t ts = m_db_blocks[calculated_estimated_height]->bl.timestamp; + if (ts > high_boundary) + { + //we moved too much forward + calculated_estimated_height -= (ts - aim) / DIFFICULTY_TOTAL_TARGET; + } + else if (ts < low_boundary) + { + //we too much in past + calculated_estimated_height += (aim - ts) / DIFFICULTY_TOTAL_TARGET; + } + else + { + res_h = calculated_estimated_height; + break; + } + } + LOG_PRINT_L0("[get_est_height_from_date] returned " << calculated_estimated_height << " with " << iteration_coun << " iterations"); return true; } //------------------------------------------------------------------ @@ -2983,11 +3023,13 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height)const { CRITICAL_REGION_LOCAL(m_read_lock); if (!find_blockchain_supplement(qblock_ids, start_height)) return false; + if (minimum_height > start_height) + start_height = minimum_height; total_height = get_current_blockchain_size(); size_t count = 0; diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 2a3bba4e..02f03839 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -252,11 +252,10 @@ namespace currency size_t get_total_transactions()const; bool get_outs(uint64_t amount, std::list& pkeys)const; bool get_short_chain_history(std::list& ids)const; - bool find_blockchain_supplement_fuzzy(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const; bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const; bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset)const; bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; - bool find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; + bool find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0)const; //bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp)const; bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res)const; @@ -422,7 +421,7 @@ namespace currency template void serialize(archive_t & ar, const unsigned int version); - + bool get_est_height_from_date(uint64_t date, uint64_t& res_h)const; //debug functions diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 4bb19be5..ee0e405b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -272,7 +272,7 @@ namespace currency } blockchain_storage::blocks_direct_container bs; - if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height)) { res.status = CORE_RPC_STATUS_FAILED; return false; @@ -289,35 +289,6 @@ namespace currency return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_blocks_fuzzy_direct(const COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request& req, COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& res, connection_context& cntx) - { - CHECK_CORE_READY(); - - if (req.block_ids.back() != m_core.get_blockchain_storage().get_block_id_by_height(0)) - { - //genesis mismatch, return specific - res.status = CORE_RPC_STATUS_GENESIS_MISMATCH; - return true; - } - - blockchain_storage::blocks_direct_container bs; - if (!m_core.get_blockchain_storage().find_blockchain_supplement2(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) - { - res.status = CORE_RPC_STATUS_FAILED; - return false; - } - - for (auto& b : bs) - { - res.blocks.resize(res.blocks.size() + 1); - res.blocks.back().block_ptr = b.first; - res.blocks.back().txs_ptr = std::move(b.second); - } - - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx) { CHECK_CORE_READY(); @@ -1292,6 +1263,17 @@ namespace currency return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_est_height_from_date(const COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& req, COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& res, connection_context& cntx) + { + bool r = m_core.get_blockchain_storage().get_est_height_from_date(req.timestamp, res.h); + + if (r) + res.status = CORE_RPC_STATUS_OK; + else + res.status = CORE_RPC_STATUS_NOT_FOUND; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_alias_by_address(const COMMAND_RPC_GET_ALIASES_BY_ADDRESS::request& req, COMMAND_RPC_GET_ALIASES_BY_ADDRESS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) { account_public_address addr = AUTO_VAL_INIT(addr); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index ac1a21ff..21e463e1 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -90,6 +90,8 @@ namespace currency bool on_get_main_block_details(const COMMAND_RPC_GET_BLOCK_DETAILS::request& req, COMMAND_RPC_GET_BLOCK_DETAILS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_alt_block_details(const COMMAND_RPC_GET_BLOCK_DETAILS::request& req, COMMAND_RPC_GET_BLOCK_DETAILS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_alt_blocks_details(const COMMAND_RPC_GET_ALT_BLOCKS_DETAILS::request& req, COMMAND_RPC_GET_ALT_BLOCKS_DETAILS::response& res, connection_context& cntx); + bool on_get_est_height_from_date(const COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& req, COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& res, connection_context& cntx); + @@ -135,6 +137,7 @@ namespace currency MAP_JON_RPC_WE("get_alias_details", on_get_alias_details, COMMAND_RPC_GET_ALIAS_DETAILS) MAP_JON_RPC_WE("get_alias_by_address", on_alias_by_address, COMMAND_RPC_GET_ALIASES_BY_ADDRESS) MAP_JON_RPC_WE("get_alias_reward", on_get_alias_reward, COMMAND_RPC_GET_ALIAS_REWARD) + MAP_JON_RPC ("get_est_height_from_date", on_get_est_height_from_date, COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE) //block explorer api MAP_JON_RPC ("get_blocks_details", on_rpc_get_blocks_details, COMMAND_RPC_GET_BLOCKS_DETAILS) MAP_JON_RPC_WE("get_tx_details", on_get_tx_details, COMMAND_RPC_GET_TX_DETAILS) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 725f5860..b99c70f5 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -115,9 +115,11 @@ namespace currency struct request { + uint64_t minimum_height; std::list block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(minimum_height) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) END_KV_SERIALIZE_MAP() }; @@ -141,38 +143,6 @@ namespace currency typedef COMMAND_RPC_GET_BLOCKS_FAST_T COMMAND_RPC_GET_BLOCKS_FAST; typedef COMMAND_RPC_GET_BLOCKS_FAST_T COMMAND_RPC_GET_BLOCKS_DIRECT; - template - struct COMMAND_RPC_GET_BLOCKS_FUZZY_FAST_T - { - - struct request - { - std::list > block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) - END_KV_SERIALIZE_MAP() - }; - - struct response - { - std::list blocks; - uint64_t start_height; - uint64_t current_height; - std::string status; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(blocks) - KV_SERIALIZE(start_height) - KV_SERIALIZE(current_height) - KV_SERIALIZE(status) - END_KV_SERIALIZE_MAP() - }; - }; - typedef COMMAND_RPC_GET_BLOCKS_FUZZY_FAST_T COMMAND_RPC_GET_BLOCKS_FUZZY_FAST; - typedef COMMAND_RPC_GET_BLOCKS_FUZZY_FAST_T COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT; - - //----------------------------------------------- struct COMMAND_RPC_GET_TRANSACTIONS { @@ -199,6 +169,30 @@ namespace currency END_KV_SERIALIZE_MAP() }; }; + + //----------------------------------------------- + struct COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE + { + struct request + { + uint64_t timestamp; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(timestamp) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t h; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(h) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; //----------------------------------------------- struct COMMAND_RPC_GET_TX_POOL { diff --git a/src/wallet/core_default_rpc_proxy.cpp b/src/wallet/core_default_rpc_proxy.cpp index 4db9330d..68a4b3b2 100644 --- a/src/wallet/core_default_rpc_proxy.cpp +++ b/src/wallet/core_default_rpc_proxy.cpp @@ -50,6 +50,11 @@ namespace tools return r; } //------------------------------------------------------------------------------------------------------------------------------ + bool default_http_core_proxy::call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(const currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& rqt, currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& rsp) + { + return invoke_http_json_rpc_update_is_disconnect("get_est_height_from_date", rqt, rsp); + } + //------------------------------------------------------------------------------------------------------------------------------ bool default_http_core_proxy::call_COMMAND_RPC_GET_INFO(const currency::COMMAND_RPC_GET_INFO::request& req, currency::COMMAND_RPC_GET_INFO::response& res) { return invoke_http_json_remote_command2_update_is_disconnect("/getinfo", req, res); diff --git a/src/wallet/core_default_rpc_proxy.h b/src/wallet/core_default_rpc_proxy.h index 9800c56c..0bba7db6 100644 --- a/src/wallet/core_default_rpc_proxy.h +++ b/src/wallet/core_default_rpc_proxy.h @@ -28,6 +28,7 @@ namespace tools bool call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(const currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& rqt, currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& rsp) override; bool call_COMMAND_RPC_GET_BLOCKS_FAST(const currency::COMMAND_RPC_GET_BLOCKS_FAST::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FAST::response& rsp) override; bool call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& rsp) override; + bool call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(const currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& rqt, currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& rsp) override; bool call_COMMAND_RPC_GET_INFO(const currency::COMMAND_RPC_GET_INFO::request& rqt, currency::COMMAND_RPC_GET_INFO::response& rsp) override; bool call_COMMAND_RPC_GET_TX_POOL(const currency::COMMAND_RPC_GET_TX_POOL::request& rqt, currency::COMMAND_RPC_GET_TX_POOL::response& rsp) override; bool call_COMMAND_RPC_GET_ALIASES_BY_ADDRESS(const currency::COMMAND_RPC_GET_ALIASES_BY_ADDRESS::request& rqt, currency::COMMAND_RPC_GET_ALIASES_BY_ADDRESS::response& rsp) override; diff --git a/src/wallet/core_fast_rpc_proxy.h b/src/wallet/core_fast_rpc_proxy.h index 7ad94805..9f1a1225 100644 --- a/src/wallet/core_fast_rpc_proxy.h +++ b/src/wallet/core_fast_rpc_proxy.h @@ -33,6 +33,10 @@ namespace tools { return m_rpc.on_get_blocks_direct(rqt, rsp, m_cntxt_stub); } + bool call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(const currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& rqt, currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& rsp) override + { + return m_rpc.on_get_est_height_from_date(rqt, rsp, m_cntxt_stub); + } //------------------------------------------------------------------------------------------------------------------------------ bool call_COMMAND_RPC_GET_INFO(const currency::COMMAND_RPC_GET_INFO::request& req, currency::COMMAND_RPC_GET_INFO::response& res) override { diff --git a/src/wallet/core_rpc_proxy.h b/src/wallet/core_rpc_proxy.h index a64c6350..184e8a4b 100644 --- a/src/wallet/core_rpc_proxy.h +++ b/src/wallet/core_rpc_proxy.h @@ -22,7 +22,7 @@ namespace tools virtual bool call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(const currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& rqt, currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_BLOCKS_FAST(const currency::COMMAND_RPC_GET_BLOCKS_FAST::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FAST::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& rsp) { return false; } - virtual bool call_COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& rsp) { return false; } + virtual bool call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(const currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& rqt, currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& rsp) { return false; } virtual bool call_COMMAND_RPC_GET_INFO(const currency::COMMAND_RPC_GET_INFO::request& rqt, currency::COMMAND_RPC_GET_INFO::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_TX_POOL(const currency::COMMAND_RPC_GET_TX_POOL::request& rqt, currency::COMMAND_RPC_GET_TX_POOL::response& rsp){ return false; } virtual bool call_COMMAND_RPC_GET_ALIASES_BY_ADDRESS(const currency::COMMAND_RPC_GET_ALIASES_BY_ADDRESS::request& rqt, currency::COMMAND_RPC_GET_ALIASES_BY_ADDRESS::response& rsp){ return false; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 3d3c8933..5a1225a8 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1211,7 +1211,7 @@ bool wallet2::lookup_item_around(uint64_t i, std::pair& return true; } //---------------------------------------------------------------------------------------------------- -void wallet2::get_short_chain_history(std::list >& ids) +void wallet2::get_short_chain_history(std::list& ids) { ids.clear(); uint64_t i = 0; @@ -1222,7 +1222,7 @@ void wallet2::get_short_chain_history(std::listfirst, it->second }); + ids.push_back(it->second); i = it->first; } @@ -1242,20 +1242,41 @@ void wallet2::get_short_chain_history(std::listcall_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(req, res); + THROW_IF_FALSE_WALLET_EX(r, error::no_connection_to_daemon, "call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE"); + THROW_IF_FALSE_WALLET_EX(res.status == CORE_RPC_STATUS_OK, error::wallet_runtime_error, "FAILED TO CALL COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE"); + return res.h; } //---------------------------------------------------------------------------------------------------- void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) { blocks_added = 0; - currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request req = AUTO_VAL_INIT(req); - currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response res = AUTO_VAL_INIT(res); + currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request req = AUTO_VAL_INIT(req); + currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response res = AUTO_VAL_INIT(res); + + req.minimum_height = get_wallet_minimum_height(); get_short_chain_history(req.block_ids); - bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT(req, res); + bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); if (!r) throw error::no_connection_to_daemon(LOCATION_STR, "getblocks.bin"); @@ -1353,7 +1374,7 @@ void wallet2::check_if_block_matched(uint64_t i, const crypto::hash& id, bool& b } //---------------------------------------------------------------------------------------------------- void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop, - currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& res) + currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& res) { size_t current_index = res.start_height; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 352abe3c..6262628d 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -322,6 +322,7 @@ namespace tools m_do_rise_transfer(false), m_watch_only(false), m_last_pow_block_h(0), + m_minimum_height(0), m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE) {}; public: @@ -336,6 +337,7 @@ namespace tools m_log_prefix("???"), m_watch_only(false), m_last_pow_block_h(0), + m_minimum_height(0), m_pos_mint_packing_size(WALLET_DEFAULT_POS_MINT_PACKING_SIZE) { m_core_runtime_config = currency::get_default_core_runtime_config(); @@ -503,6 +505,7 @@ namespace tools bool set_core_proxy(const std::shared_ptr& proxy); void set_pos_mint_packing_size(uint64_t new_size); + void set_minimum_height(uint64_t h); std::shared_ptr get_core_proxy(); uint64_t balance() const; uint64_t balance(uint64_t& unloked, uint64_t& awaiting_in, uint64_t& awaiting_out, uint64_t& mined) const; @@ -895,9 +898,10 @@ private: void check_and_throw_if_self_directed_tx_with_payment_id_requested(const construct_tx_param& ctp); void push_new_block_id(const crypto::hash& id, uint64_t height); bool lookup_item_around(uint64_t i, std::pair& result); - void get_short_chain_history(std::list >& ids); + void get_short_chain_history(std::list& ids); void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed); uint64_t detach_from_block_ids(uint64_t height); + uint64_t get_wallet_minimum_height(); currency::account_base m_account; bool m_watch_only; @@ -911,6 +915,7 @@ private: std::map m_last_144_blocks_every_10; //1 day std::map m_last_144_blocks_every_100; //10 days std::map m_last_144_blocks_every_1000; //100 days + uint64_t m_minimum_height; std::atomic m_local_bc_size; //temporary workaround std::atomic m_last_bc_timestamp; diff --git a/src/wallet/wallets_manager.cpp b/src/wallet/wallets_manager.cpp index ec505ff6..0eb87850 100644 --- a/src/wallet/wallets_manager.cpp +++ b/src/wallet/wallets_manager.cpp @@ -841,6 +841,7 @@ std::string wallets_manager::generate_wallet(const std::wstring& path, const std try { w->generate(path, password); + w->set_minimum_height(m_last_daemon_height); owr.seed = w->get_account().get_restore_braindata(); } catch (const tools::error::file_exists&) From 281e43031744e1f49f71c51dfb9c80a5127eea1e Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Wed, 29 Apr 2020 01:23:48 +0200 Subject: [PATCH 08/15] moved shortener code into separate unit to create unit tests for it --- src/rpc/core_rpc_server.h | 1 - src/wallet/wallet2.cpp | 206 +---------------------- src/wallet/wallet2.h | 38 +++-- src/wallet/wallet_chain_shortener.cpp | 225 ++++++++++++++++++++++++++ src/wallet/wallet_chain_shortener.h | 49 ++++++ src/wallet/wallet_errors.h | 19 +++ src/wallet/wallets_manager.cpp | 2 +- 7 files changed, 320 insertions(+), 220 deletions(-) create mode 100644 src/wallet/wallet_chain_shortener.cpp create mode 100644 src/wallet/wallet_chain_shortener.h diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 21e463e1..d95dd52b 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -39,7 +39,6 @@ namespace currency bool init(const boost::program_options::variables_map& vm); bool on_get_blocks_direct(const COMMAND_RPC_GET_BLOCKS_DIRECT::request& req, COMMAND_RPC_GET_BLOCKS_DIRECT::response& res, connection_context& cntx); - bool on_get_blocks_fuzzy_direct(const COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::request& req, COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& res, connection_context& cntx); bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index fd0c2697..dce7b77f 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1100,54 +1100,7 @@ void wallet2::process_new_blockchain_entry(const currency::block& b, const curre m_wcallback->on_new_block(height, b); } //---------------------------------------------------------------------------------------------------- -void wallet2::push_new_block_id(const crypto::hash& id, uint64_t height) -{ - //primary 10 - //self check - if (!m_last_10_blocks.empty()) - { - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(get_blockchain_current_size() == height, "Inernal error: get_blockchain_current_height(){" << get_blockchain_current_size() << "} == height{" << height << "} is not equal"); - } - - m_last_10_blocks[height] = id; - if (m_last_10_blocks.size() > WALLET_EVERYBLOCK_SIZE) - { - m_last_10_blocks.erase(m_last_10_blocks.begin()); - } - //every 10-th - if (height % 10 == 0) - { - //self check - if (!m_last_144_blocks_every_10.empty()) - { - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_10.end())->first + 10 == height, "Inernal error: (--m_last_144_blocks_every_10.end())->first + 10{" << (--m_last_144_blocks_every_10.end())->first + 10 << "} == height{" << height << "} is not equal"); - } - m_last_144_blocks_every_10[height] = id; - } - //every 100-th - if (height % 100 == 0) - { - //self check - if (!m_last_144_blocks_every_100.empty()) - { - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_100.end())->first + 100 == height, "Inernal error: (--m_last_144_blocks_every_100.end())->first + 100{" << (--m_last_144_blocks_every_100.end())->first + 100 << "} == height{" << height << "} is not equal"); - } - m_last_144_blocks_every_100[height] = id; - } - //every 1000-th - //every 100-th - if (height % 1000 == 0) - { - //self check - if (!m_last_144_blocks_every_1000.empty()) - { - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_1000.end())->first + 1000 == height, "Inernal error: (--m_last_144_blocks_every_1000.end())->first + 1000{" << (--m_last_144_blocks_every_1000.end())->first + 1000 << "} == height{" << height << "} is not equal"); - } - m_last_144_blocks_every_1000[height] = id; - } - -} //---------------------------------------------------------------------------------------------------- // void wallet2::get_short_chain_history(std::list& ids) // { @@ -1176,78 +1129,7 @@ void wallet2::push_new_block_id(const crypto::hash& id, uint64_t height) // if(!genesis_included) // ids.push_back(m_blockchain[0]); // } -//---------------------------------------------------------------------------------------------------- -bool wallet2::lookup_item_around(uint64_t i, std::pair& result) -{ - //in which container we are looking for? - uint64_t devider = 0; - std::map* pcontainer; - if (m_last_144_blocks_every_10.size() && i < m_last_144_blocks_every_10.begin()->first) - { - devider = 10; - pcontainer = &m_last_144_blocks_every_10; - } - else if (m_last_144_blocks_every_100.size() && i < m_last_144_blocks_every_100.begin()->first) - { - devider = 100; - pcontainer = &m_last_144_blocks_every_100; - } - else if (m_last_144_blocks_every_1000.size() && i < m_last_144_blocks_every_1000.begin()->first) - { - devider = 1000; - pcontainer = &m_last_144_blocks_every_1000; - } - else - return false; - //look in every 10'th - i = i - i % devider; - auto it = pcontainer->find(i); - //self check - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it != pcontainer->end(), - "Inernal error: index " << i << " not found for devider " << devider - << " pcontainer={" << pcontainer->begin()->first << ":"<< (--pcontainer->end())->first <<"}"); - result = *it; - return true; -} -//---------------------------------------------------------------------------------------------------- -void wallet2::get_short_chain_history(std::list& ids) -{ - ids.clear(); - uint64_t i = 0; - uint64_t sz = get_blockchain_current_size(); - if (!sz) - return; - - //first put last 10 - for (auto it = m_last_10_blocks.rbegin(); it != m_last_10_blocks.rend(); it++) - { - ids.push_back(it->second); - i = it->first; - } - - uint64_t current_back_offset = m_last_10_blocks.size(); - //self check - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(current_back_offset == sz-i, "Inernal error: current_back_offset{" << current_back_offset << "} == sz-i{" << sz << " - " << i << "} is not equal"); - - uint64_t current_offset_distance = 10; - current_back_offset += 10; - while (current_back_offset < sz) - { - uint64_t get_item_around = sz - current_back_offset; - std::pair item = AUTO_VAL_INIT(item); - if (!lookup_item_around(get_item_around, item)) - break; - - //readjust item current_back_offset - current_back_offset = sz - item.first; - - ids.push_back(item.second); - current_offset_distance *= 2; - current_back_offset += current_offset_distance; - } - ids.push_back(m_genesis); -} //---------------------------------------------------------------------------------------------------- void wallet2::set_minimum_height(uint64_t h) { @@ -1264,7 +1146,7 @@ uint64_t wallet2::get_wallet_minimum_height() req.timestamp = m_account.get_createtime(); bool r = m_core_proxy->call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(req, res); THROW_IF_FALSE_WALLET_EX(r, error::no_connection_to_daemon, "call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE"); - THROW_IF_FALSE_WALLET_EX(res.status == CORE_RPC_STATUS_OK, error::wallet_runtime_error, "FAILED TO CALL COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE"); + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(res.status == CORE_RPC_STATUS_OK, "FAILED TO CALL COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE"); return res.h; } //---------------------------------------------------------------------------------------------------- @@ -1300,7 +1182,7 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) WLT_LOG_MAGENTA("New genesis set for wallet: " << new_genesis_id, LOG_LEVEL_0); get_short_chain_history(req.block_ids); //req.block_ids.push_back(new_genesis_id); - bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT(req, res); + bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "getblocks.bin"); } if (res.status == CORE_RPC_STATUS_BUSY) @@ -1316,62 +1198,7 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) handle_pulled_blocks(blocks_added, stop, res); } -//---------------------------------------------------------------------------------------------------- -void wallet2::check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed) -{ - if (!m_last_10_blocks.empty() && i > m_last_10_blocks.begin()->first) - { - //must be in short sequence (m_last_10_blocks) - //self check - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_10_blocks.end())->first >= i, - "Inernal error: index " << i << " is not located in expected range of m_last_10_blocks={" - << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); - auto it = m_last_10_blocks.find(i); - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it != m_last_10_blocks.end(), - "Inernal error: filde to find index " << i << " in m_last_10_blocks={" - << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); - - block_found = true; - if (id == it->second) - block_matched = true; - else - block_matched = false; - } - else - { - //lazy lookup - std::pair result = AUTO_VAL_INIT(result); - bool r = lookup_item_around(i, result); - if (!r) - { - WLT_LOG_L0("Wallet is getting fully resynced due to unmatched block " << id << " at " << i ); - block_matched = block_found = false; - full_reset_needed = true; - return; - } - else - { - if (result.first == i) - { - block_found = true; - if (result.second == id) - { - block_matched = true; - } - else - { - block_matched = false; - } - } - else - { - block_found = false; - block_matched = false; - } - } - } -} //---------------------------------------------------------------------------------------------------- void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop, currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& res) @@ -1949,24 +1776,14 @@ bool wallet2::refresh(size_t & blocks_fetched, bool& received_money, bool& ok, s } return ok; } -//---------------------------------------------------------------------------------------------------- -void clean_map_from_items_above(std::map& container, uint64_t height) -{ - while (container.size() && (--container.end())->first > height) - { - container.erase(--container.end()); - } -} + //---------------------------------------------------------------------------------------------------- uint64_t wallet2::detach_from_block_ids(uint64_t height) { //calculate number of erased blocks uint64_t blocks_detached = (get_blockchain_current_size() -1 ) - height; //id at height should be kept, the rest - erased - clean_map_from_items_above(m_last_10_blocks, height); - clean_map_from_items_above(m_last_10_blocks, height); - clean_map_from_items_above(m_last_10_blocks, height); - clean_map_from_items_above(m_last_10_blocks, height); + m_chain.detach(height); return blocks_detached; } //---------------------------------------------------------------------------------------------------- @@ -2064,11 +1881,7 @@ bool wallet2::clear() reset_all(); //currency::block b; //currency::generate_genesis_block(b); - m_local_bc_size = 1; - m_last_10_blocks.clear(); - m_last_144_blocks_every_10.clear(); - m_last_144_blocks_every_100.clear(); - m_last_144_blocks_every_1000.clear(); + m_chain.clear(); //m_blockchain.push_back(get_block_hash(b)); return true; } @@ -2076,10 +1889,7 @@ bool wallet2::clear() bool wallet2::reset_all() { //m_blockchain.clear(); - m_last_10_blocks.clear(); - m_last_144_blocks_every_10.clear(); - m_last_144_blocks_every_100.clear(); - m_last_144_blocks_every_1000.clear(); + m_chain.clear(); m_transfers.clear(); m_key_images.clear(); // m_pending_key_images is not cleared intentionally @@ -2092,7 +1902,7 @@ bool wallet2::reset_all() m_transfer_history.clear(); //m_account = AUTO_VAL_INIT(m_account); - m_local_bc_size = 1; //including genesis + //m_local_bc_size = 1; //including genesis m_last_bc_timestamp = 0; m_height_of_start_sync = 0; m_last_sync_percent = 0; @@ -3207,7 +3017,7 @@ bool wallet2::is_transfer_unlocked(const transfer_details& td, bool for_pos_mini uint64_t unlock_time = get_tx_unlock_time(td.m_ptx_wallet_info->m_tx, td.m_internal_output_index); - if (for_pos_mining && m_blockchain.size() > m_core_runtime_config.hard_fork_01_starts_after_height) + if (for_pos_mining && get_blockchain_current_size() > m_core_runtime_config.hard_fork_01_starts_after_height) { //allowed of staking locked coins with stake_lock_time = unlock_time; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 00d96f7e..6aaab37e 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -39,6 +39,7 @@ #include "currency_core/bc_offers_serialization.h" #include "currency_core/bc_escrow_service.h" #include "common/pod_array_file_container.h" +#include "wallet_chain_shortener.h" #define WALLET_DEFAULT_TX_SPENDABLE_AGE 10 @@ -46,11 +47,6 @@ #define WALLET_DEFAULT_POS_MINT_PACKING_SIZE 100 -#define WALLET_EVERYBLOCK_SIZE 10 -#define WALLET_EVERY_10_BLOCKS_SIZE 144 -#define WALLET_EVERY_100_BLOCKS_SIZE 144 -#define WALLET_EVERY_1000_BLOCKS_SIZE 144 - #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "wallet" @@ -644,10 +640,10 @@ namespace tools bool get_transfer_address(const std::string& adr_str, currency::account_public_address& addr, std::string& payment_id); inline uint64_t get_blockchain_current_size() const { - return m_local_bc_size; + return m_chain.get_blockchain_current_size(); } - uint64_t get_top_block_height() const { return m_blockchain.empty() ? 0 : m_blockchain.size() - 1; } + uint64_t get_top_block_height() const { return m_chain.get_top_block_height(); } template inline void serialize(t_archive &a, const unsigned int ver) @@ -687,12 +683,13 @@ namespace tools } else { - a & m_local_bc_size; - a & m_genesis; - a & m_last_10_blocks; - a & m_last_144_blocks_every_10; - a & m_last_144_blocks_every_100; - a & m_last_144_blocks_every_1000; + a & m_chain; +// a & m_local_bc_size; +// a & m_genesis; +// a & m_last_10_blocks; +// a & m_last_144_blocks_every_10; +// a & m_last_144_blocks_every_100; +// a & m_last_144_blocks_every_1000; } @@ -825,7 +822,7 @@ private: const std::vector& recipients, const std::vector& recipients_aliases); void handle_pulled_blocks(size_t& blocks_added, std::atomic& stop, - currency::COMMAND_RPC_GET_BLOCKS_FUZZY_DIRECT::response& blocks); + currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& blocks); std::string get_alias_for_address(const std::string& addr); static bool build_kernel(const currency::pos_entry& pe, const currency::stake_modifier_type& stake_modifier, currency::stake_kernel& kernel, uint64_t& coindays_weight, uint64_t timestamp); bool is_connected_to_net(); @@ -915,14 +912,14 @@ private: std::wstring m_pending_ki_file; std::string m_password; //std::vector m_blockchain; - crypto::hash m_genesis; - std::map m_last_10_blocks; - std::map m_last_144_blocks_every_10; //1 day - std::map m_last_144_blocks_every_100; //10 days - std::map m_last_144_blocks_every_1000; //100 days +// crypto::hash m_genesis; +// std::map m_last_10_blocks; +// std::map m_last_144_blocks_every_10; //1 day +// std::map m_last_144_blocks_every_100; //10 days +// std::map m_last_144_blocks_every_1000; //100 days uint64_t m_minimum_height; - std::atomic m_local_bc_size; //temporary workaround + //std::atomic m_local_bc_size; //temporary workaround std::atomic m_last_bc_timestamp; bool m_do_rise_transfer; uint64_t m_pos_mint_packing_size; @@ -949,6 +946,7 @@ private: uint64_t m_last_pow_block_h; currency::core_runtime_config m_core_runtime_config; escrow_contracts_container m_contracts; + wallet_chain_shortener m_chain; std::list m_money_expirations; //optimization for big wallets and batch tx diff --git a/src/wallet/wallet_chain_shortener.cpp b/src/wallet/wallet_chain_shortener.cpp new file mode 100644 index 00000000..84d0c05f --- /dev/null +++ b/src/wallet/wallet_chain_shortener.cpp @@ -0,0 +1,225 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Copyright (c) 2012-2013 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "wallet_chain_shortener.h" + +#define WALLET_EVERYBLOCK_SIZE 10 +#define WALLET_EVERY_10_BLOCKS_SIZE 144 +#define WALLET_EVERY_100_BLOCKS_SIZE 144 +#define WALLET_EVERY_1000_BLOCKS_SIZE 144 + + + +void wallet_chain_shortener::clear() +{ + m_local_bc_size = 1; + m_last_10_blocks.clear(); + m_last_144_blocks_every_10.clear(); + m_last_144_blocks_every_100.clear(); + m_last_144_blocks_every_1000.clear(); + +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet_chain_shortener::get_blockchain_current_size() const +{ + return m_local_bc_size; +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet_chain_shortener::get_top_block_height() const +{ + return m_local_bc_size - 1; +} +//---------------------------------------------------------------------------------------------------- +void wallet_chain_shortener::push_new_block_id(const crypto::hash& id, uint64_t height) +{ + //primary 10 + //self check + if (!m_last_10_blocks.empty()) + { + THROW_IF_FALSE_WALLET_INT_ERR_EX(get_blockchain_current_size() == height, "Inernal error: get_blockchain_current_height(){" << get_blockchain_current_size() << "} == height{" << height << "} is not equal"); + } + + m_last_10_blocks[height] = id; + if (m_last_10_blocks.size() > WALLET_EVERYBLOCK_SIZE) + { + m_last_10_blocks.erase(m_last_10_blocks.begin()); + } + + //every 10-th + if (height % 10 == 0) + { + //self check + if (!m_last_144_blocks_every_10.empty()) + { + THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_10.end())->first + 10 == height, "Inernal error: (--m_last_144_blocks_every_10.end())->first + 10{" << (--m_last_144_blocks_every_10.end())->first + 10 << "} == height{" << height << "} is not equal"); + } + m_last_144_blocks_every_10[height] = id; + } + //every 100-th + if (height % 100 == 0) + { + //self check + if (!m_last_144_blocks_every_100.empty()) + { + THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_100.end())->first + 100 == height, "Inernal error: (--m_last_144_blocks_every_100.end())->first + 100{" << (--m_last_144_blocks_every_100.end())->first + 100 << "} == height{" << height << "} is not equal"); + } + m_last_144_blocks_every_100[height] = id; + } + //every 1000-th + //every 100-th + if (height % 1000 == 0) + { + //self check + if (!m_last_144_blocks_every_1000.empty()) + { + THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_1000.end())->first + 1000 == height, "Inernal error: (--m_last_144_blocks_every_1000.end())->first + 1000{" << (--m_last_144_blocks_every_1000.end())->first + 1000 << "} == height{" << height << "} is not equal"); + } + m_last_144_blocks_every_1000[height] = id; + } +} +//---------------------------------------------------------------------------------------------------- +void wallet_chain_shortener::get_short_chain_history(std::list& ids)const +{ + ids.clear(); + uint64_t i = 0; + uint64_t sz = get_blockchain_current_size(); + if (!sz) + return; + + //first put last 10 + for (auto it = m_last_10_blocks.rbegin(); it != m_last_10_blocks.rend(); it++) + { + ids.push_back(it->second); + i = it->first; + } + + uint64_t current_back_offset = m_last_10_blocks.size(); + //self check + THROW_IF_FALSE_WALLET_INT_ERR_EX(current_back_offset == sz - i, "Inernal error: current_back_offset{" << current_back_offset << "} == sz-i{" << sz << " - " << i << "} is not equal"); + + uint64_t current_offset_distance = 10; + current_back_offset += 10; + while (current_back_offset < sz) + { + uint64_t get_item_around = sz - current_back_offset; + std::pair item = AUTO_VAL_INIT(item); + if (!lookup_item_around(get_item_around, item)) + break; + + //readjust item current_back_offset + current_back_offset = sz - item.first; + + ids.push_back(item.second); + current_offset_distance *= 2; + current_back_offset += current_offset_distance; + } + ids.push_back(m_genesis); +} +//---------------------------------------------------------------------------------------------------- +bool wallet_chain_shortener::lookup_item_around(uint64_t i, std::pair& result)const +{ + //in which container we are looking for? + uint64_t devider = 0; + std::map* pcontainer; + if (m_last_144_blocks_every_10.size() && i < m_last_144_blocks_every_10.begin()->first) + { + devider = 10; + pcontainer = &m_last_144_blocks_every_10; + } + else if (m_last_144_blocks_every_100.size() && i < m_last_144_blocks_every_100.begin()->first) + { + devider = 100; + pcontainer = &m_last_144_blocks_every_100; + } + else if (m_last_144_blocks_every_1000.size() && i < m_last_144_blocks_every_1000.begin()->first) + { + devider = 1000; + pcontainer = &m_last_144_blocks_every_1000; + } + else + return false; + + //look in every 10'th + i = i - i % devider; + auto it = pcontainer->find(i); + //self check + THROW_IF_FALSE_WALLET_INT_ERR_EX(it != pcontainer->end(), + "Inernal error: index " << i << " not found for devider " << devider + << " pcontainer={" << pcontainer->begin()->first << ":" << (--pcontainer->end())->first << "}"); + result = *it; + return true; +} +//---------------------------------------------------------------------------------------------------- +void wallet_chain_shortener::check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed)const +{ + if (!m_last_10_blocks.empty() && i > m_last_10_blocks.begin()->first) + { + //must be in short sequence (m_last_10_blocks) + //self check + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_10_blocks.end())->first >= i, + "Inernal error: index " << i << " is not located in expected range of m_last_10_blocks={" + << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); + + auto it = m_last_10_blocks.find(i); + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it != m_last_10_blocks.end(), + "Inernal error: filde to find index " << i << " in m_last_10_blocks={" + << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); + + block_found = true; + if (id == it->second) + block_matched = true; + else + block_matched = false; + } + else + { + //lazy lookup + std::pair result = AUTO_VAL_INIT(result); + bool r = lookup_item_around(i, result); + if (!r) + { + WLT_LOG_L0("Wallet is getting fully resynced due to unmatched block " << id << " at " << i); + block_matched = block_found = false; + full_reset_needed = true; + return; + } + else + { + if (result.first == i) + { + block_found = true; + if (result.second == id) + { + block_matched = true; + } + else + { + block_matched = false; + } + } + else + { + block_found = false; + block_matched = false; + } + } + } +} +//---------------------------------------------------------------------------------------------------- +void clean_map_from_items_above(std::map& container, uint64_t height) +{ + while (container.size() && (--container.end())->first > height) + { + container.erase(--container.end()); + } +} +void wallet_chain_shortener::detach(uint64_t height) +{ + clean_map_from_items_above(m_last_10_blocks, height); + clean_map_from_items_above(m_last_144_blocks_every_10, height); + clean_map_from_items_above(m_last_144_blocks_every_100, height); + clean_map_from_items_above(m_last_144_blocks_every_1000, height); +} \ No newline at end of file diff --git a/src/wallet/wallet_chain_shortener.h b/src/wallet/wallet_chain_shortener.h new file mode 100644 index 00000000..44d0180d --- /dev/null +++ b/src/wallet/wallet_chain_shortener.h @@ -0,0 +1,49 @@ +// Copyright (c) 2014-2018 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "include_base_utils.h" +#include "crypto/crypto.h" + + +class wallet_chain_shortener +{ +public: + void push_new_block_id(const crypto::hash& id, uint64_t height); + uint64_t get_top_block_height() const; + uint64_t get_blockchain_current_size() const; + void get_short_chain_history(std::list& ids)const; + bool lookup_item_around(uint64_t i, std::pair& result)const; + void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed) const; + void detach(uint64_t height); + void clear(); + template + inline void serialize(t_archive &a, const unsigned int ver) + { + a & m_local_bc_size; + a & m_genesis; + a & m_last_10_blocks; + a & m_last_144_blocks_every_10; + a & m_last_144_blocks_every_100; + a & m_last_144_blocks_every_1000; + } +private: + std::atomic m_local_bc_size; //temporary workaround + crypto::hash m_genesis; + std::map m_last_10_blocks; + std::map m_last_144_blocks_every_10; //1 day + std::map m_last_144_blocks_every_100; //10 days + std::map m_last_144_blocks_every_1000; //100 days +}; \ No newline at end of file diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 1430b35d..657f433f 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -696,3 +696,22 @@ if (cond) LOG_ERROR(" (" << #cond << ") is FALSE. THROW EXCEPTION: wallet_common_error"); \ tools::error::throw_wallet_ex(std::string(__FILE__ ":" STRINGIZE(__LINE__)), ss.str()); \ } + + +// wallet-specific logging functions +#define WLT_LOG_L0(msg) LOG_PRINT_L0("[W:" << m_log_prefix << "] " << msg) +#define WLT_LOG_L1(msg) LOG_PRINT_L1("[W:" << m_log_prefix << "] " << msg) +#define WLT_LOG_L2(msg) LOG_PRINT_L2("[W:" << m_log_prefix << "] " << msg) +#define WLT_LOG_L3(msg) LOG_PRINT_L3("[W:" << m_log_prefix << "] " << msg) +#define WLT_LOG_L4(msg) LOG_PRINT_L4("[W:" << m_log_prefix << "] " << msg) +#define WLT_LOG_ERROR(msg) LOG_ERROR("[W:" << m_log_prefix << "] " << msg) +#define WLT_LOG_BLUE(msg, log_level) LOG_PRINT_BLUE("[W:" << m_log_prefix << "] " << msg, log_level) +#define WLT_LOG_CYAN(msg, log_level) LOG_PRINT_CYAN("[W:" << m_log_prefix << "] " << msg, log_level) +#define WLT_LOG_GREEN(msg, log_level) LOG_PRINT_GREEN("[W:" << m_log_prefix << "] " << msg, log_level) +#define WLT_LOG_MAGENTA(msg, log_level) LOG_PRINT_MAGENTA("[W:" << m_log_prefix << "] " << msg, log_level) +#define WLT_LOG_RED(msg, log_level) LOG_PRINT_RED("[W:" << m_log_prefix << "] " << msg, log_level) +#define WLT_LOG_YELLOW(msg, log_level) LOG_PRINT_YELLOW("[W:" << m_log_prefix << "] " << msg, log_level) +#define WLT_CHECK_AND_ASSERT_MES(expr, ret, msg) CHECK_AND_ASSERT_MES(expr, ret, "[W:" << m_log_prefix << "] " << msg) +#define WLT_CHECK_AND_ASSERT_MES_NO_RET(expr, msg) CHECK_AND_ASSERT_MES_NO_RET(expr, "[W:" << m_log_prefix << "] " << msg) +#define WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(cond, msg) THROW_IF_FALSE_WALLET_INT_ERR_EX(cond, "[W:" << m_log_prefix << "] " << msg) +#define WLT_THROW_IF_FALSE_WALLET_CMN_ERR_EX(cond, msg) THROW_IF_FALSE_WALLET_CMN_ERR_EX(cond, "[W:" << m_log_prefix << "] " << msg) diff --git a/src/wallet/wallets_manager.cpp b/src/wallet/wallets_manager.cpp index 346539a0..2c8b10bf 100644 --- a/src/wallet/wallets_manager.cpp +++ b/src/wallet/wallets_manager.cpp @@ -1188,7 +1188,7 @@ std::string wallets_manager::transfer(size_t wallet_id, const view::transfer_par if (tp.lock_time > CURRENCY_MAX_BLOCK_NUMBER) unlock_time = tp.lock_time; else - unlock_time = w->get()->get_blockchain_current_height() + tp.lock_time; + unlock_time = w->get()->get_blockchain_current_size() + tp.lock_time; } From 16de0b986f5be329ba68d4bca6177136735854db Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Thu, 30 Apr 2020 22:29:08 +0200 Subject: [PATCH 09/15] implemented and tested(basic test) short_chain_history --- contrib/epee/include/string_tools.h | 8 ++ src/currency_core/currency_basic.h | 2 + src/currency_core/currency_config.h | 1 + src/wallet/wallet2.cpp | 19 ++-- src/wallet/wallet_chain_shortener.cpp | 94 ++++++++++++++----- src/wallet/wallet_chain_shortener.h | 12 ++- src/wallet/wallet_errors.h | 19 ---- .../wallet_chain_shortener_test.cpp | 31 ++++++ 8 files changed, 128 insertions(+), 58 deletions(-) create mode 100644 tests/unit_tests/wallet_chain_shortener_test.cpp diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 4b153d51..6b9abed1 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -603,6 +603,14 @@ POP_GCC_WARNINGS s = *(t_pod_type*)bin_buff.data(); return true; } + //---------------------------------------------------------------------------- + template + t_pod_type hex_to_pod(const std::string& hex_str) + { + t_pod_type p = AUTO_VAL_INIT(p); + hex_to_pod(hex_str, p); + return p; + } //---------------------------------------------------------------------------- inline std::string get_extension(const std::string& str) { diff --git a/src/currency_core/currency_basic.h b/src/currency_core/currency_basic.h index 74f5c824..876155ea 100644 --- a/src/currency_core/currency_basic.h +++ b/src/currency_core/currency_basic.h @@ -51,6 +51,8 @@ namespace currency const static crypto::signature null_sig = AUTO_VAL_INIT(null_sig); const static crypto::key_derivation null_derivation = AUTO_VAL_INIT(null_derivation); + const static crypto::hash gdefault_genesis = epee::string_tools::hex_to_pod("CC608F59F8080E2FBFE3C8C80EB6E6A953D47CF2D6AEBD345BADA3A1CAB99852"); + typedef std::string payment_id_t; diff --git a/src/currency_core/currency_config.h b/src/currency_core/currency_config.h index 7be149de..c9991207 100644 --- a/src/currency_core/currency_config.h +++ b/src/currency_core/currency_config.h @@ -233,6 +233,7 @@ #endif + static_assert(CURRENCY_MINER_TX_MAX_OUTS <= CURRENCY_TX_MAX_ALLOWED_OUTS, "Miner tx must obey normal tx max outs limit"); static_assert(PREMINE_AMOUNT / WALLET_MAX_ALLOWED_OUTPUT_AMOUNT < CURRENCY_MINER_TX_MAX_OUTS, "Premine can't be divided into reasonable number of outs"); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index dce7b77f..dbf724a0 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1091,8 +1091,7 @@ void wallet2::process_new_blockchain_entry(const currency::block& b, const curre { WLT_LOG_L3( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); } - push_new_block_id(bl_id, height); //m_blockchain.push_back(bl_id); - m_local_bc_size = height + 1; + m_chain.push_new_block_id(bl_id, height); //m_blockchain.push_back(bl_id); m_last_bc_timestamp = b.timestamp; if (!is_pos_block(b)) m_last_pow_block_h = height; @@ -1178,7 +1177,7 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) r = string_tools::parse_tpod_from_hex_string(gbd_res.blocks.back().id, new_genesis_id); THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "get_blocks_details"); reset_all(); - m_genesis = new_genesis_id; + m_chain.set_genesis(new_genesis_id); WLT_LOG_MAGENTA("New genesis set for wallet: " << new_genesis_id, LOG_LEVEL_0); get_short_chain_history(req.block_ids); //req.block_ids.push_back(new_genesis_id); @@ -1812,7 +1811,6 @@ void wallet2::detach_blockchain(uint64_t height) } size_t blocks_detached = detach_from_block_ids(height); - m_local_bc_size -= height+1; //rollback spends // do not clear spent flag in spent transfers as corresponding txs are most likely in the pool @@ -4116,25 +4114,24 @@ void wallet2::process_genesis_if_needed(const currency::block& genesis) THROW_IF_TRUE_WALLET_EX(get_blockchain_current_size() > 1, error::wallet_internal_error, "Can't change wallet genesis block once the blockchain has been populated"); crypto::hash genesis_hash = get_block_hash(genesis); - if (get_blockchain_current_size() == 1 && m_genesis != genesis_hash) - WLT_LOG_L0("Changing genesis block for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_genesis << " -> " << genesis_hash); + if (get_blockchain_current_size() == 1 && m_chain.get_genesis() != genesis_hash) + WLT_LOG_L0("Changing genesis block for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_chain.get_genesis() << " -> " << genesis_hash); //m_blockchain.clear(); //m_blockchain.push_back(genesis_hash); - m_genesis = genesis_hash; - m_local_bc_size = 1; + m_chain.set_genesis(genesis_hash); m_last_bc_timestamp = genesis.timestamp; WLT_LOG_L2("Processing genesis block: " << genesis_hash); process_new_transaction(genesis.miner_tx, 0, genesis); } - +//---------------------------------------------------------------------------------------------------- void wallet2::set_genesis(const crypto::hash& genesis_hash) { THROW_IF_TRUE_WALLET_EX(get_blockchain_current_size() != 1, error::wallet_internal_error, "Can't change wallet genesis hash once the blockchain has been populated"); - WLT_LOG_L0("Changing genesis hash for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_genesis << " -> " << genesis_hash); - m_genesis = genesis_hash; + WLT_LOG_L0("Changing genesis hash for wallet " << m_account.get_public_address_str() << ":" << ENDL << " " << m_chain.get_genesis() << " -> " << genesis_hash); + m_chain.set_genesis(genesis_hash); } //---------------------------------------------------------------------------------------------------- void wallet2::print_tx_sent_message(const currency::transaction& tx, const std::string& description, uint64_t fee /* = UINT64_MAX */) diff --git a/src/wallet/wallet_chain_shortener.cpp b/src/wallet/wallet_chain_shortener.cpp index 84d0c05f..71300974 100644 --- a/src/wallet/wallet_chain_shortener.cpp +++ b/src/wallet/wallet_chain_shortener.cpp @@ -5,18 +5,24 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "wallet_chain_shortener.h" +#include "wallet_errors.h" -#define WALLET_EVERYBLOCK_SIZE 10 +#define WALLET_EVERYBLOCK_SIZE 20 #define WALLET_EVERY_10_BLOCKS_SIZE 144 #define WALLET_EVERY_100_BLOCKS_SIZE 144 #define WALLET_EVERY_1000_BLOCKS_SIZE 144 +void exception_handler(){} +wallet_chain_shortener::wallet_chain_shortener(): m_genesis(currency::gdefault_genesis) +{ + m_local_bc_size = 1; +} void wallet_chain_shortener::clear() { m_local_bc_size = 1; - m_last_10_blocks.clear(); + m_last_20_blocks.clear(); m_last_144_blocks_every_10.clear(); m_last_144_blocks_every_100.clear(); m_last_144_blocks_every_1000.clear(); @@ -33,19 +39,32 @@ uint64_t wallet_chain_shortener::get_top_block_height() const return m_local_bc_size - 1; } //---------------------------------------------------------------------------------------------------- +void wallet_chain_shortener::set_genesis(const crypto::hash& id) +{ + m_genesis = id; + m_local_bc_size = 1; +} +//---------------------------------------------------------------------------------------------------- +const crypto::hash& wallet_chain_shortener::get_genesis() +{ + return m_genesis; +} +//---------------------------------------------------------------------------------------------------- void wallet_chain_shortener::push_new_block_id(const crypto::hash& id, uint64_t height) { + //primary 10 //self check - if (!m_last_10_blocks.empty()) + if (!m_last_20_blocks.empty()) { THROW_IF_FALSE_WALLET_INT_ERR_EX(get_blockchain_current_size() == height, "Inernal error: get_blockchain_current_height(){" << get_blockchain_current_size() << "} == height{" << height << "} is not equal"); } - m_last_10_blocks[height] = id; - if (m_last_10_blocks.size() > WALLET_EVERYBLOCK_SIZE) + m_local_bc_size++; + m_last_20_blocks[height] = id; + if (m_last_20_blocks.size() > WALLET_EVERYBLOCK_SIZE) { - m_last_10_blocks.erase(m_last_10_blocks.begin()); + m_last_20_blocks.erase(m_last_20_blocks.begin()); } //every 10-th @@ -57,6 +76,10 @@ void wallet_chain_shortener::push_new_block_id(const crypto::hash& id, uint64_t THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_10.end())->first + 10 == height, "Inernal error: (--m_last_144_blocks_every_10.end())->first + 10{" << (--m_last_144_blocks_every_10.end())->first + 10 << "} == height{" << height << "} is not equal"); } m_last_144_blocks_every_10[height] = id; + if (m_last_144_blocks_every_10.size() > WALLET_EVERY_10_BLOCKS_SIZE) + { + m_last_144_blocks_every_10.erase(m_last_144_blocks_every_10.begin()); + } } //every 100-th if (height % 100 == 0) @@ -67,6 +90,10 @@ void wallet_chain_shortener::push_new_block_id(const crypto::hash& id, uint64_t THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_100.end())->first + 100 == height, "Inernal error: (--m_last_144_blocks_every_100.end())->first + 100{" << (--m_last_144_blocks_every_100.end())->first + 100 << "} == height{" << height << "} is not equal"); } m_last_144_blocks_every_100[height] = id; + if (m_last_144_blocks_every_100.size() > WALLET_EVERY_100_BLOCKS_SIZE) + { + m_last_144_blocks_every_100.erase(m_last_144_blocks_every_100.begin()); + } } //every 1000-th //every 100-th @@ -78,7 +105,14 @@ void wallet_chain_shortener::push_new_block_id(const crypto::hash& id, uint64_t THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_144_blocks_every_1000.end())->first + 1000 == height, "Inernal error: (--m_last_144_blocks_every_1000.end())->first + 1000{" << (--m_last_144_blocks_every_1000.end())->first + 1000 << "} == height{" << height << "} is not equal"); } m_last_144_blocks_every_1000[height] = id; + if (m_last_144_blocks_every_1000.size() > WALLET_EVERY_1000_BLOCKS_SIZE) + { + m_last_144_blocks_every_1000.erase(m_last_144_blocks_every_1000.begin()); + } + } + + } //---------------------------------------------------------------------------------------------------- void wallet_chain_shortener::get_short_chain_history(std::list& ids)const @@ -90,18 +124,19 @@ void wallet_chain_shortener::get_short_chain_history(std::list& id return; //first put last 10 - for (auto it = m_last_10_blocks.rbegin(); it != m_last_10_blocks.rend(); it++) + uint64_t count = 0; + for (auto it = m_last_20_blocks.rbegin(); it != m_last_20_blocks.rend() && count != 10; it++) { ids.push_back(it->second); i = it->first; + count++; } - uint64_t current_back_offset = m_last_10_blocks.size(); + uint64_t current_back_offset = ids.size()+1; //self check - THROW_IF_FALSE_WALLET_INT_ERR_EX(current_back_offset == sz - i, "Inernal error: current_back_offset{" << current_back_offset << "} == sz-i{" << sz << " - " << i << "} is not equal"); + THROW_IF_FALSE_WALLET_INT_ERR_EX(current_back_offset == sz - i + 1, "Inernal error: current_back_offset{" << current_back_offset << "} == sz-i{" << sz << " - " << i << "} is not equal"); - uint64_t current_offset_distance = 10; - current_back_offset += 10; + uint64_t current_offset_distance = 1; while (current_back_offset < sz) { uint64_t get_item_around = sz - current_back_offset; @@ -123,18 +158,23 @@ bool wallet_chain_shortener::lookup_item_around(uint64_t i, std::pair* pcontainer; - if (m_last_144_blocks_every_10.size() && i < m_last_144_blocks_every_10.begin()->first) + const std::map* pcontainer; + if (m_last_20_blocks.size() && i >= m_last_20_blocks.begin()->first) + { + devider = 1; + pcontainer = &m_last_20_blocks; + } + else if (m_last_144_blocks_every_10.size() && i >= m_last_144_blocks_every_10.begin()->first) { devider = 10; pcontainer = &m_last_144_blocks_every_10; } - else if (m_last_144_blocks_every_100.size() && i < m_last_144_blocks_every_100.begin()->first) + else if (m_last_144_blocks_every_100.size() && i >= m_last_144_blocks_every_100.begin()->first) { devider = 100; pcontainer = &m_last_144_blocks_every_100; } - else if (m_last_144_blocks_every_1000.size() && i < m_last_144_blocks_every_1000.begin()->first) + else if (m_last_144_blocks_every_1000.size() && i >= m_last_144_blocks_every_1000.begin()->first) { devider = 1000; pcontainer = &m_last_144_blocks_every_1000; @@ -155,18 +195,18 @@ bool wallet_chain_shortener::lookup_item_around(uint64_t i, std::pair m_last_10_blocks.begin()->first) + if (!m_last_20_blocks.empty() && i > m_last_20_blocks.begin()->first) { - //must be in short sequence (m_last_10_blocks) + //must be in short sequence (m_last_20_blocks) //self check - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_10_blocks.end())->first >= i, - "Inernal error: index " << i << " is not located in expected range of m_last_10_blocks={" - << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); + THROW_IF_FALSE_WALLET_INT_ERR_EX((--m_last_20_blocks.end())->first >= i, + "Inernal error: index " << i << " is not located in expected range of m_last_20_blocks={" + << m_last_20_blocks.begin()->first << ":" << (--m_last_20_blocks.end())->first << "}"); - auto it = m_last_10_blocks.find(i); - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it != m_last_10_blocks.end(), - "Inernal error: filde to find index " << i << " in m_last_10_blocks={" - << m_last_10_blocks.begin()->first << ":" << (--m_last_10_blocks.end())->first << "}"); + auto it = m_last_20_blocks.find(i); + THROW_IF_FALSE_WALLET_INT_ERR_EX(it != m_last_20_blocks.end(), + "Inernal error: filde to find index " << i << " in m_last_20_blocks={" + << m_last_20_blocks.begin()->first << ":" << (--m_last_20_blocks.end())->first << "}"); block_found = true; if (id == it->second) @@ -181,7 +221,7 @@ void wallet_chain_shortener::check_if_block_matched(uint64_t i, const crypto::ha bool r = lookup_item_around(i, result); if (!r) { - WLT_LOG_L0("Wallet is getting fully resynced due to unmatched block " << id << " at " << i); + LOG_PRINT_L0("Wallet is getting fully resynced due to unmatched block " << id << " at " << i); block_matched = block_found = false; full_reset_needed = true; return; @@ -216,10 +256,12 @@ void clean_map_from_items_above(std::map& container, uin container.erase(--container.end()); } } +//---------------------------------------------------------------------------------------------------- void wallet_chain_shortener::detach(uint64_t height) { - clean_map_from_items_above(m_last_10_blocks, height); + clean_map_from_items_above(m_last_20_blocks, height); clean_map_from_items_above(m_last_144_blocks_every_10, height); clean_map_from_items_above(m_last_144_blocks_every_100, height); clean_map_from_items_above(m_last_144_blocks_every_1000, height); + m_local_bc_size = height + 1; } \ No newline at end of file diff --git a/src/wallet/wallet_chain_shortener.h b/src/wallet/wallet_chain_shortener.h index 44d0180d..29387c00 100644 --- a/src/wallet/wallet_chain_shortener.h +++ b/src/wallet/wallet_chain_shortener.h @@ -14,6 +14,8 @@ #include #include +#include "crypto/crypto.h" + #include "include_base_utils.h" #include "crypto/crypto.h" @@ -21,6 +23,7 @@ class wallet_chain_shortener { public: + wallet_chain_shortener(); void push_new_block_id(const crypto::hash& id, uint64_t height); uint64_t get_top_block_height() const; uint64_t get_blockchain_current_size() const; @@ -29,20 +32,25 @@ public: void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed) const; void detach(uint64_t height); void clear(); + void set_genesis(const crypto::hash& id); + const crypto::hash& get_genesis(); template inline void serialize(t_archive &a, const unsigned int ver) { a & m_local_bc_size; a & m_genesis; - a & m_last_10_blocks; + a & m_last_20_blocks; a & m_last_144_blocks_every_10; a & m_last_144_blocks_every_100; a & m_last_144_blocks_every_1000; } + + //debug functions + private: std::atomic m_local_bc_size; //temporary workaround crypto::hash m_genesis; - std::map m_last_10_blocks; + std::map m_last_20_blocks; std::map m_last_144_blocks_every_10; //1 day std::map m_last_144_blocks_every_100; //10 days std::map m_last_144_blocks_every_1000; //100 days diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 657f433f..1430b35d 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -696,22 +696,3 @@ if (cond) LOG_ERROR(" (" << #cond << ") is FALSE. THROW EXCEPTION: wallet_common_error"); \ tools::error::throw_wallet_ex(std::string(__FILE__ ":" STRINGIZE(__LINE__)), ss.str()); \ } - - -// wallet-specific logging functions -#define WLT_LOG_L0(msg) LOG_PRINT_L0("[W:" << m_log_prefix << "] " << msg) -#define WLT_LOG_L1(msg) LOG_PRINT_L1("[W:" << m_log_prefix << "] " << msg) -#define WLT_LOG_L2(msg) LOG_PRINT_L2("[W:" << m_log_prefix << "] " << msg) -#define WLT_LOG_L3(msg) LOG_PRINT_L3("[W:" << m_log_prefix << "] " << msg) -#define WLT_LOG_L4(msg) LOG_PRINT_L4("[W:" << m_log_prefix << "] " << msg) -#define WLT_LOG_ERROR(msg) LOG_ERROR("[W:" << m_log_prefix << "] " << msg) -#define WLT_LOG_BLUE(msg, log_level) LOG_PRINT_BLUE("[W:" << m_log_prefix << "] " << msg, log_level) -#define WLT_LOG_CYAN(msg, log_level) LOG_PRINT_CYAN("[W:" << m_log_prefix << "] " << msg, log_level) -#define WLT_LOG_GREEN(msg, log_level) LOG_PRINT_GREEN("[W:" << m_log_prefix << "] " << msg, log_level) -#define WLT_LOG_MAGENTA(msg, log_level) LOG_PRINT_MAGENTA("[W:" << m_log_prefix << "] " << msg, log_level) -#define WLT_LOG_RED(msg, log_level) LOG_PRINT_RED("[W:" << m_log_prefix << "] " << msg, log_level) -#define WLT_LOG_YELLOW(msg, log_level) LOG_PRINT_YELLOW("[W:" << m_log_prefix << "] " << msg, log_level) -#define WLT_CHECK_AND_ASSERT_MES(expr, ret, msg) CHECK_AND_ASSERT_MES(expr, ret, "[W:" << m_log_prefix << "] " << msg) -#define WLT_CHECK_AND_ASSERT_MES_NO_RET(expr, msg) CHECK_AND_ASSERT_MES_NO_RET(expr, "[W:" << m_log_prefix << "] " << msg) -#define WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(cond, msg) THROW_IF_FALSE_WALLET_INT_ERR_EX(cond, "[W:" << m_log_prefix << "] " << msg) -#define WLT_THROW_IF_FALSE_WALLET_CMN_ERR_EX(cond, msg) THROW_IF_FALSE_WALLET_CMN_ERR_EX(cond, "[W:" << m_log_prefix << "] " << msg) diff --git a/tests/unit_tests/wallet_chain_shortener_test.cpp b/tests/unit_tests/wallet_chain_shortener_test.cpp new file mode 100644 index 00000000..0a228579 --- /dev/null +++ b/tests/unit_tests/wallet_chain_shortener_test.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + +#include "gtest/gtest.h" +#include "wallet/wallet_chain_shortener.h" + +TEST(wallet_chain_shortener, wallet_chain_shortener) +{ + uint64_t counter = 0; + wallet_chain_shortener ws; + + for (counter = 1; counter != 1000000; counter++) + { + crypto::hash id_ = AUTO_VAL_INIT(id_); + *((uint64_t*)&id_) = counter; + + ws.push_new_block_id(id_, counter); + } + + std::list short_chain; + ws.get_short_chain_history(short_chain); + for(auto& id: short_chain) + { + LOG_PRINT_L0("{" << *((uint64_t*)&id) << "}{" << counter - *((uint64_t*)&id) << "}" << ENDL); + } + LOG_PRINT_L0("Finished"); + +} + From 3d1a9779910ca8e404beec5882c54e4d2b558771 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Thu, 30 Apr 2020 23:21:14 +0200 Subject: [PATCH 10/15] fixed bugs with detach --- src/wallet/wallet_chain_shortener.cpp | 2 +- .../wallet_chain_shortener_test.cpp | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet_chain_shortener.cpp b/src/wallet/wallet_chain_shortener.cpp index 71300974..e8778b66 100644 --- a/src/wallet/wallet_chain_shortener.cpp +++ b/src/wallet/wallet_chain_shortener.cpp @@ -134,7 +134,7 @@ void wallet_chain_shortener::get_short_chain_history(std::list& id uint64_t current_back_offset = ids.size()+1; //self check - THROW_IF_FALSE_WALLET_INT_ERR_EX(current_back_offset == sz - i + 1, "Inernal error: current_back_offset{" << current_back_offset << "} == sz-i{" << sz << " - " << i << "} is not equal"); + THROW_IF_FALSE_WALLET_INT_ERR_EX(current_back_offset == sz - i + 1 || !count, "Inernal error: current_back_offset{" << current_back_offset << "} == sz-i{" << sz << " - " << i << "} is not equal"); uint64_t current_offset_distance = 1; while (current_back_offset < sz) diff --git a/tests/unit_tests/wallet_chain_shortener_test.cpp b/tests/unit_tests/wallet_chain_shortener_test.cpp index 0a228579..6b2c0f0a 100644 --- a/tests/unit_tests/wallet_chain_shortener_test.cpp +++ b/tests/unit_tests/wallet_chain_shortener_test.cpp @@ -25,6 +25,31 @@ TEST(wallet_chain_shortener, wallet_chain_shortener) { LOG_PRINT_L0("{" << *((uint64_t*)&id) << "}{" << counter - *((uint64_t*)&id) << "}" << ENDL); } + + + ws.detach(counter - 10000); + short_chain.clear(); + ws.get_short_chain_history(short_chain); + for (auto& id : short_chain) + { + LOG_PRINT_L0("{" << *((uint64_t*)&id) << "}{" << counter - *((uint64_t*)&id) << "}" << ENDL); + } + + for (counter = counter - 10000 + 1; counter != 1000000; counter++) + { + crypto::hash id_ = AUTO_VAL_INIT(id_); + *((uint64_t*)&id_) = counter; + + ws.push_new_block_id(id_, counter); + } + + short_chain.clear(); + ws.get_short_chain_history(short_chain); + for (auto& id : short_chain) + { + LOG_PRINT_L0("{" << *((uint64_t*)&id) << "}{" << counter - *((uint64_t*)&id) << "}" << ENDL); + } + LOG_PRINT_L0("Finished"); } From 00dae663202038288590fb84a6d9fe8151bbab58 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 1 May 2020 13:25:28 +0200 Subject: [PATCH 11/15] finished shortener test --- .../wallet_chain_shortener_test.cpp | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/unit_tests/wallet_chain_shortener_test.cpp b/tests/unit_tests/wallet_chain_shortener_test.cpp index 6b2c0f0a..9ef2a93d 100644 --- a/tests/unit_tests/wallet_chain_shortener_test.cpp +++ b/tests/unit_tests/wallet_chain_shortener_test.cpp @@ -19,18 +19,18 @@ TEST(wallet_chain_shortener, wallet_chain_shortener) ws.push_new_block_id(id_, counter); } - std::list short_chain; - ws.get_short_chain_history(short_chain); - for(auto& id: short_chain) + std::list short_chain1; + ws.get_short_chain_history(short_chain1); + for(auto& id: short_chain1) { LOG_PRINT_L0("{" << *((uint64_t*)&id) << "}{" << counter - *((uint64_t*)&id) << "}" << ENDL); } ws.detach(counter - 10000); - short_chain.clear(); - ws.get_short_chain_history(short_chain); - for (auto& id : short_chain) + std::list short_chain2; + ws.get_short_chain_history(short_chain2); + for (auto& id : short_chain2) { LOG_PRINT_L0("{" << *((uint64_t*)&id) << "}{" << counter - *((uint64_t*)&id) << "}" << ENDL); } @@ -43,14 +43,17 @@ TEST(wallet_chain_shortener, wallet_chain_shortener) ws.push_new_block_id(id_, counter); } - short_chain.clear(); - ws.get_short_chain_history(short_chain); - for (auto& id : short_chain) + std::list short_chain3; + ws.get_short_chain_history(short_chain3); + for (auto& id : short_chain3) { LOG_PRINT_L0("{" << *((uint64_t*)&id) << "}{" << counter - *((uint64_t*)&id) << "}" << ENDL); } LOG_PRINT_L0("Finished"); + ASSERT_EQ(short_chain3.size(), short_chain1.size()); + ASSERT_EQ(short_chain3, short_chain1); + } From d67cc07e9b4af4a62dde8ad607ec5dbd73190dbd Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 1 May 2020 13:27:32 +0200 Subject: [PATCH 12/15] got rid of eos tutorial(it triggered github Security Alert) --- .../tutorial/code/tutorial_pba_0.cpp | 56 - .../tutorial/code/tutorial_pba_1.cpp | 68 - .../tutorial/code/tutorial_pba_10.cpp | 162 -- .../tutorial/code/tutorial_pba_10b.cpp | 198 -- .../tutorial/code/tutorial_pba_11.cpp | 187 -- .../tutorial/code/tutorial_pba_2.cpp | 105 - .../tutorial/code/tutorial_pba_3.cpp | 70 - .../tutorial/code/tutorial_pba_4.cpp | 105 - .../tutorial/code/tutorial_pba_5.cpp | 111 - .../tutorial/code/tutorial_pba_6.cpp | 112 - .../tutorial/code/tutorial_pba_7.cpp | 54 - .../tutorial/code/tutorial_pba_8.cpp | 118 - .../tutorial/code/tutorial_pba_9.cpp | 134 - .../tutorial/images/boost.png | Bin 6308 -> 0 bytes .../tutorial/images/c++-source-code.png | Bin 1296 -> 0 bytes .../tutorial/scripts/jquery-1.4.min.js | 151 -- .../tutorial/styles/boost.css | 39 - .../tutorial/styles/style.css | 92 - .../tutorial/tutorial.html | 2348 ----------------- 19 files changed, 4110 deletions(-) delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_0.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_1.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_10.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_10b.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_11.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_2.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_3.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_4.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_5.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_6.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_7.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_8.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/code/tutorial_pba_9.cpp delete mode 100644 contrib/eos_portable_archive/tutorial/images/boost.png delete mode 100644 contrib/eos_portable_archive/tutorial/images/c++-source-code.png delete mode 100644 contrib/eos_portable_archive/tutorial/scripts/jquery-1.4.min.js delete mode 100644 contrib/eos_portable_archive/tutorial/styles/boost.css delete mode 100644 contrib/eos_portable_archive/tutorial/styles/style.css delete mode 100644 contrib/eos_portable_archive/tutorial/tutorial.html diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_0.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_0.cpp deleted file mode 100644 index aaec78c7..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_0.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/** tutorial_pba_0.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This quick start example shows how to store some variables - * of basic types (bool, integer, floating point numbers, STL string) - * using the portable binary archive format associated to a - * standard output file stream. - * - */ - -#include -#include - -#include -#include - -int main (void) -{ - // The name for the example data file : - std::string filename = "pba_0.data"; - - // Some variables of various primitive types : - bool b = true; - char c = 'B'; - uint32_t answer = 42; - float computing_time = 7.5e6; - double e = 2.71828182845905; - std::string slogan = "DON'T PANIC"; - - // Open an output file stream in binary mode : - std::ofstream fout (filename.c_str (), std::ios_base::binary); - - { - // Create an output portable binary archive attached to the output file : - boost::archive::portable_binary_oarchive opba (fout); - - // Store (serializing) variables : - opba & b & c & answer & computing_time & e & slogan; - } - - return 0; -} - -// end of tutorial_pba_0.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_1.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_1.cpp deleted file mode 100644 index 1b5fb8fc..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_1.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/** tutorial_pba_1.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization package. - * - * This quick start example shows how to load some variables - * of basic types (bool, integer, floating point numbers, STL string) - * using the portable binary archive format associated to a - * standard input file stream. - * - */ - -#include -#include -#include - -#include -#include - -int main (void) -{ - using namespace std; - - // The name for the example data file : - string filename = "pba_0.data"; - - // Some variables of various types : - bool b; - char c; - uint32_t answer; - float computing_time; - double e; - string slogan; - - // Open an input file stream in binary mode : - ifstream fin (filename.c_str (), ios_base::binary); - - { - // Create an input portable binary archive attached to the input file : - boost::archive::portable_binary_iarchive ipba (fin); - - // Loading (de-serializing) variables using the same - // order than for serialization (see tutorial_pba_0.cpp) : - ipba & b & c & answer & computing_time & e & slogan; - } - - cout.precision (15); - cout << "Variable 'b' is : " << b << " " << "(bool)" << endl; - cout << "Variable 'c' is : '" << c << "' " << " " << "(char)" << endl; - cout << "Variable 'answer' is : " << answer << " " << "(unsigned 32-bit integer)" << endl; - cout << "Variable 'computing_time' is : " << computing_time << " " << "(single precision 32-bit float)" << endl; - cout << "Variable 'e' is : " << e << " " << "(double precision 64-bit float)" << endl; - cout << "Variable 'slogan' is : \"" << slogan << "\" " << "(std::string)" << endl; - - return 0; -} - -// end of tutorial_pba_1.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_10.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_10.cpp deleted file mode 100644 index a2d44391..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_10.cpp +++ /dev/null @@ -1,162 +0,0 @@ -/** tutorial_pba_10.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This example shows how use PBAs combined with on-the-fly - * compressed I/O streams. - * - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -class data_type -{ -private: - friend class boost::serialization::access; - template - void serialize (Archive & ar, const unsigned int version); -public: - void print (ostream & out, const string & title) const; -public: - vector values; - data_type (); -}; - -data_type::data_type () : values () -{ - return; -} - -void data_type::print (ostream & out, const string & title) const -{ - out << endl; - out << title << " :" << endl; - for (int i = 0; i < this->values.size (); ++i) - { - out.precision (16); - out.width (18); - out << this->values [i] << ' ' ; - if ((i%4) == 3) clog << endl; - } - out << endl; - return; -} - -template -void data_type::serialize (Archive & ar, const unsigned int version) -{ - ar & values; - return; -} - -void do_gzipped_out (void) -{ - // The name for the output data file : - string filename = "pba_10.data.gz"; - - // A data structure to be stored : - data_type my_data; - - // Fill the vector with arbitrary (possibly non-finite) values : - size_t dim = 1000; - my_data.values.reserve (dim); - for (int i = 0; i < dim; ++i) - { - double val = (i + 1) * (1.0 + 3 * numeric_limits::epsilon ()); - if (i == 4) val = numeric_limits::quiet_NaN (); - if (i == 23) val = numeric_limits::infinity (); - if (i == 73) val = -numeric_limits::infinity (); - if (i == 90) val = 0.0; - my_data.values.push_back (val); - } - - // Print: - my_data.print (clog, "Stored data"); - - // Create an output filtering stream : - boost::iostreams::filtering_ostream zout; - zout.push (boost::iostreams::gzip_compressor ()); - - // Open an output file stream in binary mode : - ofstream fout (filename.c_str (), ios_base::binary); - zout.push (fout); - - // Save to PBA : - { - // Create an output portable binary archive attached to the output file : - boost::archive::portable_binary_oarchive opba (zout); - - // Store (serializing) the data : - opba & my_data; - } - - // Clean termination of the streams : - zout.flush (); - zout.reset (); - - return; -} - -void do_gzipped_in (void) -{ - // The name for the input data file : - string filename = "pba_10.data.gz"; - - // A data structure to be loaded : - data_type my_data; - - // Create an input filtering stream : - boost::iostreams::filtering_istream zin; - zin.push (boost::iostreams::gzip_decompressor ()); - - // Open an input file stream in binary mode : - ifstream fin (filename.c_str (), ios_base::binary); - zin.push (fin); - - // Load from PBA : - { - // Create an input portable binary archive attached to the input file : - boost::archive::portable_binary_iarchive ipba (zin); - - // Load (deserializing) the data : - ipba & my_data; - } - - // Print: - my_data.print (clog, "Loaded data"); - - return; -} - -int main (void) -{ - do_gzipped_out (); - do_gzipped_in (); - return 0; -} - -// end of tutorial_pba_10.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_10b.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_10b.cpp deleted file mode 100644 index f1695f4d..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_10b.cpp +++ /dev/null @@ -1,198 +0,0 @@ -/** tutorial_pba_10b.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This example shows how use PBAs combined with on-the-fly - * compressed I/O streams. - * - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -class data_type -{ -private: - friend class boost::serialization::access; - template - void serialize (Archive & ar, const unsigned int version); -public: - void print (ostream & out, const string & title) const; -public: - vector values; - data_type (); -}; - -//BOOST_CLASS_VERSION(data_type, 7) - -data_type::data_type () : values () -{ - return; -} - -void data_type::print (ostream & out, const string & title) const -{ - out << endl; - out << title << " :" << endl; - for (int i = 0; i < this->values.size (); ++i) - { - out.precision (16); - out.width (18); - out << this->values [i] << ' ' ; - if ((i%4) == 3) clog << endl; - } - out << endl; - return; -} - -template -void data_type::serialize (Archive & ar, const unsigned int version) -{ - ar & BOOST_SERIALIZATION_NVP (values); - return; -} - -class data_type2 -{ -private: - friend class boost::serialization::access; - template - void serialize (Archive & ar, const unsigned int version); -public: - double value; - data_type2 (); -}; - -BOOST_CLASS_VERSION(data_type2, 99) - -data_type2::data_type2 () : value (666.666) -{ - return; -} - -template -void data_type2::serialize (Archive & ar, const unsigned int version) -{ - ar & BOOST_SERIALIZATION_NVP (value); - return; -} - -class data_type3 -{ -private: - friend class boost::serialization::access; - template - void serialize (Archive & ar, const unsigned int version); -public: - vector values; - data_type3 (); -}; - -BOOST_CLASS_VERSION(data_type3, 33) - -data_type3::data_type3 () -{ - { - data_type2 d; - d.value = 6.66; - values.push_back (d); - } - { - data_type2 d; - d.value = 66.66; - values.push_back (d); - } - { - data_type2 d; - d.value = 666.66; - values.push_back (d); - } - return; -} - -template -void data_type3::serialize (Archive & ar, const unsigned int version) -{ - ar & BOOST_SERIALIZATION_NVP (values); - return; -} - -/********************/ - -void do_xml_out (void) -{ - // The name for the output data file : - string filename = "pba_10.xml"; - - // A data structure to be stored : - data_type my_data; - - // Fill the vector with arbitrary (possibly non-finite) values : - size_t dim = 6; - my_data.values.reserve (dim); - for (int i = 0; i < dim; ++i) - { - double val = (i + 1) * (1.0 + 3 * numeric_limits::epsilon ()); - if (i == 4) val = numeric_limits::quiet_NaN (); - if (i == 7) val = numeric_limits::infinity (); - if (i == 9) val = -numeric_limits::infinity (); - if (i == 13) val = 0.0; - my_data.values.push_back (val); - } - - // Print: - my_data.print (clog, "Stored data"); - - data_type2 my_data2; - data_type3 my_data3; - - // Open an output file stream in binary mode : - ofstream fout (filename.c_str ()); - - // Save to PBA : - { - // Create an output XML archive attached to the output file : - boost::archive::xml_oarchive oxa (fout); - - // Store (serializing) the data : - oxa & BOOST_SERIALIZATION_NVP(my_data); - oxa & BOOST_SERIALIZATION_NVP(my_data2); - oxa & BOOST_SERIALIZATION_NVP(my_data3); - } - - return; -} - -int main (void) -{ - do_xml_out (); - return 0; -} - -// end of tutorial_pba_10b.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_11.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_11.cpp deleted file mode 100644 index 3e606480..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_11.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/** tutorial_pba_11.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This example program compares the times needed to serialize - * and deserialize some large amount of data using PBA and - * text archives. - * - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -class data_type -{ -private: - friend class boost::serialization::access; - template - void serialize (Archive & ar, const unsigned int version); -public: - void print (ostream & out, const string & title) const; -public: - vector values; - data_type (); -}; - -data_type::data_type () : values () -{ - return; -} - -void data_type::print (ostream & out, const string & title) const -{ - out << endl; - out << title << " :" << endl; - bool skip = false; - for (int i = 0; i < this->values.size (); ++i) - { - if ((i >= 12) && (i < (int) this->values.size () - 8)) - { - if (! skip) out << " ..." << endl; - skip = true; - continue; - } - out.precision (16); - out.width (18); - out << this->values [i] << ' ' ; - if ((i%4) == 3) clog << endl; - } - out << endl; - return; -} - -template -void data_type::serialize (Archive & ar, const unsigned int version) -{ - ar & values; - return; -} - -double do_pba_out (const data_type & a_data) -{ - string filename = "pba_11.data"; - ofstream fout (filename.c_str (), ios_base::binary); - boost::timer io_timer; - { - boost::archive::portable_binary_oarchive opba (fout); - opba & a_data; - } - return io_timer.elapsed (); -} - -double do_pba_in (data_type & a_data) -{ - string filename = "pba_11.data"; - ifstream fin (filename.c_str (), ios_base::binary); - boost::timer io_timer; - { - boost::archive::portable_binary_iarchive ipba (fin); - ipba & a_data; - } - return io_timer.elapsed (); -} - -double do_text_out (const data_type & a_data) -{ - string filename = "pba_11.txt"; - ofstream fout (filename.c_str ()); - boost::timer io_timer; - { - boost::archive::text_oarchive ota (fout); - ota & a_data; - } - return io_timer.elapsed (); -} - -double do_text_in (data_type & a_data) -{ - string filename = "pba_11.txt"; - ifstream fin (filename.c_str ()); - boost::timer io_timer; - { - boost::archive::text_iarchive ita (fin); - ita & a_data; - } - return io_timer.elapsed (); -} - -int main (void) -{ - double elapsed_time_pba_out; - double elapsed_time_text_out; - double elapsed_time_pba_in; - double elapsed_time_text_in; - data_type my_data; // A data structure to be stored then loaded. - - { - // Fill the vector with random values : - size_t dim = 10000000; - my_data.values.reserve (dim); - boost::random::mt19937 rng; - boost::random::uniform_real_distribution<> flat (0.0, 100.0); - for (int i = 0; i < dim; ++i) - { - double val = flat (rng); - my_data.values.push_back (val); - } - my_data.print (clog, "Stored data in PBA and text archive"); - } - - { - // Store in PBA : - elapsed_time_pba_out = do_pba_out (my_data); - } - - { - // Store in text archive : - elapsed_time_text_out = do_text_out (my_data); - } - - { - my_data.values.clear (); - // Load from PBA : - elapsed_time_pba_in = do_pba_in (my_data); - my_data.print (clog, "Loaded data from PBA"); - } - - { - my_data.values.clear (); - // Load from text archive : - elapsed_time_text_in = do_text_in (my_data); - my_data.print (clog, "Loaded data from text archive"); - } - - clog << "PBA store I/O elapsed time : " << elapsed_time_pba_out << " (second)" << endl; - clog << "Text store I/O elapsed time : " << elapsed_time_text_out << " (second)" << endl; - clog << "PBA load I/O elapsed time : " << elapsed_time_pba_in << " (second)" << endl; - clog << "Text load I/O elapsed time : " << elapsed_time_text_in << " (second)" << endl; - - return 0; -} - -// end of tutorial_pba_11.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_2.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_2.cpp deleted file mode 100644 index 7f6c875b..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_2.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/** tutorial_pba_2.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This sample program shows how to use a portable binary archive - * to store/load floating point numbers including non-finite and - * special (denormalized) values. - * - */ - -#include -#include -#include - -#include -#include - -int main (void) -{ - using namespace std; - - // The name for the example data file : - string filename = "pba_2.data"; - - { - // A normal single precision floating point number : - float pi = 3.14159265; - - // Single precision zeroed floating point number : - float zero = 0.0; - - // A denormalized single precision floating point number : - float tiny = 1.e-40; - - // A single precision floating point number with `+Infinity' value : - float plus_infinity = numeric_limits::infinity (); - - // A single precision floating point number with `-Infinity' value : - float minus_infinity = -numeric_limits::infinity (); - - // A single precision `Not-a-Number' (NaN): - float nan = numeric_limits::quiet_NaN (); - - // Open an output file stream in binary mode : - ofstream fout (filename.c_str (), ios_base::binary); - - { - // Create an output portable binary archive attached to the output file : - boost::archive::portable_binary_oarchive opba (fout); - - // Store (serialize) variables : - opba & pi & zero & tiny & plus_infinity & minus_infinity & nan; - } - } - - { - // Single precision floating point numbers to be loaded : - float x[6]; - - // Open an input file stream in binary mode : - ifstream fin (filename.c_str (), ios_base::binary); - - { - // Create an input portable binary archive attached to the input file : - boost::archive::portable_binary_iarchive ipba (fin); - - // Load (de-serialize) variables using the same - // order than for serialization : - for (int i = 0; i < 6; ++i) - { - ipba & x[i]; - } - } - - // Print : - for (int i = 0; i < 6; ++i) - { - cout.precision (8); - cout << "Loaded x[" << i << "] = " << x[i]; - switch (fp::fpclassify(x[i])) - { - case FP_NAN: cout << " (NaN)"; break; - case FP_INFINITE: cout << " (infinite)"; break; - case FP_SUBNORMAL: cout << " (denormalized)"; break; - case FP_NORMAL: cout << " (normalized)"; break; - } - cout << endl; - } - } - - return 0; -} - -// end of tutorial_pba_2.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_3.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_3.cpp deleted file mode 100644 index 42df3bf9..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_3.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/** tutorial_pba_3.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This sample program shows how to use a portable binary archive - * and prevent the serialization of non-finite floating numbers. - * - */ - -#include -#include -#include - -#include - -int main (void) -{ - using namespace std; - - // The name for the example data file : - string filename = "pba_3.data"; - - try - { - // An array of single precision floating numbers: - float x[5]; - x[0] = 3.14159; // Pi - x[1] = 6.022e22; // Avogadro constant - x[2] = 1.6e-19; // Electron charge magnitude - x[3] = 1.e-40; // A tiny (denormalized) value - x[4] = numeric_limits::infinity (); // This will fail while serializing... - - // Open an output file stream in binary mode : - ofstream fout (filename.c_str (), ios_base::binary); - - { - // Create an output portable binary archive attached to the output file, - // using the special 'boost::archive::no_infnan' flag : - boost::archive::portable_binary_oarchive opba (fout, boost::archive::no_infnan); - - // Store (serialize) variables : - for (int i = 0; i < 5; ++i) - { - clog << "Serializing value : " << x[i] << " ... "; - opba & x[i]; - clog << "Ok !" << endl; - } - } - } - catch (exception & x) - { - cerr << "ERROR: " << x.what () << endl; - return 1; - } - - return 0; -} - -// end of tutorial_pba_3.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_4.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_4.cpp deleted file mode 100644 index 61d05f6f..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_4.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/** tutorial_pba_4.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This sample program shows how to use a portable binary archive - * to store/load integer numbers of various sizes using the Boost - * portable integer typedefs. - * - */ - -#include -#include - -#include -#include -#include - -int main (void) -{ - using namespace std; - - // The name for the example data file : - string filename = "pba_4.data"; - - { - // Some integer numbers : - bool t = true; - char c = 'c'; - unsigned char u = 'u'; - int8_t b = -3; // char - uint8_t B = +6; // unsigned char - int16_t s = -16; - uint16_t S = +32; - int32_t l = -128; - uint32_t L = +127; - int64_t ll = -1024; - uint64_t LL = +2048; - - // Open an output file stream in binary mode : - ofstream fout (filename.c_str (), ios_base::binary); - - { - // Create an output portable binary archive attached to the output file : - boost::archive::portable_binary_oarchive opba (fout); - - // Store (serialize) variables : - opba & t & c & u & b & B & s & S & l & L & ll & LL; - } - } - - { - // Single precision floating numbers to be loaded : - // Some integer numbers : - bool t; - char c; - unsigned char u; - int8_t b; - uint8_t B; - int16_t s; - uint16_t S; - int32_t l; - uint32_t L; - int64_t ll; - uint64_t LL; - - // Open an input file stream in binary mode : - ifstream fin (filename.c_str (), ios_base::binary); - - { - // Create an input portable binary archive attached to the input file : - boost::archive::portable_binary_iarchive ipba (fin); - - // Load (de-serialize) variables using the same - // order than for serialization : - ipba & t & c & u & b & B & s & S & l & L & ll & LL; - } - - clog << "t = " << t << " (bool)" << endl; - clog << "c = '" << c << "' (char)" << endl; - clog << "u = '" << u << "' (unsigned char)" << endl; - clog << "b = " << (int) b << " (int8_t)" << endl; - clog << "B = " << (int) B << " (uint8_t)" << endl; - clog << "s = " << s << " (int16_t)" << endl; - clog << "S = " << S << " (uint16_t)" << endl; - clog << "l = " << l << " (int32_t)" << endl; - clog << "L = " << L << " (uint32_t)" << endl; - clog << "ll = " << ll << " (int64_t)" << endl; - clog << "LL = " << LL << " (uint64_t)" << endl; - } - - return 0; -} - -// end of tutorial_pba_4.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_5.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_5.cpp deleted file mode 100644 index 18f61392..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_5.cpp +++ /dev/null @@ -1,111 +0,0 @@ -/** tutorial_pba_5.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This sample program shows how to use a portable binary archive - * to store/load data in a memory buffer. - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include - -int main (void) -{ - using namespace std; - - // The memory buffer is implemented using a STL vector : - typedef std::vector buffer_type; - buffer_type buffer; - - { - // Some data to be stored : - bool t = true; - char c = 'c'; - int16_t s = +16; - int32_t l = -128; - int64_t ll = +10000000000; - float pi = 3.14159; - double nan = numeric_limits::quiet_NaN (); - string hello = "World !"; - - buffer.reserve (1024); // pre-allocate some memory - - // The output stream interface to the buffer : - boost::iostreams::stream > output_stream (buffer); - - { - // Create an output portable binary archive attached to the output file : - boost::archive::portable_binary_oarchive opba (output_stream); - - // Store (serialize) variables : - opba & t & c & s & l & ll & pi & nan & hello; - } - - } - - clog << "Buffer content is " << buffer.size () << " bytes : " << endl << " "; - for (int i = 0; i < buffer.size (); ++i) - { - clog << (int) ((unsigned char) buffer[i]) << ' '; - if ((i + 1) % 20 == 0) clog << endl << " "; - } - clog << endl; - - { - // Some data to be loaded : - bool t; - char c; - int16_t s; - int32_t l; - int64_t ll; - float pi; - double nan; - string hello; - - // The input stream interface to the buffer : - boost::iostreams::stream input_stream (&buffer[0], - buffer.size ()); - - { - // Create an input portable binary archive attached to the input file : - boost::archive::portable_binary_iarchive ipba (input_stream); - - // Load (de-serialize) variables : - ipba & t & c & s & l & ll & pi & nan & hello; - } - - clog << "Loaded values from the buffer are: " << endl; - clog << " t = " << t << " (bool)" << endl; - clog << " c = '" << c << "' (char)" << endl; - clog << " s = " << s << " (int16_t)" << endl; - clog << " l = " << l << " (int32_t)" << endl; - clog << " ll = " << ll << " (int64_t)" << endl; - clog << " pi = " << pi << " (float)" << endl; - clog << " nan = " << nan << " (double)" << endl; - clog << " hello = \"" << hello << "\" (std::string)" << endl; - } - - return 0; -} - -// end of tutorial_pba_5.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_6.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_6.cpp deleted file mode 100644 index f32659a4..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_6.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/** tutorial_pba_6.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This sample program shows how to use a portable binary archive - * associated to a memory buffer to copy a non-copyable object. - * - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -using namespace std; - -/* A foo noncopyable class */ -struct foo : boost::noncopyable -{ - uint32_t status; - double value; - double special; - - string to_string () const - { - ostringstream sout; - sout << "foo={status=" << status << "; value=" << value << "; special=" << special<< "}"; - return sout.str(); - } - - template - void serialize (Archive & ar, const unsigned int version) - { - ar & status; - ar & value; - ar & special; - return; - } - -}; - -// A templatized copy function for Boost/Serialization equipped classes. -// Here we use PBAs associated to a memory buffer : -template -void copy (const Serializable & source, Serializable & target) -{ - namespace io = boost::iostreams; - namespace ba = boost::archive; - if (&source == &target) return; // self-copy guard - typedef std::vector buffer_type; - buffer_type buffer; - buffer.reserve (1024); - { - io::stream > output_stream (buffer); - ba::portable_binary_oarchive opba (output_stream); - opba & source; - } - { - io::stream input_stream (&buffer[0], buffer.size ()); - ba::portable_binary_iarchive ipba (input_stream); - ipba & target; - } - return; -} - -int main (void) -{ - // Some instance of the 'foo' class : - foo dummy; - dummy.status = 1; - dummy.value = 3.14159; - dummy.special = numeric_limits::quiet_NaN (); - clog << "dummy is : " << dummy.to_string () << endl; - - // Another instance of the 'foo' class : - foo clone; - - /* The following instruction is forbidden because foo - inherits 'boost::noncopyable' : - - clone = dummy; // this ends in a compilation error. - - */ - - // Anyway, we can use this workaround : - copy (dummy, clone); - clog << "clone is : " << clone.to_string () << endl; - - return 0; -} - -// end of tutorial_pba_6.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_7.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_7.cpp deleted file mode 100644 index 30722b18..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_7.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/** tutorial_pba_7.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This example shows how the default behaviour of standard - * I/O streams does not support the read/write operations of - * non-finite floating point values in a portable way. - * - */ - -#include -#include -#include -#include - -using namespace std; - -int main (void) -{ - { - float x = numeric_limits::infinity (); - double y = numeric_limits::quiet_NaN (); - cout.precision (8); - cout << "x = " << x << endl; - cout.precision (16); - cout << "y = " << y << endl; - } - - { - string input ("inf nan"); - istringstream iss (input); - float x; - double y; - iss >> x >> y; - if (! iss) - { - cerr << "Cannot read 'x' or 'y' : non finite values are not supported !" << endl; - } - } - return 0; -} - -// end of tutorial_pba_7.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_8.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_8.cpp deleted file mode 100644 index a41e46fb..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_8.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/** tutorial_pba_8.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This example shows how to store some variables - * of basic types (bool, integer, floating point numbers, STL string) - * using the text or XML archive format associated to a - * standard output file stream supporting portable non-finite - * floating point values. - * - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -using namespace std; - -void do_text_out (void) -{ - // The name for the example data text file : - string filename = "pba_8.txt"; - - // Some variables of various primitive types : - bool b = true; - char c = 'B'; - uint32_t answer = 42; - float value = numeric_limits::infinity (); - double precision = numeric_limits::quiet_NaN (); - string question = "What makes you think she's a witch?"; - - // Open an output file stream : - ofstream fout (filename.c_str ()); - - // Prepare the output file stream for inf/NaN support : - locale default_locale (locale::classic (), - new boost::archive::codecvt_null); - locale infnan_locale (default_locale, - new boost::math::nonfinite_num_put); - fout.imbue (infnan_locale); - - { - // Create an output text archive attached to the output file : - boost::archive::text_oarchive ota (fout, boost::archive::no_codecvt); - - // Store (serializing) variables : - ota & b & c & answer & value & precision & question; - } - - return; -} - -void do_xml_out (void) -{ - // The name for the example data XML file : - string filename = "pba_8.xml"; - - // Some variables of various primitive types : - bool b = true; - char c = 'B'; - uint32_t answer = 42; - float value = numeric_limits::infinity (); - double precision = numeric_limits::quiet_NaN (); - string question = "What makes you think she's a witch?"; - - // Open an output file stream : - ofstream fout (filename.c_str ()); - - // Prepare the output file stream for inf/NaN support : - locale default_locale (locale::classic (), - new boost::archive::codecvt_null); - locale infnan_locale (default_locale, - new boost::math::nonfinite_num_put); - fout.imbue (infnan_locale); - - { - // Create an output text archive attached to the output file : - boost::archive::xml_oarchive oxa (fout, boost::archive::no_codecvt); - - // Store (serializing) variables : - oxa & BOOST_SERIALIZATION_NVP(b) - & BOOST_SERIALIZATION_NVP(c) - & BOOST_SERIALIZATION_NVP(answer) - & BOOST_SERIALIZATION_NVP(value) - & BOOST_SERIALIZATION_NVP(precision) - & BOOST_SERIALIZATION_NVP(question); - } - - return; -} - -int main (void) -{ - do_text_out (); - do_xml_out (); - return 0; -} - -// end of tutorial_pba_8.cpp diff --git a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_9.cpp b/contrib/eos_portable_archive/tutorial/code/tutorial_pba_9.cpp deleted file mode 100644 index ff564be1..00000000 --- a/contrib/eos_portable_archive/tutorial/code/tutorial_pba_9.cpp +++ /dev/null @@ -1,134 +0,0 @@ -/** tutorial_pba_9.cpp - * - * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer - * - * Use, modification and distribution is subject to the Boost Software - * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - * - */ - -/** - * The intent of this program is to serve as a tutorial for - * users of the portable binary archive in the framework of - * the Boost/Serialization library. - * - * This example shows how to load some variables of basic - * types (bool, char, integer, floating point numbers, STL string) - * using the text or XML archive format associated to a - * standard file input stream supporting portable non-finite - * floating point values. - * - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -void do_text_in (void) -{ - // The name for the example data text file : - string filename = "pba_8.txt"; - // Some variables of various primitive types : - bool b; - char c; - uint32_t answer; - float value; - double precision; - string question; - - // Open an input file stream : - ifstream fin (filename.c_str ()); - - // Prepare the input file stream for inf/NaN support : - locale default_locale (locale::classic (), - new boost::archive::codecvt_null); - locale infnan_locale (default_locale, - new boost::math::nonfinite_num_get); - fin.imbue (infnan_locale); - - { - // Create an input text archive attached to the input file : - boost::archive::text_iarchive ita (fin, boost::archive::no_codecvt); - - // Store (serializing) variables : - ita & b & c & answer & value & precision & question; - } - - clog << "Loaded values from text archive are: " << endl; - clog << " b = " << b << endl; - clog << " c = '" << c << "'" << endl; - clog << " answer = " << answer << endl; - clog << " value = " << value << endl; - clog << " precision = " << precision << endl; - clog << " question = \"" << question << "\"" << endl; - - return; -} - -void do_xml_in (void) -{ - // The name for the example data text file : - string filename = "pba_8.xml"; - - // Some variables of various primitive types : - bool b; - char c; - uint32_t answer; - float value; - double precision; - string question; - - // Open an input file stream : - ifstream fin (filename.c_str ()); - - // Prepare the input file stream for inf/NaN support : - locale default_locale (locale::classic (), - new boost::archive::codecvt_null); - locale infnan_locale (default_locale, - new boost::math::nonfinite_num_get); - fin.imbue (infnan_locale); - - { - // Create an output text archive attached to the output file : - boost::archive::xml_iarchive ixa (fin, boost::archive::no_codecvt); - - // Store (serializing) variables : - ixa & BOOST_SERIALIZATION_NVP(b) - & BOOST_SERIALIZATION_NVP(c) - & BOOST_SERIALIZATION_NVP(answer) - & BOOST_SERIALIZATION_NVP(value) - & BOOST_SERIALIZATION_NVP(precision) - & BOOST_SERIALIZATION_NVP(question); - } - - clog << "Loaded values from XML archive are: " << endl; - clog << " b = " << b << endl; - clog << " c = '" << c << "'" << endl; - clog << " answer = " << answer << endl; - clog << " value = " << value << endl; - clog << " precision = " << precision << endl; - clog << " question = \"" << question << "\"" << endl; - - return; -} - -int main (void) -{ - do_text_in (); - do_xml_in (); - return 0; -} - -// end of tutorial_pba_9.cpp diff --git a/contrib/eos_portable_archive/tutorial/images/boost.png b/contrib/eos_portable_archive/tutorial/images/boost.png deleted file mode 100644 index b4d51fcd5c9149fd77f5ca6ed2b6b1b70e8fe24f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6308 zcmV;V7+dFwP)FBa0e37VcyhIDVmU z@{ls-C8mlZam5cr4-}LM9~I+5QpH5B!st{k3!T70hd^+W0NtImr#tP}?9T4a!E3dj1z_;Mr#v4xEk87nE(}rB1UTm zKa|L!Z^8i&PqZ)yF-hzFM_NzLlL93gC7ET zJdxAJ{EsJc$dZZ?qg9C?H%=Uhp!gvq7>XFJZv2qJLp}|JA4)he6fs&g_@M!h`?nMP zFcdLboA{vt4?(r_Roi-xqGx{o#=A#n-1*mS9)0ddf6I7fwa{w^kL-K>P;dG%5-xZs zQRJxBWOpk;5K!*IW}S{3Ju`jj)R{TY{OdN4Fs58ZGa@umc6HJZt5OQL&~}BD_X#6j7>kzV+ag=2Su& z*^ps;xmpNk#CEA4JC#2;tfJUh!dn3pYI(23qBlJfZQ?4fYlU?p*Kk`A1ALJEPpm~v8j|n_ll(i4PgaZ13 zG^>0oNB>dv^v7BKk`xsxS_E!aRn@A>o_1@ALEROB#w}+2tSUIKBs+tQMKi_kkd+_mRx#&-V{N+jAABQf&vNoMc;Ws2q%1) z{8=;hPnY^J(l&BDqp(fNRmFH*7d}&;)-=P+(QOaIiC{Y1G@BPrGfL<=D*F1BCZ@Sa zOsbV+f#(AG(A<7G%6oCf574a>V~i2Jxw^-Ic-e2_{?*KZz$0X1Bn=ytnM z`EQTTK;;B!PeU*KP?{+TxD9#{j(x8m z+Pi!EU+%cGy-EPX28Yf04zXsO9^m%wZ4-NTOm+g=9<&Ru7e=T~7~R42G>SJ3b?%gF z(|_%DTbpS`MX{J(=f?~61|ORUX+o&!TwbBYL1N+xJzXr<;G`BC0?Vi=JuGCj2^uww;Ps)T>wAnJg8$7-muj&@gu7T&kDXSW29Gh-<7o(O+lU5Y2oyiHd?`LK z6crqRa0!{_f8{Et)vKOTh-~)QFLgMExye)oDIxUQ!6R*Ou)C7uGp8SaWGDTA+M9O6 z1y8ubN3U%-(_Q?~EI?>!ZVGSutf$W^7Em<1y|Ge@Gm(&pp=;M0!%3mUi75b2UP0Mn z(>2%({W0YZ$F-2RC5qk!L>*bzq4I`<(AJ4DP;$tl_%r|UMp&ZHZc+koiP=eQ|4K(&ze{(0QnjmthP*Fcy0LzCrRV>Tg!5Kn#ZbqVj zWTmdvO6l}40OQj|^ij4ImtglX*wcv!x+0DuVzgeV2t4u(SwIxQqqrICP8GQ~4iWGZ z`oL@g1NfD^#HE+*!XOg5Y`43iaUoZisr}=2#5r=3LbXzjBxC$N&6eTuP+nBU7p@8XbZ%e=OWn_*B+SieC!^#3vbo)%>dVe zvKI>PI^L}=j%EV)V~50dZv5C~QCd{Wl{)BS-yD}T&4@f`M(?Z$_AxHqc8!*aqnfIz z#jKRhfC{!yL<{+b*s+Jkt%&zQ$g1Y#MRZBgLw>4!+%t@o(!1)bYU2R*^gn!;zzUuW z2et=+iOn^}k3X`@Q#FUoC%^a5-rd{D+SKLHE=VmL(}x0{5c{!P_#t?%JsGrtrCJ&O z8_n==I3Q&6HfRE`er-rUFXReZG#wX(N*%p($~!d9BV?|5{9#eAH6e0kE=s@*F2BQN zbHG?+u5t)B(CCOQj?y>)U^FFrCfpin@7o1!%CpSYo4OUx(aU9X{`jGcymZH&C4{Oh z*C2hz*CgYz>YOPuez%YFl8DNRs#Y!~lPM_i*&?dcJfMOs2qD{Vv%vWxx==QBv`9>K z8D?z{T8MhN6%42ZH;!Y-$C(5s6zLV4y$_K=>vT-@tKIg>geZJamzO{K+dsa_8*+vA z@M^KHuG^5}Zy$>pAmy9$o{fMSoQN;N5|RqRd+hcChHNC;)w+}*cNfb~q{)FyN@=uY&C4Llo&3rg=+zuLu_ zNufs`p`I_BYtw(=sl2&kpQET&kShbp*w)Qa&SRl=qKLhEC_u4H*=RiuJO>^MT%b2+QUuW8)9tL+pT`g0d8FAtfL8rQK9g z!U^Q3o_!f!uPZ+>BOR>}j zkVOl5baFv6gXP?zaR}idWXdEC{a+T1B+-D#$&o}vE75Pe6H2%{O<{LLrk!?rxw*+i zX%_dZ-K`U21b-b30!@2n-|L6=?40y zE|=ZEGWJ6U0fe4^aesSV(Cy`q!RAlN%3EF$lc{(^vGNF7 zTcOCQD#|U+=ZfV_O1k-K?o<}LMyMG9Egsfv4my!~nm+SvEwP=_h=4v^U_(mF;IsBEG z$OC^IOU1>D#Z*3@Uq3i-?GS(bOkGi(YlJwirL7SRp?fx>q^M8DN@|%CXpJA7DiX+J zV=n&jwgGH&9fe}KL-zwE$2bH_zuL8OUU=p)kRAu#IoA0iD=?@$WlmqSvv8^U!RdmvCy1?p)|xqO#PX;I-&*U1J4VY<;v?G@1O z2U1C1Rq`c$4bQEc1k(ZFFofI|k!1*7okkntdIUGBR_w^>#Q33KR`vXg`*DNco*k3a zO3~>+FaNcJM{tb>a;f#JT{7(m?o!7&4?15S7Z^V@n-8wsybAH61l)S|Vr6bgX%+B- z(8~JN_dcsE7F0n%d}0GCWx;w`ALq7S!<*5iZeIfO+IaEi4SIfAQ6)|xAjf0bT$aiY zOXers6bIBm&j+|rWt`55lXyC4gRJ82SG(a7VG;s2huc&6LypK?uJDq`7c2F*ewW2* zca{f0vqDIlTHJeQbjT$WVx5Z?3MJj*+`t=GS3n41HeS5>O0+JfyA5KtIa5!~%?%_1 z>(J<3xQ{g9gb&%-weO8Xl)UoH&)*;j0XJy-)o$BFn6UD+Ckc}f-B?-Nss3h%Q&rVO z!)N``xl+xpq}eb-HiXVxsGeJ@^Bj`WgArw^Tvd}Ix_?5jAk?bK%MiMD&=fv3kwdLkU391`!WGMpvSxUi9Br&brLGHxB=QFu#0CTTff?ETbR6xi6o@ACoOvQ z;8eIPZQJ+1c`jJUD`FzfW%ISE6U(TvG_4tZW1cqSYbtVBxEBQ4e0F*7L)+cjBh5xRfLKopq+ji}BYE@CarWa40D>dhK8Vj%aW)!D23*?v)(l7D6dDr9(@td!T)M}cNJC72> z2`RFW*QzSAd7>M8S#hYfg~-(d`fo+9Bn5@%dE5qz2=gh`KmWU*bmN6S{e$n|=IqES zpIrb3T|N!`@#Oa&YERV+yWKkB9bvq4(<{&YV~AL4fBm+i ziE*Agw^TXxWzm}E)SA&dj%CuVUxS4kSDV6R>SK9yU2=Vrmt*UBC8tFPN75X!!>1X$ zbBW?Ep`R8@@R-8N!4&4AnH#h`>pK76?GKWS?)3+a)jaZu`aJ zm_CqF2bF{?h^W?Ydy*jN_s%aVEqkUZuyBTuty-rcR8sUgpJ-HwakWBG&X;tLcD@Kl zwE#z{T9oyppdU3jisf3VToaAeKAtiM5qI8M$X*J!gH5%KY(_&F7_GzYKj2_r^P`vT zc4J^7k4k~O7ojg_AmN8Ri&CrVuCp7jPDBNv(fX}*vaifLXy287bOh`dfHWg>DWQqG7KNl+W=pqKuJ6Fw-6t&beN0%{vgFNPQr zX098h+!>>1?S{U*)OS=ey`7YQTIk`V0rWS=y3X}+v|OnrVv(J9Z20AodAmM_TpLG` zeF^uB>pFL-Tn7!#cT4y{3B2N4(4yKTfY z5Zsu1ci%@NF-%B<-Q}yJhQ5~wmH)uuNU*-Wvs_X|DGvPk-@iM5_PpE*5^Zv^g@Q(7 zhEt*8JTRmNBqlb-?;VdrBb8EK;%lBCR-nfT<;NEue&VN{cs8#yK0|Q*=Fv{(J)pC} zEP4068Omu2;1Xad-)&9%+ir&+M6llNJ=-RNhEAK_LfkHJzTTbQ;l~c4tSlrY@qAAC z#XEB?ZyN?ep1!!hnam8t`9eua^QAU~58)Rlg6LPL_J)9u^-zH9G7R1h!3dm`PR2)u z^{J20qlx>~?ph*${2=1je&i~u@zDs+zcaIR{L@^6HLVZ*G?*0&p>N;5Zqs^Esj5OX zCkk93;e*2oLGoON;9<0?i4W>La5%W)BC1r3(9bOcR;a;>4c`!Q$T7=*ke2vOgCeAA zN=1jDK^A_jv^zogmKFmKqkiHC$8c`fuyvXiRi&m5#DxdHG17vN4L1n6M4XX--`CS{ zp?&M=k7v(W>9`an z)MmEfxRm#Ob@pfQebrEy{G3+1W_oBaJ(Nk)l=Lm!9z{I4>D2o}Z?& zG6&-Pg;F&siFe;P=-7aOb0>}el}a_0;3+?#?e;{sjU1qLQrEgg2ruMQoB-|v5S2TjS7v*!p8|L-pyw4Xk>eH-ra06dPu<6~EmpW&6a z4nymNIz68u5s&`U_xIo+1ck}%{_4>0-=6w|b-H)=eK<(Gvj6bZv6J?khj&d*-gaZx z@qkGg9d~Ol2 z>Nyyhg$aMr|4w}X;PFMQ`R4$5;PL+X@Aq_7S%}ae$1}&SK{fX5jqtIE>Rfs1^&Q0~ zBzPQPXtcF9kYm#H%q+ZF8u09U{SYWfka+XZ??I_}c-LLt$eEdi-pVfJfAbP`;F?2^76#m|i4zf`Vl8XdZG>o~UXavtzt7S0+v&;65sL{RpC zkMF-zXX3Yyd_Z_e$1?zU;4yG7$1F_v+YcM^1b}T}8z-rte?ic}D|+I`|4AuM!^Dr{ zE4-ec;B_YmHMlw3H|VAfuQa8T0CeKvK`K4?n$clkEHtZcE<@2IDuawEsB z=6QZZ#DSmAkrULn21xk$^f?e3!1rW}7W4$r8KA?+3Ip^C9B&>v`q$sStHT~L$KdH^ z1Q)^NHIYW(He;|KZWcJ{hsH(FQQ_gCJQyLsZ8-b@6T@E$$b?%gy$kmkQ0A`N#^LRO z$bxr1ed06$KW2r5zX)Chi~{gZ%#I_BQ*L(%h$HCe-sx{{5#$LWU&s;i-Li-fw-%yH z8uTmZs(r8j#ytJuqYn}WX6U5J!ExU@p~FBg@4SbAYk_w`8#HZSg062G9)uP_A2X{m zx#xlVwvk1dSrtGC#tXO$-jXMDUwP{=2Q~04oQ5*UjLk^>_3VsmvcNyA~KMXvK);NCX zq2gJllUY|rjMfN#Ix0000Px#24YJ`L;(K){{a7>y{D4^000SaNLh0L01ejw01ejxLMWSf00007bV*G`2ipn| z1tk;20$QK|000?uMObu0Z*6U5Zgc=ca%Ew3Wn>_CX>@2HM@dakSAh-}000DQNkl?OsD>#jIvuEB*is z#6%a6Nmxuz0t8Hypn@A%t@KDca*|3asayB`>c0CzEgb;Af14`+i$z<}6?(S~4VX)cRS#{;g6{J!r7={5# z>6w6p5I8tEz`J+v@cQ*@^z`(A5E4`>l|7}@a9>~FKd0rO0MOFXf^0S$gK3&}<|~;@ zLen%*O5ytfEX&d|naodwkiVWid-l6hiku1-B|B@SA~D9G>pD`Y6w>K5C?zlq1BPKB zl}aTz=cCV`KmU7iaWV5nurLe}hGFv>mk=<eT+x~|7?x~?0#uJ@a!x%Bku z(+i&k3xWVa5HuBzQVP>Fkw}=Jlz?*{gT3!##yS5L!0Phy^5wc>sVP?!ONE~vA(_z&+~ri?Cd;D)E|*oQw6EU2eXF*% zw$cN8tDsFzp7$~L2#|Aq+J&hYTZZuRK$8kd> z&1TWs+KSH3pK<9@7c@-^DWxm%RuV$$Zsp?QBCcJ#)>Pw-K#vaxzeV_m;~9Z#M-`RqcuTn>bgc){pCm&-L2cjrzn_Pb0b1Ix0gQmP|=GBSd}!NIyoZQE{W@h`=FGTl6S^hoXO z?5OXpt*tFJH8rJbe&WrV#m*w76f{lyu$7$irj9&ct1t{>+oI8o-t5WZYb#a*MhH0< zU_lVnf=Mai`#z+U-wn$5eU!`Pn1R~1O+3%T?(QxQ4-fzM%7tMFAp|Ow3ikH)5Cp-o zr!6cj(5qLk!gXD&t*xQ0tqqoCfl>-7C8U(`-bV+d-hcJ9L6TA$Uy+FlA>wh!b={gl zl`k(ZOT#d#Z{EB~0>I|xCOGGDf|CvTEt@^MXN*;Gp7k$}QdMmGDNr*20000-1){i=j.data;i.beforeFilter&&i.beforeFilter[a.type]&&!i.beforeFilter[a.type](a)||f.push(j.selector)}else delete t[p]}i=c(a.target).closest(f,a.currentTarget); -n=0;for(l=i.length;n)[^>]*$|^#([\w-]+)$/,Pa=/^.[^:#\[\.,]*$/,Qa=/\S/, -Ra=/^(\s|\u00A0)+|(\s|\u00A0)+$/g,Sa=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,P=navigator.userAgent,xa=false,Q=[],M,ca=Object.prototype.toString,da=Object.prototype.hasOwnProperty,ea=Array.prototype.push,R=Array.prototype.slice,V=Array.prototype.indexOf;c.fn=c.prototype={init:function(a,b){var d,f;if(!a)return this;if(a.nodeType){this.context=this[0]=a;this.length=1;return this}if(typeof a==="string")if((d=Oa.exec(a))&&(d[1]||!b))if(d[1]){f=b?b.ownerDocument||b:s;if(a=Sa.exec(a))if(c.isPlainObject(b)){a=[s.createElement(a[1])]; -c.fn.attr.call(a,b,true)}else a=[f.createElement(a[1])];else{a=ua([d[1]],[f]);a=(a.cacheable?a.fragment.cloneNode(true):a.fragment).childNodes}}else{if(b=s.getElementById(d[2])){if(b.id!==d[2])return U.find(a);this.length=1;this[0]=b}this.context=s;this.selector=a;return this}else if(!b&&/^\w+$/.test(a)){this.selector=a;this.context=s;a=s.getElementsByTagName(a)}else return!b||b.jquery?(b||U).find(a):c(b).find(a);else if(c.isFunction(a))return U.ready(a);if(a.selector!==w){this.selector=a.selector; -this.context=a.context}return c.isArray(a)?this.setArray(a):c.makeArray(a,this)},selector:"",jquery:"1.4",length:0,size:function(){return this.length},toArray:function(){return R.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this.slice(a)[0]:this[a]},pushStack:function(a,b,d){a=c(a||null);a.prevObject=this;a.context=this.context;if(b==="find")a.selector=this.selector+(this.selector?" ":"")+d;else if(b)a.selector=this.selector+"."+b+"("+d+")";return a},setArray:function(a){this.length= -0;ea.apply(this,a);return this},each:function(a,b){return c.each(this,a,b)},ready:function(a){c.bindReady();if(c.isReady)a.call(s,c);else Q&&Q.push(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(R.apply(this,arguments),"slice",R.call(arguments).join(","))},map:function(a){return this.pushStack(c.map(this,function(b,d){return a.call(b,d,b)}))},end:function(){return this.prevObject|| -c(null)},push:ea,sort:[].sort,splice:[].splice};c.fn.init.prototype=c.fn;c.extend=c.fn.extend=function(){var a=arguments[0]||{},b=1,d=arguments.length,f=false,e,i,j,o;if(typeof a==="boolean"){f=a;a=arguments[1]||{};b=2}if(typeof a!=="object"&&!c.isFunction(a))a={};if(d===b){a=this;--b}for(;b
a";var e=d.getElementsByTagName("*"),i=d.getElementsByTagName("a")[0];if(!(!e||!e.length||!i)){c.support={leadingWhitespace:d.firstChild.nodeType===3,tbody:!d.getElementsByTagName("tbody").length, -htmlSerialize:!!d.getElementsByTagName("link").length,style:/red/.test(i.getAttribute("style")),hrefNormalized:i.getAttribute("href")==="/a",opacity:/^0.55$/.test(i.style.opacity),cssFloat:!!i.style.cssFloat,checkOn:d.getElementsByTagName("input")[0].value==="on",optSelected:s.createElement("select").appendChild(s.createElement("option")).selected,scriptEval:false,noCloneEvent:true,boxModel:null};b.type="text/javascript";try{b.appendChild(s.createTextNode("window."+f+"=1;"))}catch(j){}a.insertBefore(b, -a.firstChild);if(A[f]){c.support.scriptEval=true;delete A[f]}a.removeChild(b);if(d.attachEvent&&d.fireEvent){d.attachEvent("onclick",function o(){c.support.noCloneEvent=false;d.detachEvent("onclick",o)});d.cloneNode(true).fireEvent("onclick")}c(function(){var o=s.createElement("div");o.style.width=o.style.paddingLeft="1px";s.body.appendChild(o);c.boxModel=c.support.boxModel=o.offsetWidth===2;s.body.removeChild(o).style.display="none"});a=function(o){var p=s.createElement("div");o="on"+o;var n=o in -p;if(!n){p.setAttribute(o,"return;");n=typeof p[o]==="function"}return n};c.support.submitBubbles=a("submit");c.support.changeBubbles=a("change");a=b=d=e=i=null}})();c.props={"for":"htmlFor","class":"className",readonly:"readOnly",maxlength:"maxLength",cellspacing:"cellSpacing",rowspan:"rowSpan",colspan:"colSpan",tabindex:"tabIndex",usemap:"useMap",frameborder:"frameBorder"};var H="jQuery"+K(),Ta=0,ya={},Ua={};c.extend({cache:{},expando:H,noData:{embed:true,object:true,applet:true},data:function(a, -b,d){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==A?ya:a;var f=a[H],e=c.cache;if(!b&&!f)return null;f||(f=++Ta);if(typeof b==="object"){a[H]=f;e=e[f]=c.extend(true,{},b)}else e=e[f]?e[f]:typeof d==="undefined"?Ua:(e[f]={});if(d!==w){a[H]=f;e[b]=d}return typeof b==="string"?e[b]:e}},removeData:function(a,b){if(!(a.nodeName&&c.noData[a.nodeName.toLowerCase()])){a=a==A?ya:a;var d=a[H],f=c.cache,e=f[d];if(b){if(e){delete e[b];c.isEmptyObject(e)&&c.removeData(a)}}else{try{delete a[H]}catch(i){a.removeAttribute&& -a.removeAttribute(H)}delete f[d]}}}});c.fn.extend({data:function(a,b){if(typeof a==="undefined"&&this.length)return c.data(this[0]);else if(typeof a==="object")return this.each(function(){c.data(this,a)});var d=a.split(".");d[1]=d[1]?"."+d[1]:"";if(b===w){var f=this.triggerHandler("getData"+d[1]+"!",[d[0]]);if(f===w&&this.length)f=c.data(this[0],a);return f===w&&d[1]?this.data(d[0]):f}else return this.trigger("setData"+d[1]+"!",[d[0],b]).each(function(){c.data(this,a,b)})},removeData:function(a){return this.each(function(){c.removeData(this, -a)})}});c.extend({queue:function(a,b,d){if(a){b=(b||"fx")+"queue";var f=c.data(a,b);if(!d)return f||[];if(!f||c.isArray(d))f=c.data(a,b,c.makeArray(d));else f.push(d);return f}},dequeue:function(a,b){b=b||"fx";var d=c.queue(a,b),f=d.shift();if(f==="inprogress")f=d.shift();if(f){b==="fx"&&d.unshift("inprogress");f.call(a,function(){c.dequeue(a,b)})}}});c.fn.extend({queue:function(a,b){if(typeof a!=="string"){b=a;a="fx"}if(b===w)return c.queue(this[0],a);return this.each(function(){var d=c.queue(this, -a,b);a==="fx"&&d[0]!=="inprogress"&&c.dequeue(this,a)})},dequeue:function(a){return this.each(function(){c.dequeue(this,a)})},delay:function(a,b){a=c.fx?c.fx.speeds[a]||a:a;b=b||"fx";return this.queue(b,function(){var d=this;setTimeout(function(){c.dequeue(d,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])}});var za=/[\n\t]/g,fa=/\s+/,Va=/\r/g,Wa=/href|src|style/,Xa=/(button|input)/i,Ya=/(button|input|object|select|textarea)/i,Za=/^(a|area)$/i,Aa=/radio|checkbox/;c.fn.extend({attr:function(a, -b){return $(this,a,b,true,c.attr)},removeAttr:function(a){return this.each(function(){c.attr(this,a,"");this.nodeType===1&&this.removeAttribute(a)})},addClass:function(a){if(c.isFunction(a))return this.each(function(p){var n=c(this);n.addClass(a.call(this,p,n.attr("class")))});if(a&&typeof a==="string")for(var b=(a||"").split(fa),d=0,f=this.length;d-1)return true;return false},val:function(a){if(a===w){var b=this[0];if(b){if(c.nodeName(b,"option"))return(b.attributes.value||{}).specified?b.value:b.text;if(c.nodeName(b,"select")){var d=b.selectedIndex,f=[],e=b.options;b=b.type==="select-one";if(d<0)return null;var i=b?d:0;for(d=b?d+1:e.length;i=0;else if(c.nodeName(this,"select")){var z=c.makeArray(t);c("option",this).each(function(){this.selected=c.inArray(c(this).val(),z)>=0});if(!z.length)this.selectedIndex= --1}else this.value=t}})}});c.extend({attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(a,b,d,f){if(!a||a.nodeType===3||a.nodeType===8)return w;if(f&&b in c.attrFn)return c(a)[b](d);f=a.nodeType!==1||!c.isXMLDoc(a);var e=d!==w;b=f&&c.props[b]||b;if(a.nodeType===1){var i=Wa.test(b);if(b in a&&f&&!i){if(e){if(b==="type"&&Xa.test(a.nodeName)&&a.parentNode)throw"type property can't be changed";a[b]=d}if(c.nodeName(a,"form")&&a.getAttributeNode(b))return a.getAttributeNode(b).nodeValue; -if(b==="tabIndex")return(b=a.getAttributeNode("tabIndex"))&&b.specified?b.value:Ya.test(a.nodeName)||Za.test(a.nodeName)&&a.href?0:w;return a[b]}if(!c.support.style&&f&&b==="style"){if(e)a.style.cssText=""+d;return a.style.cssText}e&&a.setAttribute(b,""+d);a=!c.support.hrefNormalized&&f&&i?a.getAttribute(b,2):a.getAttribute(b);return a===null?w:a}return c.style(a,b,d)}});var $a=function(a){return a.replace(/[^\w\s\.\|`]/g,function(b){return"\\"+b})};c.event={add:function(a,b,d,f){if(!(a.nodeType=== -3||a.nodeType===8)){if(a.setInterval&&a!==A&&!a.frameElement)a=A;if(!d.guid)d.guid=c.guid++;if(f!==w){d=c.proxy(d);d.data=f}var e=c.data(a,"events")||c.data(a,"events",{}),i=c.data(a,"handle"),j;if(!i){j=function(){return typeof c!=="undefined"&&!c.event.triggered?c.event.handle.apply(j.elem,arguments):w};i=c.data(a,"handle",j)}if(i){i.elem=a;b=b.split(/\s+/);for(var o,p=0;o=b[p++];){var n=o.split(".");o=n.shift();d.type=n.slice(0).sort().join(".");var t=e[o],z=this.special[o]||{};if(!t){t=e[o]={}; -if(!z.setup||z.setup.call(a,f,n,d)===false)if(a.addEventListener)a.addEventListener(o,i,false);else a.attachEvent&&a.attachEvent("on"+o,i)}if(z.add)if((n=z.add.call(a,d,f,n,t))&&c.isFunction(n)){n.guid=n.guid||d.guid;d=n}t[d.guid]=d;this.global[o]=true}a=null}}},global:{},remove:function(a,b,d){if(!(a.nodeType===3||a.nodeType===8)){var f=c.data(a,"events"),e,i,j;if(f){if(b===w||typeof b==="string"&&b.charAt(0)===".")for(i in f)this.remove(a,i+(b||""));else{if(b.type){d=b.handler;b=b.type}b=b.split(/\s+/); -for(var o=0;i=b[o++];){var p=i.split(".");i=p.shift();var n=!p.length,t=c.map(p.slice(0).sort(),$a);t=new RegExp("(^|\\.)"+t.join("\\.(?:.*\\.)?")+"(\\.|$)");var z=this.special[i]||{};if(f[i]){if(d){j=f[i][d.guid];delete f[i][d.guid]}else for(var B in f[i])if(n||t.test(f[i][B].type))delete f[i][B];z.remove&&z.remove.call(a,p,j);for(e in f[i])break;if(!e){if(!z.teardown||z.teardown.call(a,p)===false)if(a.removeEventListener)a.removeEventListener(i,c.data(a,"handle"),false);else a.detachEvent&&a.detachEvent("on"+ -i,c.data(a,"handle"));e=null;delete f[i]}}}}for(e in f)break;if(!e){if(B=c.data(a,"handle"))B.elem=null;c.removeData(a,"events");c.removeData(a,"handle")}}}},trigger:function(a,b,d,f){var e=a.type||a;if(!f){a=typeof a==="object"?a[H]?a:c.extend(c.Event(e),a):c.Event(e);if(e.indexOf("!")>=0){a.type=e=e.slice(0,-1);a.exclusive=true}if(!d){a.stopPropagation();this.global[e]&&c.each(c.cache,function(){this.events&&this.events[e]&&c.event.trigger(a,b,this.handle.elem)})}if(!d||d.nodeType===3||d.nodeType=== -8)return w;a.result=w;a.target=d;b=c.makeArray(b);b.unshift(a)}a.currentTarget=d;var i=c.data(d,"handle");i&&i.apply(d,b);var j,o;try{if(!(d&&d.nodeName&&c.noData[d.nodeName.toLowerCase()])){j=d[e];o=d["on"+e]}}catch(p){}i=c.nodeName(d,"a")&&e==="click";if(!f&&j&&!a.isDefaultPrevented()&&!i){this.triggered=true;try{d[e]()}catch(n){}}else if(o&&d["on"+e].apply(d,b)===false)a.result=false;this.triggered=false;if(!a.isPropagationStopped())(d=d.parentNode||d.ownerDocument)&&c.event.trigger(a,b,d,true)}, -handle:function(a){var b,d;a=arguments[0]=c.event.fix(a||A.event);a.currentTarget=this;d=a.type.split(".");a.type=d.shift();b=!d.length&&!a.exclusive;var f=new RegExp("(^|\\.)"+d.slice(0).sort().join("\\.(?:.*\\.)?")+"(\\.|$)");d=(c.data(this,"events")||{})[a.type];for(var e in d){var i=d[e];if(b||f.test(i.type)){a.handler=i;a.data=i.data;i=i.apply(this,arguments);if(i!==w){a.result=i;if(i===false){a.preventDefault();a.stopPropagation()}}if(a.isImmediatePropagationStopped())break}}return a.result}, -props:"altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY originalTarget pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),fix:function(a){if(a[H])return a;var b=a;a=c.Event(b);for(var d=this.props.length,f;d;){f=this.props[--d];a[f]=b[f]}if(!a.target)a.target=a.srcElement|| -s;if(a.target.nodeType===3)a.target=a.target.parentNode;if(!a.relatedTarget&&a.fromElement)a.relatedTarget=a.fromElement===a.target?a.toElement:a.fromElement;if(a.pageX==null&&a.clientX!=null){b=s.documentElement;d=s.body;a.pageX=a.clientX+(b&&b.scrollLeft||d&&d.scrollLeft||0)-(b&&b.clientLeft||d&&d.clientLeft||0);a.pageY=a.clientY+(b&&b.scrollTop||d&&d.scrollTop||0)-(b&&b.clientTop||d&&d.clientTop||0)}if(!a.which&&(a.charCode||a.charCode===0?a.charCode:a.keyCode))a.which=a.charCode||a.keyCode;if(!a.metaKey&& -a.ctrlKey)a.metaKey=a.ctrlKey;if(!a.which&&a.button!==w)a.which=a.button&1?1:a.button&2?3:a.button&4?2:0;return a},guid:1E8,proxy:c.proxy,special:{ready:{setup:c.bindReady,teardown:c.noop},live:{add:function(a,b){c.extend(a,b||{});a.guid+=b.selector+b.live;c.event.add(this,b.live,qa,b)},remove:function(a){if(a.length){var b=0,d=new RegExp("(^|\\.)"+a[0]+"(\\.|$)");c.each(c.data(this,"events").live||{},function(){d.test(this.type)&&b++});b<1&&c.event.remove(this,a[0],qa)}},special:{}},beforeunload:{setup:function(a, -b,d){if(this.setInterval)this.onbeforeunload=d;return false},teardown:function(a,b){if(this.onbeforeunload===b)this.onbeforeunload=null}}}};c.Event=function(a){if(!this.preventDefault)return new c.Event(a);if(a&&a.type){this.originalEvent=a;this.type=a.type}else this.type=a;this.timeStamp=K();this[H]=true};c.Event.prototype={preventDefault:function(){this.isDefaultPrevented=ba;var a=this.originalEvent;if(a){a.preventDefault&&a.preventDefault();a.returnValue=false}},stopPropagation:function(){this.isPropagationStopped= -ba;var a=this.originalEvent;if(a){a.stopPropagation&&a.stopPropagation();a.cancelBubble=true}},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=ba;this.stopPropagation()},isDefaultPrevented:aa,isPropagationStopped:aa,isImmediatePropagationStopped:aa};var Ba=function(a){for(var b=a.relatedTarget;b&&b!==this;)try{b=b.parentNode}catch(d){break}if(b!==this){a.type=a.data;c.event.handle.apply(this,arguments)}},Ca=function(a){a.type=a.data;c.event.handle.apply(this,arguments)};c.each({mouseenter:"mouseover", -mouseleave:"mouseout"},function(a,b){c.event.special[a]={setup:function(d){c.event.add(this,b,d&&d.selector?Ca:Ba,a)},teardown:function(d){c.event.remove(this,b,d&&d.selector?Ca:Ba)}}});if(!c.support.submitBubbles)c.event.special.submit={setup:function(a,b,d){if(this.nodeName.toLowerCase()!=="form"){c.event.add(this,"click.specialSubmit."+d.guid,function(f){var e=f.target,i=e.type;if((i==="submit"||i==="image")&&c(e).closest("form").length)return pa("submit",this,arguments)});c.event.add(this,"keypress.specialSubmit."+ -d.guid,function(f){var e=f.target,i=e.type;if((i==="text"||i==="password")&&c(e).closest("form").length&&f.keyCode===13)return pa("submit",this,arguments)})}else return false},remove:function(a,b){c.event.remove(this,"click.specialSubmit"+(b?"."+b.guid:""));c.event.remove(this,"keypress.specialSubmit"+(b?"."+b.guid:""))}};if(!c.support.changeBubbles){var ga=/textarea|input|select/i;function Da(a){var b=a.type,d=a.value;if(b==="radio"||b==="checkbox")d=a.checked;else if(b==="select-multiple")d=a.selectedIndex> --1?c.map(a.options,function(f){return f.selected}).join("-"):"";else if(a.nodeName.toLowerCase()==="select")d=a.selectedIndex;return d}function ha(a,b){var d=a.target,f,e;if(!(!ga.test(d.nodeName)||d.readOnly)){f=c.data(d,"_change_data");e=Da(d);if(e!==f){if(a.type!=="focusout"||d.type!=="radio")c.data(d,"_change_data",e);if(d.type!=="select"&&(f!=null||e)){a.type="change";return c.event.trigger(a,b,this)}}}}c.event.special.change={filters:{focusout:ha,click:function(a){var b=a.target,d=b.type;if(d=== -"radio"||d==="checkbox"||b.nodeName.toLowerCase()==="select")return ha.call(this,a)},keydown:function(a){var b=a.target,d=b.type;if(a.keyCode===13&&b.nodeName.toLowerCase()!=="textarea"||a.keyCode===32&&(d==="checkbox"||d==="radio")||d==="select-multiple")return ha.call(this,a)},beforeactivate:function(a){a=a.target;a.nodeName.toLowerCase()==="input"&&a.type==="radio"&&c.data(a,"_change_data",Da(a))}},setup:function(a,b,d){for(var f in W)c.event.add(this,f+".specialChange."+d.guid,W[f]);return ga.test(this.nodeName)}, -remove:function(a,b){for(var d in W)c.event.remove(this,d+".specialChange"+(b?"."+b.guid:""),W[d]);return ga.test(this.nodeName)}};var W=c.event.special.change.filters}s.addEventListener&&c.each({focus:"focusin",blur:"focusout"},function(a,b){function d(f){f=c.event.fix(f);f.type=b;return c.event.handle.call(this,f)}c.event.special[b]={setup:function(){this.addEventListener(a,d,true)},teardown:function(){this.removeEventListener(a,d,true)}}});c.each(["bind","one"],function(a,b){c.fn[b]=function(d, -f,e){if(typeof d==="object"){for(var i in d)this[b](i,f,d[i],e);return this}if(c.isFunction(f)){thisObject=e;e=f;f=w}var j=b==="one"?c.proxy(e,function(o){c(this).unbind(o,j);return e.apply(this,arguments)}):e;return d==="unload"&&b!=="one"?this.one(d,f,e,thisObject):this.each(function(){c.event.add(this,d,j,f)})}});c.fn.extend({unbind:function(a,b){if(typeof a==="object"&&!a.preventDefault){for(var d in a)this.unbind(d,a[d]);return this}return this.each(function(){c.event.remove(this,a,b)})},trigger:function(a, -b){return this.each(function(){c.event.trigger(a,b,this)})},triggerHandler:function(a,b){if(this[0]){a=c.Event(a);a.preventDefault();a.stopPropagation();c.event.trigger(a,b,this[0]);return a.result}},toggle:function(a){for(var b=arguments,d=1;d0){y=u;break}}u=u[g]}m[r]=y}}}var f=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^[\]]*\]|['"][^'"]*['"]|[^[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g, -e=0,i=Object.prototype.toString,j=false,o=true;[0,0].sort(function(){o=false;return 0});var p=function(g,h,k,m){k=k||[];var r=h=h||s;if(h.nodeType!==1&&h.nodeType!==9)return[];if(!g||typeof g!=="string")return k;for(var q=[],v,u,y,S,I=true,N=x(h),J=g;(f.exec(""),v=f.exec(J))!==null;){J=v[3];q.push(v[1]);if(v[2]){S=v[3];break}}if(q.length>1&&t.exec(g))if(q.length===2&&n.relative[q[0]])u=ia(q[0]+q[1],h);else for(u=n.relative[q[0]]?[h]:p(q.shift(),h);q.length;){g=q.shift();if(n.relative[g])g+=q.shift(); -u=ia(g,u)}else{if(!m&&q.length>1&&h.nodeType===9&&!N&&n.match.ID.test(q[0])&&!n.match.ID.test(q[q.length-1])){v=p.find(q.shift(),h,N);h=v.expr?p.filter(v.expr,v.set)[0]:v.set[0]}if(h){v=m?{expr:q.pop(),set:B(m)}:p.find(q.pop(),q.length===1&&(q[0]==="~"||q[0]==="+")&&h.parentNode?h.parentNode:h,N);u=v.expr?p.filter(v.expr,v.set):v.set;if(q.length>0)y=B(u);else I=false;for(;q.length;){var E=q.pop();v=E;if(n.relative[E])v=q.pop();else E="";if(v==null)v=h;n.relative[E](y,v,N)}}else y=[]}y||(y=u);if(!y)throw"Syntax error, unrecognized expression: "+ -(E||g);if(i.call(y)==="[object Array]")if(I)if(h&&h.nodeType===1)for(g=0;y[g]!=null;g++){if(y[g]&&(y[g]===true||y[g].nodeType===1&&F(h,y[g])))k.push(u[g])}else for(g=0;y[g]!=null;g++)y[g]&&y[g].nodeType===1&&k.push(u[g]);else k.push.apply(k,y);else B(y,k);if(S){p(S,r,k,m);p.uniqueSort(k)}return k};p.uniqueSort=function(g){if(D){j=o;g.sort(D);if(j)for(var h=1;h":function(g,h){var k=typeof h==="string";if(k&&!/\W/.test(h)){h=h.toLowerCase();for(var m=0,r=g.length;m=0))k||m.push(v);else if(k)h[q]=false;return false},ID:function(g){return g[1].replace(/\\/g,"")},TAG:function(g){return g[1].toLowerCase()}, -CHILD:function(g){if(g[1]==="nth"){var h=/(-?)(\d*)n((?:\+|-)?\d*)/.exec(g[2]==="even"&&"2n"||g[2]==="odd"&&"2n+1"||!/\D/.test(g[2])&&"0n+"+g[2]||g[2]);g[2]=h[1]+(h[2]||1)-0;g[3]=h[3]-0}g[0]=e++;return g},ATTR:function(g,h,k,m,r,q){h=g[1].replace(/\\/g,"");if(!q&&n.attrMap[h])g[1]=n.attrMap[h];if(g[2]==="~=")g[4]=" "+g[4]+" ";return g},PSEUDO:function(g,h,k,m,r){if(g[1]==="not")if((f.exec(g[3])||"").length>1||/^\w/.test(g[3]))g[3]=p(g[3],null,null,h);else{g=p.filter(g[3],h,k,true^r);k||m.push.apply(m, -g);return false}else if(n.match.POS.test(g[0])||n.match.CHILD.test(g[0]))return true;return g},POS:function(g){g.unshift(true);return g}},filters:{enabled:function(g){return g.disabled===false&&g.type!=="hidden"},disabled:function(g){return g.disabled===true},checked:function(g){return g.checked===true},selected:function(g){return g.selected===true},parent:function(g){return!!g.firstChild},empty:function(g){return!g.firstChild},has:function(g,h,k){return!!p(k[3],g).length},header:function(g){return/h\d/i.test(g.nodeName)}, -text:function(g){return"text"===g.type},radio:function(g){return"radio"===g.type},checkbox:function(g){return"checkbox"===g.type},file:function(g){return"file"===g.type},password:function(g){return"password"===g.type},submit:function(g){return"submit"===g.type},image:function(g){return"image"===g.type},reset:function(g){return"reset"===g.type},button:function(g){return"button"===g.type||g.nodeName.toLowerCase()==="button"},input:function(g){return/input|select|textarea|button/i.test(g.nodeName)}}, -setFilters:{first:function(g,h){return h===0},last:function(g,h,k,m){return h===m.length-1},even:function(g,h){return h%2===0},odd:function(g,h){return h%2===1},lt:function(g,h,k){return hk[3]-0},nth:function(g,h,k){return k[3]-0===h},eq:function(g,h,k){return k[3]-0===h}},filter:{PSEUDO:function(g,h,k,m){var r=h[1],q=n.filters[r];if(q)return q(g,k,h,m);else if(r==="contains")return(g.textContent||g.innerText||a([g])||"").indexOf(h[3])>=0;else if(r==="not"){h= -h[3];k=0;for(m=h.length;k=0}},ID:function(g,h){return g.nodeType===1&&g.getAttribute("id")===h},TAG:function(g,h){return h==="*"&&g.nodeType===1||g.nodeName.toLowerCase()===h},CLASS:function(g,h){return(" "+(g.className||g.getAttribute("class"))+" ").indexOf(h)>-1},ATTR:function(g,h){var k=h[1];g=n.attrHandle[k]?n.attrHandle[k](g):g[k]!=null?g[k]:g.getAttribute(k);k=g+"";var m=h[2];h=h[4];return g==null?m==="!=":m=== -"="?k===h:m==="*="?k.indexOf(h)>=0:m==="~="?(" "+k+" ").indexOf(h)>=0:!h?k&&g!==false:m==="!="?k!==h:m==="^="?k.indexOf(h)===0:m==="$="?k.substr(k.length-h.length)===h:m==="|="?k===h||k.substr(0,h.length+1)===h+"-":false},POS:function(g,h,k,m){var r=n.setFilters[h[2]];if(r)return r(g,k,h,m)}}},t=n.match.POS;for(var z in n.match){n.match[z]=new RegExp(n.match[z].source+/(?![^\[]*\])(?![^\(]*\))/.source);n.leftMatch[z]=new RegExp(/(^(?:.|\r|\n)*?)/.source+n.match[z].source.replace(/\\(\d+)/g,function(g, -h){return"\\"+(h-0+1)}))}var B=function(g,h){g=Array.prototype.slice.call(g,0);if(h){h.push.apply(h,g);return h}return g};try{Array.prototype.slice.call(s.documentElement.childNodes,0)}catch(C){B=function(g,h){h=h||[];if(i.call(g)==="[object Array]")Array.prototype.push.apply(h,g);else if(typeof g.length==="number")for(var k=0,m=g.length;k";var k=s.documentElement;k.insertBefore(g,k.firstChild);if(s.getElementById(h)){n.find.ID=function(m,r,q){if(typeof r.getElementById!=="undefined"&&!q)return(r=r.getElementById(m[1]))?r.id===m[1]||typeof r.getAttributeNode!=="undefined"&& -r.getAttributeNode("id").nodeValue===m[1]?[r]:w:[]};n.filter.ID=function(m,r){var q=typeof m.getAttributeNode!=="undefined"&&m.getAttributeNode("id");return m.nodeType===1&&q&&q.nodeValue===r}}k.removeChild(g);k=g=null})();(function(){var g=s.createElement("div");g.appendChild(s.createComment(""));if(g.getElementsByTagName("*").length>0)n.find.TAG=function(h,k){k=k.getElementsByTagName(h[1]);if(h[1]==="*"){h=[];for(var m=0;k[m];m++)k[m].nodeType===1&&h.push(k[m]);k=h}return k};g.innerHTML=""; -if(g.firstChild&&typeof g.firstChild.getAttribute!=="undefined"&&g.firstChild.getAttribute("href")!=="#")n.attrHandle.href=function(h){return h.getAttribute("href",2)};g=null})();s.querySelectorAll&&function(){var g=p,h=s.createElement("div");h.innerHTML="

";if(!(h.querySelectorAll&&h.querySelectorAll(".TEST").length===0)){p=function(m,r,q,v){r=r||s;if(!v&&r.nodeType===9&&!x(r))try{return B(r.querySelectorAll(m),q)}catch(u){}return g(m,r,q,v)};for(var k in g)p[k]=g[k];h=null}}(); -(function(){var g=s.createElement("div");g.innerHTML="
";if(!(!g.getElementsByClassName||g.getElementsByClassName("e").length===0)){g.lastChild.className="e";if(g.getElementsByClassName("e").length!==1){n.order.splice(1,0,"CLASS");n.find.CLASS=function(h,k,m){if(typeof k.getElementsByClassName!=="undefined"&&!m)return k.getElementsByClassName(h[1])};g=null}}})();var F=s.compareDocumentPosition?function(g,h){return g.compareDocumentPosition(h)&16}:function(g, -h){return g!==h&&(g.contains?g.contains(h):true)},x=function(g){return(g=(g?g.ownerDocument||g:0).documentElement)?g.nodeName!=="HTML":false},ia=function(g,h){var k=[],m="",r;for(h=h.nodeType?[h]:h;r=n.match.PSEUDO.exec(g);){m+=r[0];g=g.replace(n.match.PSEUDO,"")}g=n.relative[g]?g+"*":g;r=0;for(var q=h.length;r=0===d})};c.fn.extend({find:function(a){for(var b=this.pushStack("","find",a),d=0,f=0,e=this.length;f0)for(var i=d;i0},closest:function(a,b){if(c.isArray(a)){var d=[],f=this[0],e,i= -{},j;if(f&&a.length){e=0;for(var o=a.length;e-1:c(f).is(e)){d.push({selector:j,elem:f});delete i[j]}}f=f.parentNode}}return d}var p=c.expr.match.POS.test(a)?c(a,b||this.context):null;return this.map(function(n,t){for(;t&&t.ownerDocument&&t!==b;){if(p?p.index(t)>-1:c(t).is(a))return t;t=t.parentNode}return null})},index:function(a){if(!a||typeof a=== -"string")return c.inArray(this[0],a?c(a):this.parent().children());return c.inArray(a.jquery?a[0]:a,this)},add:function(a,b){a=typeof a==="string"?c(a,b||this.context):c.makeArray(a);b=c.merge(this.get(),a);return this.pushStack(sa(a[0])||sa(b[0])?b:c.unique(b))},andSelf:function(){return this.add(this.prevObject)}});c.each({parent:function(a){return(a=a.parentNode)&&a.nodeType!==11?a:null},parents:function(a){return c.dir(a,"parentNode")},parentsUntil:function(a,b,d){return c.dir(a,"parentNode", -d)},next:function(a){return c.nth(a,2,"nextSibling")},prev:function(a){return c.nth(a,2,"previousSibling")},nextAll:function(a){return c.dir(a,"nextSibling")},prevAll:function(a){return c.dir(a,"previousSibling")},nextUntil:function(a,b,d){return c.dir(a,"nextSibling",d)},prevUntil:function(a,b,d){return c.dir(a,"previousSibling",d)},siblings:function(a){return c.sibling(a.parentNode.firstChild,a)},children:function(a){return c.sibling(a.firstChild)},contents:function(a){return c.nodeName(a,"iframe")? -a.contentDocument||a.contentWindow.document:c.makeArray(a.childNodes)}},function(a,b){c.fn[a]=function(d,f){var e=c.map(this,b,d);ab.test(a)||(f=d);if(f&&typeof f==="string")e=c.filter(f,e);e=this.length>1?c.unique(e):e;if((this.length>1||cb.test(f))&&bb.test(a))e=e.reverse();return this.pushStack(e,a,R.call(arguments).join(","))}});c.extend({filter:function(a,b,d){if(d)a=":not("+a+")";return c.find.matches(a,b)},dir:function(a,b,d){var f=[];for(a=a[b];a&&a.nodeType!==9&&(d===w||!c(a).is(d));){a.nodeType=== -1&&f.push(a);a=a[b]}return f},nth:function(a,b,d){b=b||1;for(var f=0;a;a=a[d])if(a.nodeType===1&&++f===b)break;return a},sibling:function(a,b){for(var d=[];a;a=a.nextSibling)a.nodeType===1&&a!==b&&d.push(a);return d}});var Ga=/ jQuery\d+="(?:\d+|null)"/g,Y=/^\s+/,db=/(<([\w:]+)[^>]*?)\/>/g,eb=/^(?:area|br|col|embed|hr|img|input|link|meta|param)$/i,Ha=/<([\w:]+)/,fb=/"},G={option:[1,""], -legend:[1,"
","
"],thead:[1,"","
"],tr:[2,"","
"],td:[3,"","
"],col:[2,"","
"],area:[1,"",""],_default:[0,"",""]};G.optgroup=G.option;G.tbody=G.tfoot=G.colgroup=G.caption=G.thead;G.th=G.td;if(!c.support.htmlSerialize)G._default=[1,"div
","
"];c.fn.extend({text:function(a){if(c.isFunction(a))return this.each(function(b){var d=c(this); -return d.text(a.call(this,b,d.text()))});if(typeof a!=="object"&&a!==w)return this.empty().append((this[0]&&this[0].ownerDocument||s).createTextNode(a));return c.getText(this)},wrapAll:function(a){if(c.isFunction(a))return this.each(function(d){c(this).wrapAll(a.call(this,d))});if(this[0]){var b=c(a,this[0].ownerDocument).eq(0).clone(true);this[0].parentNode&&b.insertBefore(this[0]);b.map(function(){for(var d=this;d.firstChild&&d.firstChild.nodeType===1;)d=d.firstChild;return d}).append(this)}return this}, -wrapInner:function(a){return this.each(function(){var b=c(this),d=b.contents();d.length?d.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){c(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){c.nodeName(this,"body")||c(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,true,function(a){this.nodeType===1&& -this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this)});else if(arguments.length){var a=c(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,false,function(b){this.parentNode.insertBefore(b,this.nextSibling)});else if(arguments.length){var a=this.pushStack(this, -"after",arguments);a.push.apply(a,c(arguments[0]).toArray());return a}},clone:function(a){var b=this.map(function(){if(!c.support.noCloneEvent&&!c.isXMLDoc(this)){var d=this.outerHTML,f=this.ownerDocument;if(!d){d=f.createElement("div");d.appendChild(this.cloneNode(true));d=d.innerHTML}return c.clean([d.replace(Ga,"").replace(Y,"")],f)[0]}else return this.cloneNode(true)});if(a===true){ta(this,b);ta(this.find("*"),b.find("*"))}return b},html:function(a){if(a===w)return this[0]&&this[0].nodeType=== -1?this[0].innerHTML.replace(Ga,""):null;else if(typeof a==="string"&&!/ - - - - - - - - - - - - - - -
-

- Boost

- 		-

Boost/Serialization ‐ Portable Binary Archives (PBA)

-

Tutorial

-

NOTE: Naming differs from the source code! This tutorial reflects our hope to get - the eos portable archives an official part of the boost serialization library. - Work is in progress :-) -

-
- - -
-
-
Quick start
- -
- -
Format -
- -
Examples
- -
-
- -

-

-
- -

Quick start

- -Are you impatient to enjoy the Boost Portable Binary Archives (PBA) ? -If so, this section is for you. - -

How to store some simple data in a portable binary output archive

- -

-The tutorial_pba_0.cpp sample -program uses a boost::archive::portable_binary_oarchive object -attached to a standard output file stream to store a couple of -variables of primitive types (bool, -char, integer -numbers, floating numbers) and even a std::string. -

- -
- The tutorial_pba_0.cpp source code Download -  Show/hide -
-/** tutorial_pba_0.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This quick start example shows how to store some variables
- * of basic types (bool, integer, floating point numbers, STL string) 
- * using the portable binary archive format associated to a 
- * standard output file stream.
- *
- */
-
-#include <string>
-#include <fstream>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
-
-int main (void)
-{
-  // The name for the example data file :  
-  std::string filename = "pba_0.data"; 
-
-  // Some variables of various primitive types :
-  bool        b              = true;
-  char        c              = 'B';
-  uint32_t    answer         = 42;
-  float       computing_time = 7.5e6;
-  double      e              = 2.71828182845905;
-  std::string slogan         = "DON'T PANIC";
-  
-  // Open an output file stream in binary mode :
-  std::ofstream fout (filename.c_str (), std::ios_base::binary);
-  
-  {
-    // Create an output portable binary archive attached to the output file :
-    boost::archive::portable_binary_oarchive opba (fout);
-    
-    // Store (serializing) variables :
-    opba & b & c & answer & computing_time & e & slogan;
-  }
-
-  return 0;   
-}
-
-// end of tutorial_pba_0.cpp
-
-
-

- - -

-The compiled executable creates the pba_0.data file which -contains the following bytes: -

127   1   9   1  84   1  66   1  42   4 192 225 228  74   8 116
- 87  20 139  10 191   5  64   1  11  68  79  78  39  84  32  80
- 65  78  73  67
-
-This format is explained in details below. - - -

Note: -

    -
  • the output of the byte content in the pba_0.data file -can be obtained invoking some command like -  od -t u1 pba_0.data  -on a GNU/Linux system.
    -
  • one should notice that this program makes use of -the typedef-ed integer types from -the boost/cstdint.hpp header (uint32_t...). It is a -strong recommendation to ensure cross-environment portability while -(de)serializing integer numbers (see also -the Examples section). -
-

- - - -
-To top -
-
- - -

How to load some simple data from a portable binary input archive

- -

-The tutorial_pba_1.cpp sample program uses -a boost::archive::portable_binary_iarchive object -attached to a standard input file stream in order to load the -data previously stored by -the tutorial_pba_0.cpp program in -the pba_0.data file. -

- - -
- The tutorial_pba_1.cpp source code Download -  Show/hide -
-/** tutorial_pba_1.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization package. 
- *
- * This quick start example shows how to load some variables
- * of basic types (bool, integer, floating point numbers, STL string) 
- * using the portable binary archive format associated to a 
- * standard input file stream. 
- *
- */
-
-#include <iostream>
-#include <string>
-#include <fstream>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-
-int main (void)
-{
-  using namespace std;
-
-  // The name for the example data file :  
-  string filename = "pba_0.data"; 
-
-  // Some variables of various types :
-  bool     b;
-  char     c;
-  uint32_t answer;
-  float    computing_time;
-  double   e;
-  string   slogan;
-   
-  // Open an input file stream in binary mode :
-  ifstream fin (filename.c_str (), ios_base::binary);
-  
-  {
-    // Create an input portable binary archive attached to the input file :
-    boost::archive::portable_binary_iarchive ipba (fin);
-    
-    // Loading (de-serializing) variables using the same 
-    // order than for serialization (see tutorial_pba_0.cpp) :
-    ipba & b & c & answer & computing_time & e & slogan;
-  }
-
-  cout.precision (15);
-  cout << "Variable 'b' is              : " << b << " " << "(bool)" << endl;
-  cout << "Variable 'c' is              : '" << c << "' " << " " << "(char)" << endl;
-  cout << "Variable 'answer' is         : " << answer  << " " << "(unsigned 32-bit integer)" << endl;
-  cout << "Variable 'computing_time' is : " << computing_time  << " " << "(single precision 32-bit float)" << endl;
-  cout << "Variable 'e' is              : " << e  << " " << "(double precision 64-bit float)" << endl;
-  cout << "Variable 'slogan' is         : \"" << slogan  << "\" " << "(std::string)" << endl;
-
-  return 0;   
-}
-
-// end of tutorial_pba_1.cpp
-
-
-

- - -

-The executable reads the pba_0.data file and deserializes its -contents in the same order it has been stored. It then prints the -restored values of the variables: -

Variable 'b' is              : 1 (bool)
-Variable 'c' is              : 'B'  (char)
-Variable 'answer' is         : 42 (unsigned 32-bit integer)
-Variable 'computing_time' is : 7500000 (single precision 32-bit float)
-Variable 'e' is              : 2.71828182845905 (double precision 64-bit float)
-Variable 'slogan' is         : "DON'T PANIC" (std::string)
-
-

- - - -
-To top -
-
- - - -
-

Format

- -

-This section aims to give some details about the binary format of -portable binary archives (PBA). We will analyse the byte contents of -the sample binary archive pba_0.data file -created by the tutorial_pba_0.cpp -program (see the previous section). -

- -

-Like any other archive format within Boost/Serialization, a PBA starts -with a header (this is the default behaviour but it is possible to -deactivate the use of this header using a special flag at -construction, see this example). This -header is made of two informations : -

    -
  • - a magic byte with the conventionnal decimal value : - 
    127
    -
  • - the Boost library version number which is encoded as an integer - number. -

    The PBA encoding of integer numbers uses the following - scheme: - <size> <content>, - where first the size stores the minimal - number of non zero bytes needed to store the binary - representation of the integer value; then the bytes corresponding to the content are stored - starting from the less significant ones (see - also this example). For the library - version number we have here: - 

    1 9
    - where 1 is the number of byte needed to store the - value 9 which comes with the Serialization library for - Boost version 1.47.0. Here, the 9 value being less than 256, - one unique byte is enough to store this number. -
-

-

- -

-Now we are done with the header, let's have a look on the serialized data ! -

- -
    -
  • -The first variable is of boolean type with value true. Here -the PBA conventionnaly encodes the true value with the -character 'T' which is stored using the corresponding ASCII -integer value (84). As 84 is less -than 256, this uses only 1 byte: -
    1  84
    -

    - -
  • The next variable uses 1 byte to store a 8-bit -character (char) taking value 'B'. Again the PBA -scheme encodes it using its ASCII integer value -(66). This gives: -
    1  66
    -

    - -
  • Then we have an unsigned 32-bit integer with -value 42 (a fondamental constant from -the H2G2). As the natural binary 32-bit encoding of this value -(<256) only needs -1 non-zero byte, the PBA stores : -
    1  42
    -This scheme results in saving 2 bytes compared to the size of the transient value. -

    - -
  • The next variable is a single precision number with -value 7.5×106. Following the IEEE 754 standard one thus uses -32 bits to encode the sign bit, -the 8-bit exponent and -the 23-bit significant (mantissa). -As 7.5×106 can be rewritten -in +1.78813934×222, we have the -following bits contents:
    -
    -010010101111001001110000111000000

    -where the phantom bit (not stored) is conventionaly set -at 1 and the exponent is stored after -being shifted conventionaly by 27-1=127, thus -(10010101)2=(149)10 -(subscripts indicate the number base) and 149-127=22.
    - -
    -Packing these bits using 4 -bytes, we get:

    - - - - - - - - - - - - - -
    01001010111001001110000111000000
    74228225192
    -
    - -Thus the PBA streams the following 5 bytes (first the size then the content -from the least significant byte to the most significant byte), giving : -
    4 192 225 228 74
    - -
  • The next floating number 2.71828182845905 = -+1.35914091422952×2(1024-1023), -is stored using the double precision IEEE 754 64-bit pattern (using the conventionnal exponent shift 210-1=1023): -

    - - - - - - - - - - - - - - - - - - - - - -
    0100000000000101110111110000101010001011000101000101011101110100
    645191101392087116
    -
    - -Thus the PBA uses the following 9 bytes, the first one for the size (8) and the other ones -for the content (starting with the LSB): -
    8 116 87 20 139 10 191 5 64
    - -
  • Finally the string "DON'T PANIC" is stored: -
      -
    • - first is given the number of characters using the PBA - integer encoding scheme; here - 1 byte is enough to store the - value 11:
      - 
      1  11
      -
    • then the array of 11 characters is given using the - corresponding ASCII codes: -

      - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      DON'T PANIC
      6879783984328065787367
      -
      -
      68  79  78  39  84  32  80 65  78  73  67
      -
    - -
- -

- -

-Now the contents of the pba_0.data file can be fully -understood : -

127   1   9   1  84   1  66   1  42   4 192 225 228  74   8 116
- 87  20 139  10 191   5  64   1  11  68  79  78  39  84  32  80
- 65  78  73  67
-
- -More details about the format (non finite floating point values, -negative integer numbers) will be given in -the sample codes below. -

- - - -
-To top -
-
- - - -
-

Examples

- - - -

Handling special floating point values

- -

-The PBA has been designed in the aims to handle single and double -precision floating point numbers, including non-finite and special values: -

    -
  • ±infinity -
  • NaN (not a number) -
  • denormalized numbers (i.e. floating point numbers with non-guaranteed roundoff precision) -
-

- -

-The tutorial_pba_2.cpp sample program -illustrates the use of such special cases while serializing single precision -floating point numbers: -

- -
- The tutorial_pba_2.cpp source code Download -  Show/hide -
-/** tutorial_pba_2.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This sample program shows how to use a portable binary archive 
- * to store/load floating point numbers including non-finite and 
- * special (denormalized) values.
- *
- */
-
-#include <string>
-#include <fstream>
-#include <limits>
-
-#include <boost/archive/portable_binary_oarchive.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-
-int main (void)
-{
-  using namespace std;
-
-  // The name for the example data file :  
-  string filename = "pba_2.data"; 
-
-  {
-    // A normal single precision floating point number :
-    float pi = 3.14159265; 
-
-    // Single precision zeroed floating point number :
-    float zero = 0.0;
-
-    // A denormalized single precision floating point number :
-    float tiny = 1.e-40;
-    
-    // A single precision floating point number with `+Infinity' value :
-    float plus_infinity = numeric_limits<float>::infinity ();
-    
-    // A single precision floating point number with `-Infinity' value :
-    float minus_infinity = -numeric_limits<float>::infinity ();
-    
-    // A single precision `Not-a-Number' (NaN):
-    float nan = numeric_limits<float>::quiet_NaN ();
-    
-    // Open an output file stream in binary mode :
-    ofstream fout (filename.c_str (), ios_base::binary);
-    
-    {
-      // Create an output portable binary archive attached to the output file :
-      boost::archive::portable_binary_oarchive opba (fout);
-      
-      // Store (serialize) variables :
-      opba & pi & zero & tiny & plus_infinity & minus_infinity & nan;
-    }
-  }
-
-  { 
-    // Single precision floating point numbers to be loaded :
-    float x[6];
-
-    // Open an input file stream in binary mode :
-    ifstream fin (filename.c_str (), ios_base::binary);
-  
-    {
-      // Create an input portable binary archive attached to the input file :
-      boost::archive::portable_binary_iarchive ipba (fin);
-      
-      // Load (de-serialize) variables using the same 
-      // order than for serialization :
-      for (int i = 0; i < 6; ++i)
-	{
-	  ipba & x[i];
-	}
-    }
-
-    // Print :
-    for (int i = 0; i < 6; ++i)
-      {
-	cout.precision (8);
-	cout << "Loaded x[" << i << "] = " << x[i];
-	switch (fp::fpclassify(x[i]))
-	  {
-	  case FP_NAN: cout << " (NaN)"; break;
-	  case FP_INFINITE: cout << " (infinite)"; break;
-	  case FP_SUBNORMAL: cout << " (denormalized)"; break;
-	  case FP_NORMAL:  cout << " (normalized)"; break;
-	  }
-	cout << endl;
-      }
-  }
-
-  return 0;   
-}
-
-// end of tutorial_pba_2.cpp
-
-
-

- - -

-The pba_2.data output data file thus contains the following bytes: -

127   1   9   4 219  15  73  64   0   3 194  22   1   4   0   0
-128 127   4   0   0 128 255   4 255 255 255 127
-
-where: -
    - -
  • 127 1 9 is the standard archive header - -
  • 5 bytes are used to encode the value of π : one byte for the size=4 and 4 bytes for the content - (reversed order): -

    - - - - - - - - - - - - - -
    647315219
    01000000010010010000111111011011
    -
    - and π = +1.5707963×2(128-127) = 3.14159265. - -
  • the value zero is stored using only - one 0 byte (this is called zero optimization and this save bytes for storage). - -
  • a denormalized value then comes with only 3 bytes (note that the MSB byte is zero so it is omitted) -

    - - - - - - - - - - - - - -
    122194
    00000000000000010001011011000010
    -
    - Here the value - is +0.017014027×2(0-127) - = 0.9999946×10-40 which, as expected for - denormalized numbers, misses the requested - value 10-40 by the relative error of - magnitude 5.4×10-6, which is larger than the machine - roundoff precision (~10-7 for the single precision scheme). - -
  • the +∞ value is stored using one byte for the size - plus 4 content bytes, fulfilling the IEEE 754 standard, - i.e. maximal exponent and zero mantissa : -

    - - - - - - - - - - - - - -
    12712800
    01111111100000000000000000000000
    -
    - -
  • the -∞ value is stored using the same scheme as above, but the sign bit : -

    - - - - - - - - - - - - - -
    25512800
    11111111100000000000000000000000
    -
    - -
  • the final NaN value is stored also using one size byte plus 4 content bytes with maximal exponent and non-zero mantissa : -

    - - - - - - - - - - - - - -
    127255255255
    01111111111111111111111111111111
    -
    - See this link for an explanation of the IEEE 754 standard. - -
-

- - - -
-To top -
-
- - - -

Forbidding the serialization of non finite float values

- -

-One can ask a PBA to reject non-finite values. This is done by -passing the boost::archive::no_infnan flag to the constructor -of the output archive. Note that in this case, denormalized values are -still accepted, but infinite and NaNs aren't. -

- -

-The tutorial_pba_3.cpp sample -program that illustrates this special case: -

- -
- The tutorial_pba_3.cpp source code Download -  Show/hide -
-/** tutorial_pba_3.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This sample program shows how to use a portable binary archive 
- * and prevent the serialization of non-finite floating numbers.
- *
- */
-
-#include <string>
-#include <fstream>
-#include <limits>
-
-#include <boost/archive/portable_binary_oarchive.hpp>
-
-int main (void)
-{
-  using namespace std;
-
-  // The name for the example data file :  
-  string filename = "pba_3.data"; 
-
-  try 
-    {
-      // An array of single precision floating numbers:
-      float x[5]; 
-      x[0] = 3.14159;  // Pi
-      x[1] = 6.022e22; // Avogadro constant
-      x[2] = 1.6e-19;  // Electron charge magnitude
-      x[3] = 1.e-40;   // A tiny (denormalized) value
-      x[4] = numeric_limits<float>::infinity (); // This will fail while serializing...
-
-      // Open an output file stream in binary mode :
-      ofstream fout (filename.c_str (), ios_base::binary);
-    
-      {
-	// Create an output portable binary archive attached to the output file,
-	// using the special 'boost::archive::no_infnan' flag :
-	boost::archive::portable_binary_oarchive opba (fout, boost::archive::no_infnan);
-	
-	// Store (serialize) variables :
-	for (int i = 0; i < 5; ++i)
-	  {
-	    clog << "Serializing value : " << x[i] << " ... ";
-	    opba & x[i];
-	    clog << "Ok !" << endl;
-	  }
-      }
-    }
-  catch (exception & x)
-    {
-      cerr << "ERROR: " << x.what () << endl;
-      return 1;
-    }
-
-  return 0;   
-}
-
-// end of tutorial_pba_3.cpp
-
-
-

- - -

-We can check that the PBA now throws an exception as soon as it -encounters a non finite floating point value during the serialization -process: -

-Serializing value : 3.14159 ... Ok !
-Serializing value : 6.022e+22 ... Ok !
-Serializing value : 1.6e-19 ... Ok !
-Serializing value : 9.99995e-41 ... Ok !
-Serializing value : inf ... ERROR: serialization of illegal floating point value: inf
-
-
-

- - - -
-To top -
-
- - - -

Serializing integer numbers

- -

-The PBA obviously handles integer numbers. Unfortunately, C/C++ does -not garantee the portable size of its primitive integer types (short, -int, long... and their unsigned versions). It depends on the -architecture (32-bit/64-bit) and the compiler. -

-

The Boost library -addresses this issue through a collection of typedefs for -integer types of common sizes. This technique is supposed to allow -the manipulation of integer variables in a portable way, typically with -text or XML archives. So, we are generally - encouraged to use the boost/cstdint.hpp header file and -the typedefs defined therein. -

-

Due to its encoding scheme -of integer numbers, the PBA does not strictly need such technique to ensure -a correct behaviour while (de)serializing integer numbers. -This is because the little endian encoding approach allows to only store the non-zero bytes. -It is thus possible to serialize a value using one integer type (short int) and then -deserialize it using another integer type (long long). -

-

-However, for a strict and safe portable behaviour -of PBA, we recommend that, in most cases, the user should systematically use such typedefs for all -serializable integer values. This applies particularly for member attributes -in structs and classes and should allows the transparent switching -to another kind of archive (text, XML) thanks to the serialize template method. -

- -

-The tutorial_pba_4.cpp sample -program illustrates the serialization/deserialization of 8-bit, -16-bit, 32-bit and 64-bit integer numbers: -

- -
- The tutorial_pba_4.cpp source code Download -  Show/hide -
-/** tutorial_pba_4.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This sample program shows how to use a portable binary archive 
- * to store/load integer numbers of various sizes using the Boost 
- * portable integer typedefs.
- *
- */
-
-#include <string>
-#include <fstream>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-
-int main (void)
-{
-  using namespace std;
-
-  // The name for the example data file :  
-  string filename = "pba_4.data"; 
-
-  {
-    // Some integer numbers :
-    bool t = true;
-    char c = 'c';
-    unsigned char u = 'u';
-    int8_t   b = -3; // char
-    uint8_t  B = +6; // unsigned char 
-    int16_t  s = -16;
-    uint16_t S = +32;
-    int32_t  l = -128;
-    uint32_t L = +127;
-    int64_t  ll = -1024;
-    uint64_t LL = +2048;
-
-    // Open an output file stream in binary mode :
-    ofstream fout (filename.c_str (), ios_base::binary);
-    
-    {
-      // Create an output portable binary archive attached to the output file :
-      boost::archive::portable_binary_oarchive opba (fout);
-      
-      // Store (serialize) variables :
-      opba & t & c & u & b & B & s & S & l & L & ll & LL;
-    }
-  }
-
-  { 
-    // Single precision floating numbers to be loaded :
-    // Some integer numbers :
-    bool t;
-    char c;
-    unsigned char u;
-    int8_t   b;
-    uint8_t  B;
-    int16_t  s;
-    uint16_t S;
-    int32_t  l;
-    uint32_t L;
-    int64_t  ll;
-    uint64_t LL;
-
-    // Open an input file stream in binary mode :
-    ifstream fin (filename.c_str (), ios_base::binary);
-  
-    {
-      // Create an input portable binary archive attached to the input file :
-      boost::archive::portable_binary_iarchive ipba (fin);
-      
-      // Load (de-serialize) variables using the same 
-      // order than for serialization :
-      ipba & t & c & u & b & B & s & S & l & L & ll & LL;
-    }
-
-    clog << "t  = " << t << " (bool)" << endl;
-    clog << "c  = '" << c << "' (char)" << endl;
-    clog << "u  = '" << u << "' (unsigned char)" << endl;
-    clog << "b  = " << (int) b << " (int8_t)" << endl;
-    clog << "B  = " << (int) B << " (uint8_t)" << endl;
-    clog << "s  = " << s << " (int16_t)" << endl;
-    clog << "S  = " << S << " (uint16_t)" << endl;
-    clog << "l  = " << l << " (int32_t)" << endl;
-    clog << "L  = " << L << " (uint32_t)" << endl;
-    clog << "ll = " << ll << " (int64_t)" << endl;
-    clog << "LL = " << LL << " (uint64_t)" << endl;
-  }
-
-  return 0;   
-}
-
-// end of tutorial_pba_4.cpp
-
-
-

- - -

-The resulting PBA file is: -

127   1   9   1  84   1  99   1 117 255 253   1   6 255 240   1
- 32 255 128   1 127 254   0 252   2   0   8
-
- -where: -
    - -
  • 127 1 9 is the archive header. - -
  • 1 84 indicates 1 byte to -store the true boolean value (ASCII code -is 84). - -
  • 1 99 indicates 1 byte to -store the ASCII code of character 'c' (99). - -
  • 1 117 indicates 1 byte to -store the ASCII code of character 'u' -(117). - -
  • 255 253 corresponds to the special case for a -negative integer: - 255 is the binary coding for value -1 -which means that the integer is negative and needs 1 byte to be -stored, 253, i.e. the 8-bit encoding of the signed -decimal value -3. - -
  • 1 6 indicates 1 byte to store -value 6. - -
  • 255 240 corresponds again to a negative -integer: - 255 is the byte coding for value -1 -(negative integer encoded using 1 single byte) -and 240 is the 8-bit encoding of decimal -value -16. - -
  • the same scheme is used for all remaining values in the archive : - the verification is let as an exercise. - -
- -

- -

Note that this coding scheme optimizes the number of streamed -bytes. Particularly, it discards the leading zero-ed bytes -(MSB) of the binary encoding of any integer value in order to save -storage. Also we recall that the exact 0 value (zero -or false for a boolean data) is always encoded using a -unique 0 byte (zero optimization). Note this approach is also -used for floating point numbers. -

- - - -
-To top -
-
- - - -

Using PBA serialization with a memory buffer

- -

-In some case, we don't want to serialize some data in a file -(std::ofstream), but we simply plan to stream it in a memory buffer. -

- -

-The tutorial_pba_5.cpp sample -program makes use of a memory buffer implemented with a STL vector of -characters. The PBA is associated to this buffer thanks to a special -streaming interface mechanism provided by the Boost/Iostreams -library. With such technique one can stream serializable data in -some memory buffer in place of a file : -

- -
- The tutorial_pba_5.cpp source code Download -  Show/hide -
-/** tutorial_pba_5.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This sample program shows how to use a portable binary archive 
- * to store/load data in a memory buffer.
- *
- */
-
-#include <string>
-#include <vector>
-
-#include <boost/iostreams/stream.hpp>
-#include <boost/iostreams/device/back_inserter.hpp>
-#include <boost/iostreams/device/array.hpp>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-
-int main (void)
-{
-  using namespace std;
-
-  // The memory buffer is implemented using a STL vector :
-  typedef std::vector<char> buffer_type;
-  buffer_type buffer; 
-
-  {
-    // Some data to be stored :
-    bool    t = true;
-    char    c = 'c';
-    int16_t s = +16;
-    int32_t l = -128;
-    int64_t ll = +10000000000;
-    float   pi = 3.14159;
-    double  nan = numeric_limits<double>::quiet_NaN ();
-    string  hello = "World !";
-
-    buffer.reserve (1024); // pre-allocate some memory
-
-    // The output stream interface to the buffer :
-    boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type> > output_stream (buffer);
-
-    {    
-      // Create an output portable binary archive attached to the output file :
-      boost::archive::portable_binary_oarchive opba (output_stream);
-      
-      // Store (serialize) variables :
-      opba & t & c & s & l & ll & pi & nan & hello;
-    }
-
-  }
-
-  clog << "Buffer content is " << buffer.size () << " bytes : " << endl << "  ";
-  for (int i = 0; i < buffer.size (); ++i)
-    {
-      clog << (int) ((unsigned char) buffer[i]) << ' '; 
-      if ((i + 1) % 20 == 0) clog << endl << "  ";
-    }
-  clog << endl;
-
-  { 
-    // Some data to be loaded :
-    bool    t;
-    char    c;
-    int16_t s;
-    int32_t l;
-    int64_t ll;
-    float   pi;
-    double  nan;
-    string  hello;
-
-    // The input stream interface to the buffer :
-    boost::iostreams::stream<boost::iostreams::array_source> input_stream (&buffer[0], 
-									   buffer.size ());
-
-    {
-      // Create an input portable binary archive attached to the input file :
-      boost::archive::portable_binary_iarchive ipba (input_stream);
-      
-      // Load (de-serialize) variables :
-      ipba & t & c & s & l & ll & pi & nan & hello;
-    }
-
-    clog << "Loaded values from the buffer are: " << endl;
-    clog << "  t  = " << t << " (bool)" << endl;
-    clog << "  c  = '" << c << "' (char)" << endl;
-    clog << "  s  = " << s << " (int16_t)" << endl;
-    clog << "  l  = " << l << " (int32_t)" << endl;
-    clog << "  ll = " << ll << " (int64_t)" << endl;
-    clog << "  pi = " << pi << " (float)" << endl;
-    clog << "  nan = " << nan << " (double)" << endl;
-    clog << "  hello = \"" << hello << "\" (std::string)" << endl;
-  }
-
-  return 0;   
-}
-
-// end of tutorial_pba_5.cpp
-
-
-

- - -

-After the storing of data in the archive, the content of the buffer of -characters is printed: -

-Buffer content is 40 bytes : 
-  127 1 9 1 84 1 99 1 16 255 128 5 0 228 11 84 2 4 208 15 
-  73 64 8 255 255 255 255 255 255 255 127 1 7 87 111 114 108 100 32 33 
-  
-Loaded values from the buffer are: 
-  t  = 1 (bool)
-  c  = 'c' (char)
-  s  = 16 (int16_t)
-  l  = -128 (int32_t)
-  ll = 10000000000 (int64_t)
-  pi = 3.14159 (float)
-  nan = nan (double)
-  hello = "World !" (std::string)
-
-
-Again the PBA encoding scheme can be easily interpreted. This is let -as an exercise. -

- -

Extra:

-

-You may have a look on the tutorial_pba_6.cpp program that shows a possible — and -provocative — combined usage of the Boost/Serialization concepts, the -Boost/Iostreams facilities and the PBA; it enables the copy of an object -of a non-copyable class. -

- -
- The tutorial_pba_6.cpp source code Download -  Show/hide -
-/** tutorial_pba_6.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of
- * the Boost/Serialization library.
- *
- * This sample program shows how to use a portable binary archive
- * associated to a memory buffer to copy a non-copyable object.
- *
- */
-
-#include <iostream>
-#include <string>
-#include <sstream>
-#include <vector>
-
-#include <boost/utility.hpp>
-#include <boost/iostreams/stream.hpp>
-#include <boost/iostreams/device/back_inserter.hpp>
-#include <boost/iostreams/device/array.hpp>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-
-using namespace std;
-
-/* A foo noncopyable class */
-struct foo : boost::noncopyable
-{
-  uint32_t status;
-  double   value;
-  double   special;
-
-  string to_string () const
-  {
-    ostringstream sout;
-    sout << "foo={status=" << status << "; value=" << value  << "; special=" << special<< "}";
-    return sout.str();
-  }
-
-  template<class Archive>
-  void serialize (Archive & ar, const unsigned int version)
-  {
-    ar & status;
-    ar & value;
-    ar & special;
-    return;
-  }
-
-};
-
-// A templatized copy function for Boost/Serialization equipped classes.
-// Here we use PBAs associated to a memory buffer :
-template <class Serializable>
-void copy (const Serializable & source, Serializable & target)
-{
-  namespace io = boost::iostreams;
-  namespace ba = boost::archive;
-  if (&source == &target) return; // self-copy guard
-  typedef std::vector<char> buffer_type;
-  buffer_type buffer;
-  buffer.reserve (1024);
-  {
-    io::stream<io::back_insert_device<buffer_type> > output_stream (buffer);
-    ba::portable_binary_oarchive opba (output_stream);
-    opba & source;
-  }
-  {
-    io::stream<io::array_source> input_stream (&buffer[0], buffer.size ());
-    ba::portable_binary_iarchive ipba (input_stream);
-    ipba & target;
-  }
-  return;
-}
-
-int main (void)
-{
-  // Some instance of the 'foo' class :
-  foo dummy;
-  dummy.status = 1;
-  dummy.value = 3.14159;
-  dummy.special = numeric_limits<double>::quiet_NaN ();
-  clog << "dummy is : " << dummy.to_string () << endl;
-
-  // Another instance of the 'foo' class :
-  foo clone;
-
-  /* The following instruction is forbidden because foo 
-     inherits 'boost::noncopyable' :
-   
-   clone = dummy; // this ends in a compilation error.
-   
-   */
-
-  // Anyway, we can use this workaround :
-  copy (dummy, clone);
-  clog << "clone is : " << clone.to_string () << endl;
-
-  return 0;
-}
-
-// end of tutorial_pba_6.cpp
-
-
-

- - -

Remark : if a class has been made non-copyable at design, -it is likely for a good reason; so it is not recommended to workaround -this trait using such a trick, unless you know what you are doing -and all the consequences ! -

- - - -
-To top -
-
- - - -

An alternative to PBA using text or XML archives made portable

- -

-In some circonstances, it may be useful to use the Boost text -and XML archives in somewhat portable way. For example, we may -want to benefit of the XML archive's human-friendly format for -debugging purpose before to switch to the PBA for production runs. -However, the text and XML archives provided by the Boost -serialization library are not strictly portable, particularly because -they does not support the serialization of non-finite floating point -numbers. This is because the serialization of floating point numbers -depends on some formatting features of standard I/O streams. See the -tutorial_pba_7.cpp sample program below : -

- -
- The tutorial_pba_7.cpp source code Download -  Show/hide -
-/** tutorial_pba_7.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This example shows how the default behaviour of standard 
- * I/O streams does not support the read/write operations of 
- * non-finite floating point values in a portable way.
- *
- */
-
-#include <string>
-#include <iostream>
-#include <sstream>
-#include <limits>
-
-using namespace std;
-
-int main (void)
-{
-  {
-    float x = numeric_limits<float>::infinity ();
-    double y = numeric_limits<double>::quiet_NaN ();
-    cout.precision (8);
-    cout << "x = " << x << endl;
-    cout.precision (16);
-    cout << "y = " << y << endl;
-  }
-
-  {
-    string input ("inf nan");
-    istringstream iss (input);
-    float x; 
-    double y;
-    iss >> x >> y;
-    if (! iss)
-      {
-	cerr << "Cannot read 'x' or 'y' : non finite values are not supported !" << endl;
-      }
-  }
-  return 0;
-}
-
-// end of tutorial_pba_7.cpp
-
-
-

- - -

-Depending on the system, one can get some various representation -respectively for the infinity and NaN values : -

    -
  • typically on Windows : -
    -1.#INF
-
    -and -
    --1.#IND
-
    -
  • and on Linux : -
    -inf
-
    -and -
    -nan
-
    - -
-Usually one can print such non finite values in an output stream -(using such a non portable representation), but parsing it from an -input stream fails ! -

- -

-Hopefully this issue can be solved by configuring the I/O streams with -some special locale features provided by Boost -(see this -link). -

- -

-The tutorial_pba_8.cpp program -shows how this can be achieved through the use of special resources -from the boost/archive/codecvt_null.hpp and -boost/math/special_functions/nonfinite_num_facets.hpp headers : -

- -
- The tutorial_pba_8.cpp source code Download -  Show/hide -
-/** tutorial_pba_8.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This example shows how to store some variables
- * of basic types (bool, integer, floating point numbers, STL string) 
- * using the text or XML archive format associated to a 
- * standard output file stream supporting portable non-finite
- * floating point values.
- *
- */
-
-#include <string>
-#include <fstream>
-#include <limits>
-#include <locale>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/xml_oarchive.hpp>
-#include <boost/archive/text_oarchive.hpp>
-#include <boost/serialization/nvp.hpp>
-#include <boost/archive/codecvt_null.hpp>
-#include <boost/math/special_functions/nonfinite_num_facets.hpp>
-
-using namespace std;
-
-void do_text_out (void)
-{
-  // The name for the example data text file :  
-  string filename = "pba_8.txt"; 
-
-  // Some variables of various primitive types :
-  bool        b         = true;
-  char        c         = 'B';
-  uint32_t    answer    = 42;
-  float       value     = numeric_limits<float>::infinity ();
-  double      precision = numeric_limits<double>::quiet_NaN ();
-  string      question  = "What makes you think she's a witch?";
-  
-  // Open an output file stream :
-  ofstream fout (filename.c_str ());
-
-  // Prepare the output file stream for inf/NaN support :
-  locale default_locale (locale::classic (),
-			 new boost::archive::codecvt_null<char>);
-  locale infnan_locale (default_locale,
-			new boost::math::nonfinite_num_put<char>);
-  fout.imbue (infnan_locale);
-  
-  {
-    // Create an output text archive attached to the output file :
-    boost::archive::text_oarchive ota (fout, boost::archive::no_codecvt);
-    
-    // Store (serializing) variables :
-    ota & b & c & answer & value & precision & question;
-  }
-
-  return;   
-}
-
-void do_xml_out (void)
-{
-  // The name for the example data XML file :  
-  string filename = "pba_8.xml"; 
-
-  // Some variables of various primitive types :
-  bool        b         = true;
-  char        c         = 'B';
-  uint32_t    answer    = 42;
-  float       value     = numeric_limits<float>::infinity ();
-  double      precision = numeric_limits<double>::quiet_NaN ();
-  string      question  = "What makes you think she's a witch?";
-  
-  // Open an output file stream :
-  ofstream fout (filename.c_str ());
-
-  // Prepare the output file stream for inf/NaN support :
-  locale default_locale (locale::classic (),
-			 new boost::archive::codecvt_null<char>);
-  locale infnan_locale (default_locale,
-			new boost::math::nonfinite_num_put<char>);
-  fout.imbue (infnan_locale);
-   
-  {
-    // Create an output text archive attached to the output file :
-    boost::archive::xml_oarchive oxa (fout, boost::archive::no_codecvt);
-    
-    // Store (serializing) variables :
-    oxa & BOOST_SERIALIZATION_NVP(b)
-      & BOOST_SERIALIZATION_NVP(c)
-      & BOOST_SERIALIZATION_NVP(answer)
-      & BOOST_SERIALIZATION_NVP(value)
-      & BOOST_SERIALIZATION_NVP(precision) 
-      & BOOST_SERIALIZATION_NVP(question);
-  }
-
-  return;   
-}
-
-int main (void)
-{
-  do_text_out ();
-  do_xml_out ();
-  return 0;
-}
-
-// end of tutorial_pba_8.cpp
-
-
-

- - -

-The program creates two output files : -

    -
  • pba_8.txt stores a text archive with - non finite floating point values : -
    -22 serialization::archive 9 1 66 42 inf nan 35 What makes you think she's a witch?
-
-
    -
  • pba_8.xml which stored the equivalent content - using the XML archive format : - -
    -<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
-<!DOCTYPE boost_serialization>
-<boost_serialization signature="serialization::archive" version="9">
-<b>1</b>
-<c>66</c>
-<answer>42</answer>
-<value>inf</value>
-<precision>nan</precision>
-<question>What makes you think she&apos;s a witch?</question>
-</boost_serialization>
-
-
    - - -
-

- -

-The tutorial_pba_9.cpp program -deserializes the data from the text and XML archive files (respectively -pba_8.txt and pba_8.xml) and prints the restored variables : -

-Loaded values from text archive are: 
-  b         = 1
-  c         = 'B'
-  answer    = 42
-  value     = inf
-  precision = nan
-  question  = "What makes you think she's a witch?"
-Loaded values from XML archive are: 
-  b         = 1
-  c         = 'B'
-  answer    = 42
-  value     = inf
-  precision = nan
-  question  = "What makes you think she's a witch?"
-
-
-

- -

-

- The tutorial_pba_9.cpp source code Download -  Show/hide -
-/** tutorial_pba_9.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This example shows how to load some variables of basic 
- * types (bool, char, integer, floating point numbers, STL string) 
- * using the text or XML archive format associated to a 
- * standard file input stream supporting portable non-finite
- * floating point values.
- *
- */
-
-#include <string>
-#include <fstream>
-#include <limits>
-#include <locale>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/xml_iarchive.hpp>
-#include <boost/archive/text_iarchive.hpp>
-#include <boost/serialization/nvp.hpp>
-#include <boost/scoped_ptr.hpp>
-#include <boost/archive/codecvt_null.hpp>
-#include <boost/math/special_functions/nonfinite_num_facets.hpp>
-
-using namespace std;
-
-void do_text_in (void)
-{
-  // The name for the example data text file :  
-  string filename = "pba_8.txt"; 
-  // Some variables of various primitive types :
-  bool        b;
-  char        c;
-  uint32_t    answer;
-  float       value;
-  double      precision;
-  string      question;
-  
-  // Open an input file stream :
-  ifstream fin (filename.c_str ());
-
-  // Prepare the input file stream for inf/NaN support :
-  locale default_locale (locale::classic (),
-			 new boost::archive::codecvt_null<char>);
-  locale infnan_locale (default_locale,
-			new boost::math::nonfinite_num_get<char>);
-  fin.imbue (infnan_locale);
- 
-  {
-    // Create an input text archive attached to the input file :
-    boost::archive::text_iarchive ita (fin, boost::archive::no_codecvt);
-    
-    // Store (serializing) variables :
-    ita & b & c & answer & value & precision & question;
-  }
-
-  clog << "Loaded values from text archive are: " << endl;
-  clog << "  b         = " << b << endl;
-  clog << "  c         = '" << c << "'" <<  endl;
-  clog << "  answer    = " << answer << endl;
-  clog << "  value     = " << value << endl;
-  clog << "  precision = " << precision << endl;
-  clog << "  question  = \"" << question << "\"" << endl;
-
-  return;   
-}
-
-void do_xml_in (void)
-{
-  // The name for the example data text file :  
-  string filename = "pba_8.xml"; 
-
-  // Some variables of various primitive types :
-  bool        b;
-  char        c;
-  uint32_t    answer;
-  float       value;
-  double      precision;
-  string      question;
-  
-  // Open an input file stream :
-  ifstream fin (filename.c_str ());
-
-  // Prepare the input file stream for inf/NaN support :
-  locale default_locale (locale::classic (),
-			 new boost::archive::codecvt_null<char>);
-  locale infnan_locale (default_locale,
-			new boost::math::nonfinite_num_get<char>);
-  fin.imbue (infnan_locale);
-
-  {
-    // Create an output text archive attached to the output file :
-    boost::archive::xml_iarchive ixa (fin, boost::archive::no_codecvt);
-    
-    // Store (serializing) variables :
-    ixa & BOOST_SERIALIZATION_NVP(b)
-      & BOOST_SERIALIZATION_NVP(c)
-      & BOOST_SERIALIZATION_NVP(answer)
-      & BOOST_SERIALIZATION_NVP(value)
-      & BOOST_SERIALIZATION_NVP(precision) 
-      & BOOST_SERIALIZATION_NVP(question);
-  }
-
-  clog << "Loaded values from XML archive are: " << endl;
-  clog << "  b         = " << b << endl;
-  clog << "  c         = '" << c << "'" <<  endl;
-  clog << "  answer    = " << answer << endl;
-  clog << "  value     = " << value << endl;
-  clog << "  precision = " << precision << endl;
-  clog << "  question  = \"" << question << "\"" << endl;
-
-  return;   
-}
-
-int main (void)
-{
-  do_text_in ();
-  do_xml_in ();
-  return 0;
-}
-
-// end of tutorial_pba_9.cpp
-
-
-

- -

- - - -
-To top -
-
- - - -

Using PBA serialization associated with on-the-fly (de)compressed file streams

- -

-The tutorial_pba_10.cpp program -illustrates how to serialize, then deserialize, a class from a PBA associated -to a GZIP compressed file stream, thanks to a technique -provided by the Boost/Iostreams library. The class contains a large -STL vector of double precision floating point numbers with arbitrary values: -

- -
- The tutorial_pba_10.cpp source code Download -  Show/hide -
-/** tutorial_pba_10.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This example shows how use PBAs combined with on-the-fly 
- * compressed I/O streams.
- *
- */
-
-#include <string>
-#include <fstream>
-#include <limits>
-#include <vector>
-
-#include <boost/cstdint.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-#include <boost/iostreams/filtering_stream.hpp>
-#include <boost/iostreams/filter/gzip.hpp>
-#include <boost/serialization/access.hpp>
-#include <boost/serialization/vector.hpp>
-
-using namespace std;
-
-class data_type
-{
-private:
-  friend class boost::serialization::access;
-  template<class Archive>
-  void serialize (Archive & ar, const unsigned int version);
-public:
-  void print (ostream & out, const string & title) const;
-public:
-  vector<double> values;
-  data_type ();
-};
-
-data_type::data_type () : values ()
-{
-  return;
-}
-
-void data_type::print (ostream & out, const string & title) const
-{
-  out << endl;
-  out << title << " :" << endl;
-  for (int i = 0; i < this->values.size (); ++i)
-    {
-      out.precision (16);
-      out.width (18);
-      out << this->values [i] << ' ' ;
-      if ((i%4) == 3) clog << endl;
-    }
-  out << endl;
-  return;
-}
-  
-template<class Archive>
-void data_type::serialize (Archive & ar, const unsigned int version)
-{
-  ar & values;
-  return;
-}
-
-void do_gzipped_out (void)
-{
-  // The name for the output data file :  
-  string filename = "pba_10.data.gz"; 
-
-  // A data structure to be stored :
-  data_type my_data;
-
-  // Fill the vector with arbitrary (possibly non-finite) values :
-  size_t dim = 1000;
-  my_data.values.reserve (dim);
-  for (int i = 0; i < dim; ++i)
-    {      
-      double val = (i + 1) * (1.0 + 3 * numeric_limits<double>::epsilon ());
-      if (i == 4) val = numeric_limits<double>::quiet_NaN ();
-      if (i == 23) val = numeric_limits<double>::infinity ();
-      if (i == 73) val = -numeric_limits<double>::infinity ();
-      if (i == 90) val = 0.0;
-      my_data.values.push_back (val);
-    }
-
-  // Print:
-  my_data.print (clog, "Stored data");
-
-  // Create an output filtering stream :
-  boost::iostreams::filtering_ostream zout;
-  zout.push (boost::iostreams::gzip_compressor ());
-  
-  // Open an output file stream in binary mode :
-  ofstream fout (filename.c_str (), ios_base::binary);
-  zout.push (fout);
-
-  // Save to PBA :
-  {
-    // Create an output portable binary archive attached to the output file :
-    boost::archive::portable_binary_oarchive opba (zout);
-    
-    // Store (serializing) the data :
-    opba & my_data;
-  }
-
-  // Clean termination of the streams :
-  zout.flush ();
-  zout.reset ();
-
-  return;   
-}
-
-void do_gzipped_in (void)
-{
-  // The name for the input data file :  
-  string filename = "pba_10.data.gz"; 
-
-  // A data structure to be loaded :
-  data_type my_data;
-
-  // Create an input filtering stream :
-  boost::iostreams::filtering_istream zin;
-  zin.push (boost::iostreams::gzip_decompressor ());
-  
-  // Open an input file stream in binary mode :
-  ifstream fin (filename.c_str (), ios_base::binary);
-  zin.push (fin);
-
-  // Load from PBA :
-  {
-    // Create an input portable binary archive attached to the input file :
-    boost::archive::portable_binary_iarchive ipba (zin);
-    
-    // Load (deserializing) the data :
-    ipba & my_data;
-  }
-
-  // Print:
-  my_data.print (clog, "Loaded data");
-
-  return;   
-}
-
-int main (void)
-{
-  do_gzipped_out (); 
-  do_gzipped_in ();
-  return 0;
-}
-
-// end of tutorial_pba_10.cpp
-
-
-

- - -

The resulting compressed pba_10.data.gz file contains 1,574 bytes. -This has to be compared with the size of the plain (uncompressed) -binary archive which equals 9,001 bytes: - -

127   1   9   0   0   2 232   3   0   8   3   0   0   0   0   0
-240  63   8   3   0   0   0   0   0   0  64   8   4   0   0   0
-  0   0   8  64   8   3   0   0   0   0   0  16  64   8 255 255
-255 255 255 255 255 127   8   4   0   0   0   0   0  24  64   8
-...
-
- -which can be interpreted as : -
    -
  • 3 bytes for the usual archive header : 127 1 -9, -
  • 2 zero-optimized bytes to store the class ID - and the class version :0 0, -
  • 3 bytes to store the size of the vector (1000) using the PBA -representation of integers (2 232 3), -
  • 1 zero-optimized byte to store the class version of the -objects stored in the STL vector (here -the primitive double type) : 0, -
  • 999 non-zero double precision floating point values (including non-finite values) each using 9 bytes (1 byte for -the size and 8 bytes for the content) : 8 ? ? ? ? ? -? ? ?, -
  • 1 zero floating point value that uses only one zero-optimized byte : 0. -
-Thus one here achieves a very interesting compression level. -

-

-It is also possible to use BZIP2 in a similar fashion -(using ressources from the boost/iostreams/filter/bzip2.hpp header -in place of boost/iostreams/filter/gzip.hpp). -

- - - -
-To top -
-
- - -

A simple PBA versus text archive benchmark test

- -

-The tutorial_pba_11.cpp program -runs a benchmark test in the aim to compare the relative fastness of PBA and text archives -both for read and write operations. It stores then loads a vector of many (107) -random double values and prints the associated (de)serialization time for both kinds of archives: -

- -
- The tutorial_pba_11.cpp source code Download -  Show/hide -
-/** tutorial_pba_11.cpp
- *
- * (C) Copyright 2011 François Mauger, Christian Pfligersdorffer
- *
- * Use, modification and distribution is subject to the Boost Software
- * License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- *
- */
-
-/**
- * The intent of this program is to serve as a tutorial for
- * users of the portable binary archive in the framework of 
- * the Boost/Serialization library. 
- *
- * This example program compares the times needed to serialize
- * and deserialize some large amount of data using PBA and 
- * text archives.
- *
- */
-
-#include <string>
-#include <fstream>
-#include <vector>
-
-#include <boost/archive/portable_binary_oarchive.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-#include <boost/archive/text_oarchive.hpp>
-#include <boost/archive/text_iarchive.hpp>
-#include <boost/serialization/access.hpp>
-#include <boost/serialization/vector.hpp>
-#include <boost/random/mersenne_twister.hpp>
-#include <boost/random/uniform_real_distribution.hpp>
-#include <boost/timer.hpp>
-
-using namespace std;
-
-class data_type
-{
-private:
-  friend class boost::serialization::access;
-  template<class Archive>
-  void serialize (Archive & ar, const unsigned int version);
-public:
-  void print (ostream & out, const string & title) const;
-public:
-  vector<double> values;
-  data_type ();
-};
-
-data_type::data_type () : values ()
-{
-  return;
-}
-
-void data_type::print (ostream & out, const string & title) const
-{
-  out << endl;
-  out << title << " :" << endl;
-  bool skip = false;
-  for (int i = 0; i < this->values.size (); ++i)
-    {
-      if ((i >= 12) && (i < (int) this->values.size () - 8)) 
-	{
-	  if (! skip) out << " ..." << endl;
-	  skip = true;
-	  continue;
-	}
-      out.precision (16);
-      out.width (18);
-      out << this->values [i] << ' ' ;
-      if ((i%4) == 3) clog << endl;
-    }
-  out << endl;
-  return;
-}
-  
-template<class Archive>
-void data_type::serialize (Archive & ar, const unsigned int version)
-{
-  ar & values;
-  return;
-}
-
-double do_pba_out (const data_type & a_data)
-{
-  string filename = "pba_11.data"; 
-  ofstream fout (filename.c_str (), ios_base::binary);
-  boost::timer io_timer;    
-  {
-    boost::archive::portable_binary_oarchive opba (fout);
-    opba & a_data;
-  }
-  return io_timer.elapsed ();
-}
-
-double do_pba_in (data_type & a_data)
-{
-  string filename = "pba_11.data"; 
-  ifstream fin (filename.c_str (), ios_base::binary);
-  boost::timer io_timer;    
-  {
-    boost::archive::portable_binary_iarchive ipba (fin);
-    ipba & a_data;
-  }
-  return io_timer.elapsed ();
-}
-
-double do_text_out (const data_type & a_data)
-{
-  string filename = "pba_11.txt"; 
-  ofstream fout (filename.c_str ());
-  boost::timer io_timer;    
-  {
-    boost::archive::text_oarchive ota (fout);
-    ota & a_data;
-  }
-  return io_timer.elapsed ();
-}
-
-double do_text_in (data_type & a_data)
-{
-  string filename = "pba_11.txt"; 
-  ifstream fin (filename.c_str ());
-  boost::timer io_timer;    
-  {
-    boost::archive::text_iarchive ita (fin);
-    ita & a_data;
-  }
-  return io_timer.elapsed ();
-}
-
-int main (void)
-{
-  double elapsed_time_pba_out; 
-  double elapsed_time_text_out; 
-  double elapsed_time_pba_in; 
-  double elapsed_time_text_in; 
-  data_type my_data; // A data structure to be stored then loaded.
-
-  {
-    // Fill the vector with random values :
-    size_t dim = 10000000;
-    my_data.values.reserve (dim);
-    boost::random::mt19937 rng;
-    boost::random::uniform_real_distribution<> flat (0.0, 100.0);
-    for (int i = 0; i < dim; ++i)
-      {      
-	double val = flat (rng);
-	my_data.values.push_back (val);
-      }
-    my_data.print (clog, "Stored data in PBA and text archive");
-  }
-
-  {
-    // Store in PBA :
-    elapsed_time_pba_out = do_pba_out (my_data);
-  }
-
-  {
-    // Store in text archive :
-    elapsed_time_text_out = do_text_out (my_data);
-  }     
-
-  {
-    my_data.values.clear ();
-    // Load from PBA :
-    elapsed_time_pba_in = do_pba_in (my_data);
-    my_data.print (clog, "Loaded data from PBA");
-  }
-
-  {
-    my_data.values.clear ();
-    // Load from text archive :
-    elapsed_time_text_in = do_text_in (my_data);
-    my_data.print (clog, "Loaded data from text archive");
-  }
-  
-  clog << "PBA  store I/O elapsed time : " << elapsed_time_pba_out  << " (second)" << endl;
-  clog << "Text store I/O elapsed time : " << elapsed_time_text_out << " (second)" << endl;  
-  clog << "PBA  load  I/O elapsed time : " << elapsed_time_pba_in   << " (second)" << endl;
-  clog << "Text load  I/O elapsed time : " << elapsed_time_text_in  << " (second)" << endl;
-
-  return 0;
-}
-
-// end of tutorial_pba_11.cpp
-
-
-

- - -

On a 1.60 GHz processor running gcc 4.5.2 on Linux 2.6.38, the result is the following: -

PBA  store I/O elapsed time : 1.86 (second)
-Text store I/O elapsed time : 22.66 (second)
-PBA  load  I/O elapsed time : 1.53 (second)
-Text load  I/O elapsed time : 19.71 (second)
-
-It this simple case, the use of portable binary archives is faster by at least a factor 10 -compared to the traditional Boost text archives. This is a -significant saving in time. These performances are highly desirable in the typical framework -of scientific/computing activities where large amounts of data are accessed through files. -The PBA concept is thus a valuable candidate for such applications. -

- -

-One can also consider the sizes of the resulting archive files: -

    -
  • pba_11.data (PBA) : 90,000,010 bytes -
  • pba_11.txt (text archive) : 189,000,040 bytes -
-The PBA allows to save a typical factor 2 in storage space compared to text archive. -This is another strong argument for using PBA. -

- - - -
-To top -
-
- - - -
-

Revised 2011-11-07 -

© Copyright François Mauger, -Christian Pfligersdorffer 2011.
-Distributed under the Boost Software License, Version 1.0.
-(See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -

- - - - - From d3d5af1954b921fc47c20427b7dbdfb879c7727f Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Fri, 1 May 2020 20:39:39 +0200 Subject: [PATCH 13/15] bug fixes of new wallet sync protocol --- src/currency_core/blockchain_storage.cpp | 35 +++++++++++++++++++----- src/currency_core/blockchain_storage.h | 3 +- src/wallet/wallet2.cpp | 10 +++---- src/wallet/wallet2.h | 4 +-- src/wallet/wallet_chain_shortener.cpp | 2 +- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 696e7422..e7632e63 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -70,7 +70,6 @@ using namespace currency; #endif #define BLOCK_POS_STRICT_SEQUENCE_LIMIT 20 -#define BLOCKCHAIN_FIRST_BLOCK_TIMESTAMP 1557342384 DISABLE_VS_WARNINGS(4267) @@ -111,7 +110,8 @@ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m m_is_reorganize_in_process(false), m_deinit_is_done(false), m_cached_next_pow_difficulty(0), - m_cached_next_pos_difficulty(0) + m_cached_next_pos_difficulty(0), + m_blockchain_launch_timestamp(0) { @@ -2941,16 +2941,31 @@ bool blockchain_storage::find_blockchain_supplement(const std::list 2) + { + m_blockchain_launch_timestamp = m_db_blocks[1]->bl.timestamp; + } + return m_blockchain_launch_timestamp; +} +//------------------------------------------------------------------ bool blockchain_storage::get_est_height_from_date(uint64_t date, uint64_t& res_h)const { CRITICAL_REGION_LOCAL(m_read_lock); #define GET_EST_HEIGHT_FROM_DATE_THRESHOLD 1440 - if (date < BLOCKCHAIN_FIRST_BLOCK_TIMESTAMP) - return false; + if (date < get_blockchain_launch_timestamp()) + { + res_h = 0; + return true; + } - uint64_t calculated_estimated_height = (date - BLOCKCHAIN_FIRST_BLOCK_TIMESTAMP) / DIFFICULTY_TOTAL_TARGET; + uint64_t calculated_estimated_height = (date - get_blockchain_launch_timestamp()) / DIFFICULTY_TOTAL_TARGET; if (date > m_db_blocks[m_db_blocks.size() - 1]->bl.timestamp) { @@ -2965,7 +2980,7 @@ bool blockchain_storage::get_est_height_from_date(uint64_t date, uint64_t& res_h else { //likely impossible, but just in case - res_h = 1; + res_h = 0; } } @@ -2987,7 +3002,13 @@ bool blockchain_storage::get_est_height_from_date(uint64_t date, uint64_t& res_h if (ts > high_boundary) { //we moved too much forward - calculated_estimated_height -= (ts - aim) / DIFFICULTY_TOTAL_TARGET; + uint64_t offset = (ts - aim) / DIFFICULTY_TOTAL_TARGET; + if (offset > calculated_estimated_height) + { + res_h = 0; + break; + } + calculated_estimated_height -= offset; } else if (ts < low_boundary) { diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 02f03839..7802a8bc 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -535,7 +535,7 @@ namespace currency mutable uint64_t m_current_fee_median_effective_index; bool m_is_reorganize_in_process; mutable std::atomic m_deinit_is_done; - + mutable uint64_t m_blockchain_launch_timestamp; bool init_tx_fee_median(); bool update_tx_fee_median(); @@ -616,6 +616,7 @@ namespace currency void pop_block_from_per_block_increments(uint64_t height_); void calculate_local_gindex_lookup_table_for_height(uint64_t split_height, std::map& increments) const; void do_erase_altblock(alt_chain_container::iterator it); + uint64_t get_blockchain_launch_timestamp()const; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index dbf724a0..5e89ae22 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1156,7 +1156,7 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response res = AUTO_VAL_INIT(res); req.minimum_height = get_wallet_minimum_height(); - get_short_chain_history(req.block_ids); + m_chain.get_short_chain_history(req.block_ids); bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); if (!r) throw error::no_connection_to_daemon(LOCATION_STR, "getblocks.bin"); @@ -1179,7 +1179,7 @@ void wallet2::pull_blocks(size_t& blocks_added, std::atomic& stop) reset_all(); m_chain.set_genesis(new_genesis_id); WLT_LOG_MAGENTA("New genesis set for wallet: " << new_genesis_id, LOG_LEVEL_0); - get_short_chain_history(req.block_ids); + m_chain.get_short_chain_history(req.block_ids); //req.block_ids.push_back(new_genesis_id); bool r = m_core_proxy->call_COMMAND_RPC_GET_BLOCKS_DIRECT(req, res); THROW_IF_TRUE_WALLET_EX(!r, error::no_connection_to_daemon, "getblocks.bin"); @@ -1203,7 +1203,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& res) { size_t current_index = res.start_height; - + bool been_matched_block = false; if (res.start_height == 0 && get_blockchain_current_size() == 1 && !res.blocks.empty()) { const currency::block& genesis = res.blocks.front().block_ptr->bl; @@ -1211,10 +1211,10 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop process_genesis_if_needed(genesis); res.blocks.pop_front(); ++current_index; + been_matched_block = true; } uint64_t last_matched_index = 0; - bool been_matched_block = false; for(const auto& bl_entry: res.blocks) { if (stop) @@ -1249,7 +1249,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop bool block_found = false; bool block_matched = false; bool full_reset_needed = false; - check_if_block_matched(height, bl_id, block_found, block_matched, full_reset_needed); + m_chain.check_if_block_matched(height, bl_id, block_found, block_matched, full_reset_needed); if (block_found && block_matched) { //block matched in that number diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 6aaab37e..a9e214f3 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -900,8 +900,8 @@ private: void check_and_throw_if_self_directed_tx_with_payment_id_requested(const construct_tx_param& ctp); void push_new_block_id(const crypto::hash& id, uint64_t height); bool lookup_item_around(uint64_t i, std::pair& result); - void get_short_chain_history(std::list& ids); - void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed); + //void get_short_chain_history(std::list& ids); + //void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed); uint64_t detach_from_block_ids(uint64_t height); uint64_t get_wallet_minimum_height(); diff --git a/src/wallet/wallet_chain_shortener.cpp b/src/wallet/wallet_chain_shortener.cpp index e8778b66..3d3bba28 100644 --- a/src/wallet/wallet_chain_shortener.cpp +++ b/src/wallet/wallet_chain_shortener.cpp @@ -12,7 +12,7 @@ #define WALLET_EVERY_100_BLOCKS_SIZE 144 #define WALLET_EVERY_1000_BLOCKS_SIZE 144 -void exception_handler(){} +static void exception_handler(){} wallet_chain_shortener::wallet_chain_shortener(): m_genesis(currency::gdefault_genesis) From 2ca5913ad565c1574a3dbe2d354009c61b6cfecc Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 2 May 2020 01:27:55 +0200 Subject: [PATCH 14/15] fixed core tests --- tests/core_tests/wallet_test_core_proxy.cpp | 7 +++++++ tests/core_tests/wallet_test_core_proxy.h | 1 + 2 files changed, 8 insertions(+) diff --git a/tests/core_tests/wallet_test_core_proxy.cpp b/tests/core_tests/wallet_test_core_proxy.cpp index a3ddb4d0..75dac59e 100644 --- a/tests/core_tests/wallet_test_core_proxy.cpp +++ b/tests/core_tests/wallet_test_core_proxy.cpp @@ -93,6 +93,13 @@ bool wallet_test_core_proxy::call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency:: } +bool wallet_test_core_proxy::call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(const currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& rqt, currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& rsp) +{ + rsp.h = 0; + rsp.status = CORE_RPC_STATUS_OK; + return true; +} + bool wallet_test_core_proxy::call_COMMAND_RPC_GET_INFO(const currency::COMMAND_RPC_GET_INFO::request& rqt, currency::COMMAND_RPC_GET_INFO::response& rsp) { rsp.synchronized_connections_count = 1; diff --git a/tests/core_tests/wallet_test_core_proxy.h b/tests/core_tests/wallet_test_core_proxy.h index 4a531a19..322a7bb3 100644 --- a/tests/core_tests/wallet_test_core_proxy.h +++ b/tests/core_tests/wallet_test_core_proxy.h @@ -19,6 +19,7 @@ struct wallet_test_core_proxy : public tools::i_core_proxy virtual bool call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(const currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& rqt, currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& rsp) override; virtual bool call_COMMAND_RPC_GET_BLOCKS_FAST(const currency::COMMAND_RPC_GET_BLOCKS_FAST::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FAST::response& rsp) override; virtual bool call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& rsp) override; + virtual bool call_COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE(const currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& rqt, currency::COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& rsp) override; virtual bool call_COMMAND_RPC_GET_INFO(const currency::COMMAND_RPC_GET_INFO::request& rqt, currency::COMMAND_RPC_GET_INFO::response& rsp) override; virtual bool call_COMMAND_RPC_SEND_RAW_TX(const currency::COMMAND_RPC_SEND_RAW_TX::request& rqt, currency::COMMAND_RPC_SEND_RAW_TX::response& rsp) override; virtual bool call_COMMAND_RPC_GET_TX_POOL(const currency::COMMAND_RPC_GET_TX_POOL::request& rqt, currency::COMMAND_RPC_GET_TX_POOL::response& rsp) override; From 3136e311e04d1f9e0ca909cf0bdeb08a149e1698 Mon Sep 17 00:00:00 2001 From: cryptozoidberg Date: Sat, 2 May 2020 22:53:52 +0200 Subject: [PATCH 15/15] all tests got fixed --- src/wallet/wallet2.cpp | 27 +++++++------- src/wallet/wallet2.h | 2 +- src/wallet/wallet_chain_shortener.cpp | 41 +++++++++++++++++---- src/wallet/wallet_chain_shortener.h | 2 +- tests/core_tests/chaingen.cpp | 4 ++ tests/core_tests/wallet_test_core_proxy.cpp | 26 +++++++++++-- 6 files changed, 75 insertions(+), 27 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 5e89ae22..26a0e24d 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1263,6 +1263,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop if (full_reset_needed) { last_matched_index = 0; + been_matched_block = true; } else { @@ -1271,7 +1272,7 @@ void wallet2::handle_pulled_blocks(size_t& blocks_added, std::atomic& stop } //TODO: take into account date of wallet creation //reorganize - detach_blockchain(last_matched_index); + detach_blockchain(last_matched_index+1); process_new_blockchain_entry(bl, bl_entry, bl_id, height); ++blocks_added; } @@ -1777,23 +1778,23 @@ bool wallet2::refresh(size_t & blocks_fetched, bool& received_money, bool& ok, s } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::detach_from_block_ids(uint64_t height) +uint64_t wallet2::detach_from_block_ids(uint64_t including_height) { //calculate number of erased blocks - uint64_t blocks_detached = (get_blockchain_current_size() -1 ) - height; + uint64_t blocks_detached = get_blockchain_current_size() - including_height; //id at height should be kept, the rest - erased - m_chain.detach(height); + m_chain.detach(including_height); return blocks_detached; } //---------------------------------------------------------------------------------------------------- -void wallet2::detach_blockchain(uint64_t height) +void wallet2::detach_blockchain(uint64_t including_height) { - WLT_LOG_L0("Detaching blockchain on height " << height); + WLT_LOG_L0("Detaching blockchain on height " << including_height); size_t transfers_detached = 0; // rollback incoming transfers from detaching subchain { - auto it = std::find_if(m_transfers.begin(), m_transfers.end(), [&](const transfer_details& td){return td.m_ptx_wallet_info->m_block_height >= height; }); + auto it = std::find_if(m_transfers.begin(), m_transfers.end(), [&](const transfer_details& td){return td.m_ptx_wallet_info->m_block_height >= including_height; }); if (it != m_transfers.end()) { size_t i_start = it - m_transfers.begin(); @@ -1802,7 +1803,7 @@ void wallet2::detach_blockchain(uint64_t height) { auto it_ki = m_key_images.find(m_transfers[i].m_key_image); WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(it_ki != m_key_images.end(), "key image " << m_transfers[i].m_key_image << " not found"); - WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(m_transfers[i].m_ptx_wallet_info->m_block_height >= height, "transfer #" << i << " block height is less than " << height); + WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(m_transfers[i].m_ptx_wallet_info->m_block_height >= including_height, "transfer #" << i << " block height is less than " << including_height); m_key_images.erase(it_ki); ++transfers_detached; } @@ -1810,7 +1811,7 @@ void wallet2::detach_blockchain(uint64_t height) } } - size_t blocks_detached = detach_from_block_ids(height); + size_t blocks_detached = detach_from_block_ids(including_height); //rollback spends // do not clear spent flag in spent transfers as corresponding txs are most likely in the pool @@ -1818,7 +1819,7 @@ void wallet2::detach_blockchain(uint64_t height) for (size_t i = 0, sz = m_transfers.size(); i < sz; ++i) { auto& tr = m_transfers[i]; - if (tr.m_spent_height >= height) + if (tr.m_spent_height >= including_height) { WLT_LOG_BLUE("Transfer [" << i << "] spent height: " << tr.m_spent_height << " -> 0, reason: detaching blockchain", LOG_LEVEL_1); tr.m_spent_height = 0; @@ -1829,7 +1830,7 @@ void wallet2::detach_blockchain(uint64_t height) auto tr_hist_it = m_transfer_history.rend(); for (auto it = m_transfer_history.rbegin(); it != m_transfer_history.rend(); it++) { - if (it->height < height) + if (it->height < including_height) break; tr_hist_it = it; // note that tr_hist_it->height >= height } @@ -1859,13 +1860,13 @@ void wallet2::detach_blockchain(uint64_t height) //rollback payments for (auto it = m_payments.begin(); it != m_payments.end(); ) { - if(height <= it->second.m_block_height) + if(including_height <= it->second.m_block_height) it = m_payments.erase(it); else ++it; } - WLT_LOG_L0("Detached blockchain on height " << height << ", transfers detached " << transfers_detached << ", blocks detached " << blocks_detached); + WLT_LOG_L0("Detached blockchain on height " << including_height << ", transfers detached " << transfers_detached << ", blocks detached " << blocks_detached); } //---------------------------------------------------------------------------------------------------- bool wallet2::deinit() diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index a9e214f3..2e9a55e3 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -785,7 +785,7 @@ private: void remove_transfer_from_expiration_list(uint64_t transfer_index); void load_keys(const std::string& keys_file_name, const std::string& password); void process_new_transaction(const currency::transaction& tx, uint64_t height, const currency::block& b); - void detach_blockchain(uint64_t height); + void detach_blockchain(uint64_t including_height); bool extract_offers_from_transfer_entry(size_t i, std::unordered_map& offers_local); bool select_my_offers(std::list& offers); bool clear(); diff --git a/src/wallet/wallet_chain_shortener.cpp b/src/wallet/wallet_chain_shortener.cpp index 3d3bba28..c89b100d 100644 --- a/src/wallet/wallet_chain_shortener.cpp +++ b/src/wallet/wallet_chain_shortener.cpp @@ -52,6 +52,12 @@ const crypto::hash& wallet_chain_shortener::get_genesis() //---------------------------------------------------------------------------------------------------- void wallet_chain_shortener::push_new_block_id(const crypto::hash& id, uint64_t height) { + if (height == 0) + { + m_genesis = id; + m_local_bc_size = 1; + return; + } //primary 10 //self check @@ -195,6 +201,25 @@ bool wallet_chain_shortener::lookup_item_around(uint64_t i, std::pair 0) + { + block_found = true; + block_matched = id == m_genesis; + if (!block_matched) { + full_reset_needed = true; + } + } + else + { + block_found = false; + block_matched = false; + full_reset_needed = true; + } + return; + } if (!m_last_20_blocks.empty() && i > m_last_20_blocks.begin()->first) { //must be in short sequence (m_last_20_blocks) @@ -221,7 +246,7 @@ void wallet_chain_shortener::check_if_block_matched(uint64_t i, const crypto::ha bool r = lookup_item_around(i, result); if (!r) { - LOG_PRINT_L0("Wallet is getting fully resynced due to unmatched block " << id << " at " << i); + LOG_PRINT_L0("Wallet is getting fully resynced due to lookup_item_around failed at " << i); block_matched = block_found = false; full_reset_needed = true; return; @@ -251,17 +276,17 @@ void wallet_chain_shortener::check_if_block_matched(uint64_t i, const crypto::ha //---------------------------------------------------------------------------------------------------- void clean_map_from_items_above(std::map& container, uint64_t height) { - while (container.size() && (--container.end())->first > height) + while (container.size() && (--container.end())->first >= height) { container.erase(--container.end()); } } //---------------------------------------------------------------------------------------------------- -void wallet_chain_shortener::detach(uint64_t height) +void wallet_chain_shortener::detach(uint64_t including_height) { - clean_map_from_items_above(m_last_20_blocks, height); - clean_map_from_items_above(m_last_144_blocks_every_10, height); - clean_map_from_items_above(m_last_144_blocks_every_100, height); - clean_map_from_items_above(m_last_144_blocks_every_1000, height); - m_local_bc_size = height + 1; + clean_map_from_items_above(m_last_20_blocks, including_height); + clean_map_from_items_above(m_last_144_blocks_every_10, including_height); + clean_map_from_items_above(m_last_144_blocks_every_100, including_height); + clean_map_from_items_above(m_last_144_blocks_every_1000, including_height); + m_local_bc_size = including_height; } \ No newline at end of file diff --git a/src/wallet/wallet_chain_shortener.h b/src/wallet/wallet_chain_shortener.h index 29387c00..2ea61366 100644 --- a/src/wallet/wallet_chain_shortener.h +++ b/src/wallet/wallet_chain_shortener.h @@ -30,7 +30,7 @@ public: void get_short_chain_history(std::list& ids)const; bool lookup_item_around(uint64_t i, std::pair& result)const; void check_if_block_matched(uint64_t i, const crypto::hash& id, bool& block_found, bool& block_matched, bool& full_reset_needed) const; - void detach(uint64_t height); + void detach(uint64_t including_height); void clear(); void set_genesis(const crypto::hash& id); const crypto::hash& get_genesis(); diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 4093e710..2a4171bf 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -923,6 +923,10 @@ bool test_generator::refresh_test_wallet(const std::vector& ev std::atomic atomic_false = ATOMIC_VAR_INIT(false); bool r = w->refresh(blocks_fetched, received_money, ok, atomic_false); CHECK_AND_ASSERT_MES(r, false, "test wallet refersh failed"); + if (expected_blocks_to_be_fetched != blocks_fetched) + { + std::cout << "dd"; + } CHECK_AND_ASSERT_MES(expected_blocks_to_be_fetched == std::numeric_limits::max() || expected_blocks_to_be_fetched == blocks_fetched, false, "test wallet refresh fetched " << blocks_fetched << ", expected: " << expected_blocks_to_be_fetched); bool has_aliases; diff --git a/tests/core_tests/wallet_test_core_proxy.cpp b/tests/core_tests/wallet_test_core_proxy.cpp index 75dac59e..1b423523 100644 --- a/tests/core_tests/wallet_test_core_proxy.cpp +++ b/tests/core_tests/wallet_test_core_proxy.cpp @@ -56,8 +56,25 @@ bool wallet_test_core_proxy::call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(cons bool wallet_test_core_proxy::call_COMMAND_RPC_GET_BLOCKS_FAST(const currency::COMMAND_RPC_GET_BLOCKS_FAST::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FAST::response& rsp) { - rsp.current_height = 0; + //might be not best way to do it. + std::unordered_map blocks_map; + for (uint64_t i = 0; i != m_blocks.size(); i++) + { + blocks_map[currency::get_block_hash(m_blocks[i]->b)] = i; + } + rsp.start_height = 0; + //find out where we supposed to start refresh + for (auto id : rqt.block_ids) + { + auto it = blocks_map.find(id); + if (it == blocks_map.end()) + continue; + rsp.start_height = it->second; + break; + } + + rsp.current_height = m_blocks.size(); rsp.status = CORE_RPC_STATUS_OK; if (!m_first_call) { @@ -65,20 +82,21 @@ bool wallet_test_core_proxy::call_COMMAND_RPC_GET_BLOCKS_FAST(const currency::CO return true; // respond with empty blocks on second call to gracefully stop wallet refreshing } m_first_call = false; - for (auto b : m_blocks) + for (size_t i = rsp.start_height; i != m_blocks.size(); i++) { + auto b = m_blocks[i]; currency::block_complete_entry bce = AUTO_VAL_INIT(bce); for (auto tx : b->m_transactions) bce.txs.push_back(tx_to_blob(tx)); bce.block = block_to_blob(b->b); rsp.blocks.push_back(bce); } - rsp.current_height = m_blocks.size() - 1; + rsp.current_height = m_blocks.size(); return true; } bool wallet_test_core_proxy::call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& rsp) { - currency::COMMAND_RPC_GET_BLOCKS_FAST::request req; + currency::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); req.block_ids = rqt.block_ids; currency::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); bool r = this->call_COMMAND_RPC_GET_BLOCKS_FAST(req, res);