transaction pool optimisation: basic implemntation

src/currency_core/blockchain_storage.cpp

@@ -439,7 +439,7 @@ bool blockchain_storage::deinit()
   return true;
 }
 //------------------------------------------------------------------
-bool blockchain_storage::pop_block_from_blockchain()
+bool blockchain_storage::pop_block_from_blockchain(transactions_map& onboard_transactions)
 {
   CRITICAL_REGION_LOCAL(m_read_lock);
 
@@ -449,7 +449,7 @@ bool blockchain_storage::pop_block_from_blockchain()
   CHECK_AND_ASSERT_MES(bei_ptr.get(), false, "pop_block_from_blockchain: can't pop from blockchain");
 
   uint64_t fee_total = 0;
-  bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total);
+  bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total, onboard_transactions);
   CHECK_AND_ASSERT_MES(r, false, "Failed to purge_block_data_from_blockchain for block " << get_block_hash(bei_ptr->bl) << " on height " << h);
 
   pop_block_from_per_block_increments(bei_ptr->height);
@@ -649,7 +649,7 @@ bool blockchain_storage::purge_transaction_keyimages_from_blockchain(const trans
   return true;
 }
 //------------------------------------------------------------------
-bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee)
+bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx_)
 {
   fee = 0;
   CRITICAL_REGION_LOCAL(m_read_lock);
@@ -657,6 +657,7 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t
   auto tx_res_ptr = m_db_transactions.find(tx_id);
   CHECK_AND_ASSERT_MES(tx_res_ptr != m_db_transactions.end(), false, "transaction " << tx_id << " is not found in blockchain index!!");
   const transaction& tx = tx_res_ptr->tx;
+  tx_ = tx;
 
   fee = get_tx_fee(tx_res_ptr->tx);
   purge_transaction_keyimages_from_blockchain(tx, true);
@@ -687,10 +688,11 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t
 bool blockchain_storage::purge_block_data_from_blockchain(const block& b, size_t processed_tx_count)
 {
   uint64_t total_fee = 0;
-  return purge_block_data_from_blockchain(b, processed_tx_count, total_fee);
+  transactions_map onboard_transactions;
+  return purge_block_data_from_blockchain(b, processed_tx_count, total_fee, onboard_transactions);
 }
 //------------------------------------------------------------------
-bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total)
+bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total, transactions_map& onboard_transactions)
 {
   CRITICAL_REGION_LOCAL(m_read_lock);
   fee_total = 0;
@@ -699,11 +701,13 @@ bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_
   CHECK_AND_ASSERT_MES(processed_tx_count <= bl.tx_hashes.size(), false, "wrong processed_tx_count in purge_block_data_from_blockchain");
   for(size_t count = 0; count != processed_tx_count; count++)
   {
-    res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee) && res;
+    transaction tx = AUTO_VAL_INIT(tx);
+    res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee, tx) && res;
     fee_total += fee;
+    onboard_transactions[bl.tx_hashes[(processed_tx_count - 1) - count]] = tx;
   }
-
+  transaction tx = AUTO_VAL_INIT(tx);
-  res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee) && res;
+  res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee, tx) && res;
   return res;
 }
 //------------------------------------------------------------------
@@ -866,20 +870,22 @@ bool blockchain_storage::get_block_by_height(uint64_t h, block &blk) const
 //     invalid.push_back(v.first);
 // } 
 //------------------------------------------------------------------
-bool blockchain_storage::rollback_blockchain_switching(std::list<block>& original_chain, size_t rollback_height)
+bool blockchain_storage::rollback_blockchain_switching(std::list<block_ws_txs>& original_chain, size_t rollback_height)
 {
   CRITICAL_REGION_LOCAL(m_read_lock);
   //remove failed subchain
   for(size_t i = m_db_blocks.size()-1; i >=rollback_height; i--)
   {
-    bool r = pop_block_from_blockchain();
+    transactions_map ot;
+    bool r = pop_block_from_blockchain(ot);
     CHECK_AND_ASSERT_MES(r, false, "PANIC!!! failed to remove block while chain switching during the rollback!");
   }
   //return back original chain
-  BOOST_FOREACH(auto& bl, original_chain)
+  BOOST_FOREACH(auto& oce, original_chain)
   {
     block_verification_context bvc = boost::value_initialized<block_verification_context>();
-    bool r = handle_block_to_main_chain(bl, bvc);
+    bvc.m_onboard_transactions.swap(oce.onboard_transactions);
+    bool r = handle_block_to_main_chain(oce.b, bvc);
     CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC!!! failed to add (again) block while chain switching during the rollback!");
   }
 
@@ -942,13 +948,15 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch
     );
 
   //disconnecting old chain
-  std::list<block> disconnected_chain;
+  std::list<block_ws_txs> disconnected_chain;
   for(size_t i = m_db_blocks.size()-1; i >=split_height; i--)
   {
-    block b = m_db_blocks[i]->bl;
+    disconnected_chain.push_front(block_ws_txs());
-    bool r = pop_block_from_blockchain();
+    block_ws_txs& bwt = disconnected_chain.front();
-    CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(b) << " @ " << get_block_height(b) << " on chain switching");
+    bwt.b = m_db_blocks[i]->bl;
-    disconnected_chain.push_front(b);
+    bool r = pop_block_from_blockchain(bwt.onboard_transactions);
+    CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(bwt.b) << " @ " << get_block_height(bwt.b) << " on chain switching");
+    
     CHECK_AND_ASSERT_MES(validate_blockchain_prev_links(), false, "EPIC FAIL!");
   }
 
@@ -957,6 +965,7 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch
   {
     auto ch_ent = *alt_ch_iter;
     block_verification_context bvc = boost::value_initialized<block_verification_context>();
+    bvc.m_onboard_transactions = ch_ent->second.onboard_transactions;
     bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc);
     if(!r || !bvc.m_added_to_main_chain)
     {
@@ -978,7 +987,8 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch
   for(auto& old_ch_ent : disconnected_chain)
   {
     block_verification_context bvc = boost::value_initialized<block_verification_context>();
-    bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc);
+    bvc.m_onboard_transactions.swap(old_ch_ent.onboard_transactions);
+    bool r = handle_alternative_block(old_ch_ent.b, get_block_hash(old_ch_ent.b), bvc);
     if(!r)
     {
       LOG_ERROR("Failed to push ex-main chain blocks to alternative chain ");
@@ -1602,6 +1612,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto::
 
     alt_block_extended_info abei = AUTO_VAL_INIT(abei);
     abei.bl = b;
+    abei.onboard_transactions.swap(bvc.m_onboard_transactions);
     abei.timestamp = m_core_runtime_config.get_core_time();
     abei.height = alt_chain.size() ? it_prev->second.height + 1 : *ptr_main_prev + 1;
     CHECK_AND_ASSERT_MES_CUSTOM(coinbase_height == abei.height, false, bvc.m_verification_failed = true, "block coinbase height doesn't match with altchain height, declined");
@@ -1716,7 +1727,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto::
     alt_chain.push_back(i_res.first);
     //check if difficulty bigger then in main chain
 
-    bvc.height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0;
+    bvc.m_height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0;
 
     crypto::hash proof = null_hash;
     std::stringstream ss_pow_pos_info;
@@ -1753,7 +1764,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto::
         bvc.m_verification_failed = true;
       return r;
     }
-    bvc.added_to_altchain = true;
+    bvc.m_added_to_altchain = true;
 
     //protect ourself from altchains container flood
     if (m_alternative_chains.size() > m_core_runtime_config.max_alt_blocks)
@@ -4763,7 +4774,7 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise
   return 0;
 }
 //------------------------------------------------------------------
-bool get_tx_from_cache(const crypto::hash& tx_id, std::unordered_map<crypto::hash, transaction>& tx_cache, transaction& tx, size_t& blob_size, uint64_t fee)
+bool get_tx_from_cache(const crypto::hash& tx_id, transactions_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t fee)
 {
   auto it = tx_cache.find(tx_id);
   if (it == tx_cache.end())
@@ -5344,7 +5355,8 @@ bool blockchain_storage::truncate_blockchain(uint64_t to_height)
   uint64_t inital_height = get_current_blockchain_size();
   while (get_current_blockchain_size() > to_height)
   {
-    pop_block_from_blockchain();
+    transactions_map ot;
+    pop_block_from_blockchain(ot);
   }
   CRITICAL_REGION_LOCAL(m_alternative_chains_lock);
   m_alternative_chains.clear();

src/currency_core/blockchain_storage.h

@@ -156,6 +156,9 @@ namespace currency
       
       //date added to alt chain storage
       uint64_t timestamp; 
+      
+      //transactions associated with the block
+      transactions_map onboard_transactions;
     };
     typedef std::unordered_map<crypto::hash, alt_block_extended_info> alt_chain_container;
     //typedef std::list<alt_chain_container::iterator> alt_chain_type;
@@ -543,10 +546,10 @@ namespace currency
     bool switch_to_alternative_blockchain(alt_chain_type& alt_chain);
     void purge_alt_block_txs_hashs(const block& b);
     void add_alt_block_txs_hashs(const block& b);   
-    bool pop_block_from_blockchain();
+    bool pop_block_from_blockchain(transactions_map& onboard_transactions);
     bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count);
-    bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee);
+    bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee, transactions_map& onboard_transactions);
-    bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee);
+    bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx);
     bool purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check);
     wide_difficulty_type get_next_difficulty_for_alternative_chain(const alt_chain_type& alt_chain, block_extended_info& bei, bool pos) const;
     bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc);
@@ -565,7 +568,7 @@ namespace currency
     bool get_transaction_from_pool_or_db(const crypto::hash& tx_id, std::shared_ptr<transaction>& tx_ptr, uint64_t min_allowed_block_height = 0) const;
     void get_last_n_x_blocks(uint64_t n, bool pos_blocks, std::list<std::shared_ptr<const block_extended_info>>& blocks) const;
     bool prevalidate_miner_transaction(const block& b, uint64_t height, bool pos)const;
-    bool rollback_blockchain_switching(std::list<block>& original_chain, size_t rollback_height);
+    bool rollback_blockchain_switching(std::list<block_ws_txs>& original_chain, size_t rollback_height);
     bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, uint64_t timestamp);
     bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes);
     bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id);

src/currency_core/blockchain_storage_basic.h

@@ -145,6 +145,12 @@ namespace currency
     uint64_t height;
   };
 
-  
+  typedef std::unordered_map<crypto::hash, transaction> transactions_map;
+
+  struct block_ws_txs
+  {
+    block b;
+    transactions_map onboard_transactions;
+  };
 
 }

src/currency_core/verification_context.h

@@ -25,11 +25,11 @@ namespace currency
     bool m_verification_failed; //bad block, should drop connection
     bool m_marked_as_orphaned;
     bool m_already_exists;
-    bool added_to_altchain;
+    bool m_added_to_altchain;
-    uint64_t height_difference;
+    uint64_t m_height_difference;
     //this is work like a first-level cache for transactions while block is getting handled. It lets transactions 
     //associated with the block to get handled directly to core without being handled by tx_pool(which makes full
     //inputs validation, including signatures check)
-    std::unordered_map<crypto::hash, transaction> m_onboard_transactions; 
+    transactions_map m_onboard_transactions;
   };
 }

src/currency_protocol/currency_protocol_handler.inl

@@ -327,10 +327,10 @@ namespace currency
     LOG_PRINT_GREEN("[HANDLE]NOTIFY_NEW_BLOCK EXTRA " << block_id 
       << " bvc.m_added_to_main_chain=" << bvc.m_added_to_main_chain
       //<< ", prevalidate_result=" << prevalidate_relayed
-      << ", bvc.added_to_altchain=" << bvc.added_to_altchain
+      << ", bvc.added_to_altchain=" << bvc.m_added_to_altchain
       << ", bvc.m_marked_as_orphaned=" << bvc.m_marked_as_orphaned, LOG_LEVEL_2);
 
-    if (bvc.m_added_to_main_chain || (bvc.added_to_altchain && bvc.height_difference < 2))
+    if (bvc.m_added_to_main_chain || (bvc.m_added_to_altchain && bvc.m_height_difference < 2))
     { 
       if (true/*!prevalidate_relayed*/)
       {

src/rpc/core_rpc_server.cpp

@@ -851,7 +851,7 @@ namespace currency
     block_verification_context bvc = AUTO_VAL_INIT(bvc);
     if(!m_core.handle_block_found(b, &bvc))
     {
-      if (bvc.added_to_altchain)
+      if (bvc.m_added_to_altchain)
       {
         error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_ADDED_AS_ALTERNATIVE;
         error_resp.message = "Block added as alternative";

src/stratum/stratum_server.cpp

@@ -532,7 +532,7 @@ namespace
       r = m_p_core->handle_block_found(m_block_template, &bvc, false);
       if (r)
       {
-        if (!bvc.m_verification_failed && !bvc.added_to_altchain && bvc.m_added_to_main_chain && !bvc.m_already_exists && !bvc.m_marked_as_orphaned)
+        if (!bvc.m_verification_failed && !bvc.m_added_to_altchain && bvc.m_added_to_main_chain && !bvc.m_already_exists && !bvc.m_marked_as_orphaned)
         {
           LP_CC_WORKER_GREEN(p_ph->get_context(), "found block " << block_hash << " at height " << height << " was successfully added to the blockchain, difficulty " << m_network_difficulty, LOG_LEVEL_0);
           r = update_block_template();
@@ -544,7 +544,7 @@ namespace
         {
           LP_CC_WORKER_RED(p_ph->get_context(), "block " << block_hash << " at height " << height << " was NOT added to the blockchain:" << ENDL <<
             "    verification_failed: " << bvc.m_verification_failed  << ENDL <<
-            "    added_to_altchain:   " << bvc.added_to_altchain      << ENDL <<
+            "    added_to_altchain:   " << bvc.m_added_to_altchain      << ENDL <<
             "    added_to_main_chain: " << bvc.m_added_to_main_chain  << ENDL <<
             "    already_exists:      " << bvc.m_already_exists       << ENDL <<
             "    marked_as_orphaned:  " << bvc.m_marked_as_orphaned, LOG_LEVEL_0);