some optimizations over tx pool synchronization

contrib/zlib/test/minigzip.c

@@ -15,7 +15,7 @@
 
 /* @(#) $Id$ */
 
-#include "zlib.h"
+#include "../zlib.h"
 #include <stdio.h>
 
 #ifdef STDC

src/currency_core/blockchain_storage.cpp

@@ -4654,7 +4654,7 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont
 
     block bl = bl_;
     crypto::hash id = get_block_hash(bl);
-    CRITICAL_REGION_LOCAL(m_tx_pool);
+    //CRITICAL_REGION_LOCAL(m_tx_pool);
     //CRITICAL_REGION_LOCAL1(m_read_lock);
     if (have_block(id))
     {

src/currency_core/currency_config.h

@@ -192,7 +192,7 @@
 #define CURRENCY_CORE_INSTANCE_LOCK_FILE                "lock.lck"
 
 
-#define CURRENCY_POOLDATA_FOLDERNAME                    "poolstate"
+#define CURRENCY_POOLDATA_FILENAME                      "poolstate.bin"
 #define CURRENCY_BLOCKCHAINDATA_FOLDERNAME              "blockchain"
 #define P2P_NET_DATA_FILENAME                           "p2pstate.bin"
 #define MINER_CONFIG_FILENAME                           "miner_conf.json"
@@ -214,7 +214,7 @@
 
 #define WALLET_FILE_SERIALIZATION_VERSION               (CURRENCY_FORMATION_VERSION+62)
 
-#define CURRENT_MEMPOOL_ARCHIVE_VER                     (CURRENCY_FORMATION_VERSION+31)
+#define CURRENT_MEMPOOL_ARCHIVE_VER                     (CURRENCY_FORMATION_VERSION+0)
 
 
 

src/currency_core/currency_core.cpp

@@ -386,7 +386,9 @@ namespace currency
   //-----------------------------------------------------------------------------------------------
   bool core::add_new_tx(const transaction& tx, const crypto::hash& tx_hash, size_t blob_size, tx_verification_context& tvc, bool kept_by_block)
   {
-    if(m_mempool.have_tx(tx_hash))
+    //extra check "kept_by_block" to let happen call "m_mempool.add_tx" for transactions that came with block
+    //which should update "last_touch_time", to avoid removing transaction as "stuck" while it adding with new block(possible when mem_pool is over flooded)
+    if(!kept_by_block && m_mempool.have_tx(tx_hash))
     {
       LOG_PRINT_L3("add_new_tx: already have tx " << tx_hash << " in the pool");
       return true;

src/currency_core/tx_pool.h

@@ -50,6 +50,7 @@ namespace currency
       uint64_t last_failed_height;
       crypto::hash last_failed_id;
       time_t receive_time;
+      time_t last_touch_time;
 
       BEGIN_SERIALIZE_OBJECT()
         FIELD(tx)
@@ -61,7 +62,8 @@ namespace currency
         FIELD(last_failed_height)
         FIELD(last_failed_id)
         FIELD(receive_time)
-        END_SERIALIZE()
+        FIELD(last_touch_time) // last time new block added this tx
+      END_SERIALIZE()
 
     };
 
@@ -108,8 +110,8 @@ namespace currency
 
     void on_idle();
 
-    void lock();
+    //void lock();
-    void unlock();
+    //void unlock();
     void purge_transactions();
     void clear();
 
@@ -128,15 +130,27 @@ namespace currency
     bool get_transaction(const crypto::hash& h, transaction& tx)const;
     bool get_transaction(const crypto::hash& h, tx_details& txd)const;
     size_t get_transactions_count() const;
-    bool have_key_images(const std::unordered_set<crypto::key_image>& kic, const transaction& tx)const;
+    static bool have_key_images(const std::unordered_set<crypto::key_image>& kic, const transaction& tx);
-    bool append_key_images(std::unordered_set<crypto::key_image>& kic, const transaction& tx);
+    static bool append_key_images(std::unordered_set<crypto::key_image>& kic, const transaction& tx);
     std::string print_pool(bool short_format)const;
     uint64_t get_core_time() const;
     bool get_aliases_from_tx_pool(std::list<extra_alias_entry>& aliases)const;
     bool get_aliases_from_tx_pool(std::map<std::string, size_t>& aliases)const;
-    //crypto::hash get_last_core_hash() {return m_last_core_top_hash;}
-    //void set_last_core_hash(const crypto::hash& h) { m_last_core_top_hash = h; }
 
+    template<class archive_t>
+    void serialize(archive_t & ar, const unsigned int version)
+    {
+      if (version < CURRENT_MEMPOOL_ARCHIVE_VER)
+        return;
+      CHECK_PROJECT_NAME();
+      CRITICAL_REGION_LOCAL(m_transactions_lock);
+      ar & m_transactions;
+      ar & m_cancel_offer_hashes;
+      ar & m_black_tx_list;
+      ar & m_key_images_set;
+      ar & m_alias_names_set;
+      ar & m_alias_addresses_set;
+    }
 
   private:
     bool on_tx_add(const transaction& tx, bool kept_by_block);
@@ -146,43 +160,31 @@ namespace currency
     bool insert_alias_info(const transaction& tx);
     bool remove_alias_info(const transaction& tx);
 
+    bool touch_tx(const crypto::hash &id);
     bool is_valid_contract_finalization_tx(const transaction &tx)const;
-    void initialize_db_solo_options_values();
     bool remove_stuck_transactions();
     bool is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const;
     bool validate_alias_info(const transaction& tx, bool is_in_block)const;
     bool get_key_images_from_tx_pool(std::unordered_set<crypto::key_image>& key_images)const;
-    //bool push_alias_info(const transaction& tx);
-    //bool pop_alias_info(const transaction& tx);
     bool process_cancel_offer_rules(const transaction& tx);
     bool unprocess_cancel_offer_rules(const transaction& tx);
-    bool check_is_taken(const crypto::hash& id) const;
+//     bool check_is_taken(const crypto::hash& id) const;
-    void set_taken(const crypto::hash& id);
+//     void set_taken(const crypto::hash& id);
-    void reset_all_taken();
+//     void reset_all_taken();
-    
-    typedef tools::db::cached_key_value_accessor<crypto::hash, tx_details, true, false> transactions_container;
-    typedef tools::db::cached_key_value_accessor<crypto::hash, bool, false, false> hash_container; 
-    typedef tools::db::cached_key_value_accessor<crypto::key_image, uint64_t, false, false> key_images_container;
-    typedef tools::db::cached_key_value_accessor<uint64_t, uint64_t, false, true> solo_options_container;
-    typedef tools::db::cached_key_value_accessor<std::string, bool, false, false> aliases_container; 
-    typedef tools::db::cached_key_value_accessor<account_public_address, bool, false, false> address_to_aliases_container;
 
+    typedef std::unordered_map<crypto::hash, tx_details> transactions_container;
+    typedef std::unordered_map<crypto::key_image, uint64_t> key_images_container;
+    typedef std::unordered_set<crypto::hash> hash_container;
+    typedef std::unordered_set<std::string> aliases_container;
+    typedef std::unordered_set<account_public_address> aliases_addresses_container;
 
-    //main accessor
-    epee::shared_recursive_mutex m_dummy_rw_lock;
-    tools::db::basic_db_accessor m_db;
     //containers
-
+    transactions_container m_transactions;
-    transactions_container m_db_transactions;
+    hash_container m_cancel_offer_hashes;
-    hash_container m_db_cancel_offer_hash;
+    hash_container  m_black_tx_list;
-    hash_container  m_db_black_tx_list;
+    key_images_container m_key_images_set;
-    key_images_container m_db_key_images_set;
+    aliases_container m_alias_names_set;
-    aliases_container m_db_alias_names;
+    aliases_addresses_container m_alias_addresses_set;
-    address_to_aliases_container m_db_alias_addresses;
-    solo_options_container m_db_solo_options;
-    tools::db::solo_db_value<uint64_t, uint64_t, solo_options_container> m_db_storage_major_compatibility_version;
-    //crypto::hash m_last_core_top_hash;
-
 
     epee::math_helper::once_a_time_seconds<30> m_remove_stuck_tx_interval;
 
@@ -192,9 +194,21 @@ namespace currency
     i_currency_protocol* m_pprotocol;
 
     //in memory containers
-    mutable epee::critical_section m_taken_txs_lock;
+    //mutable epee::critical_section m_taken_txs_lock;
-    std::unordered_set<crypto::hash> m_taken_txs;
+    //std::unordered_set<crypto::hash> m_taken_txs;
-    mutable epee::critical_section m_remove_stuck_txs_lock;
+
+
+    mutable epee::critical_section m_add_tx_lock; 
+    std::atomic<crypto::hash> m_current_processing_tx_id;
+    //main accessor
+    mutable epee::critical_section m_transactions_lock;
+    //other containers locks
+    //mutable epee::critical_section m_remove_stuck_txs_lock;
+    mutable epee::critical_section m_cancel_offer_hashes_lock;
+    mutable epee::critical_section m_aliases_lock;
+    mutable epee::critical_section m_black_tx_list_lock;
+    mutable epee::critical_section m_key_images_set_lock;
+
 
     
     /************************************************************************/
@@ -239,6 +253,3 @@ namespace boost
 }
 
 BOOST_CLASS_VERSION(currency::tx_memory_pool, CURRENT_MEMPOOL_ARCHIVE_VER)
-
-
-

tests/core_tests/chaingen_main.cpp

@@ -56,7 +56,7 @@ bool clean_data_directory()
 {
   std::string config_folder = command_line::get_arg(g_vm, command_line::arg_data_dir);
 
-  static const char* const files[] = { CURRENCY_BLOCKCHAINDATA_FOLDERNAME, CURRENCY_POOLDATA_FOLDERNAME, MINER_CONFIG_FILENAME };
+  static const char* const files[] = { CURRENCY_BLOCKCHAINDATA_FOLDERNAME, CURRENCY_POOLDATA_FILENAME, MINER_CONFIG_FILENAME };
   for (size_t i = 0; i < sizeof files / sizeof files[0]; ++i)
   {
     boost::filesystem::path filename(config_folder + "/" + files[i]);