added full warm up feature

contrib/epee/include/cache_helper.h

@@ -70,12 +70,17 @@ namespace epee
       cache_base() : mac_allowed_elements(max_elements)
       {}
 
-      size_t size()
+      size_t size() const
       {
         
         return data.size();
       }
 
+      uint64_t get_max_elements() const
+      {
+        return mac_allowed_elements;
+      }
+
       void set_max_elements(uint64_t e)
       {
         mac_allowed_elements = e;

src/common/command_line.cpp

@@ -40,6 +40,7 @@ namespace command_line
   const arg_descriptor<std::string> arg_process_predownload_from_path("predownload-from-local-path", "Instead of downloading file use downloaded local file");
   const arg_descriptor<bool>        arg_validate_predownload  ( "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database");
   const arg_descriptor<std::string> arg_predownload_link      ( "predownload-link", "Override url for blockchain database pre-downloading");
+  
 
   const arg_descriptor<std::string> arg_deeplink  ( "deeplink-params", "Deeplink parameter, in that case app just forward params to running app");
 

src/common/db_abstract_accessor.h

@@ -766,6 +766,16 @@ namespace tools
         m_cache.clear();
       }
 
+      uint64_t get_cacheed_items_count() const
+      {
+        return m_cache.size();
+      }
+
+      uint64_t get_cache_size() const
+      {
+        return m_cache.get_max_elements();
+      }
+
       void set_cache_size(uint64_t max_cache_size)
       {
         m_cache.set_max_elements(max_cache_size);
@@ -806,6 +816,11 @@ namespace tools
         }          
         return res;
       }
+     
+      std::shared_ptr<const t_value> find(const t_key& k) const
+      {
+        return get(k);
+      }
 
       size_t clear()
       {

src/currency_core/blockchain_storage.cpp

@@ -5331,6 +5331,54 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) const
   return currency::is_tx_spendtime_unlocked(unlock_time, get_current_blockchain_size(), m_core_runtime_config.get_core_time());
 }
 //------------------------------------------------------------------
+void blockchain_storage::do_full_db_warm_up() const
+{
+
+  uint64_t ticks_last_print = 0;
+  uint64_t current_sz = get_current_blockchain_size();
+  LOG_PRINT_CYAN("Warming up starting, total blocks..." << current_sz, LOG_LEVEL_0);
+  for (uint64_t i = current_sz - 1; i != 0; --i)
+  {
+    CRITICAL_REGION_LOCAL(m_read_lock);
+    auto blk_ptr = m_db_blocks[i];
+    std::list<crypto::hash> mis;
+    std::list<std::shared_ptr<const transaction_chain_entry> > txs;
+    get_transactions_direct(blk_ptr->bl.tx_hashes, txs, mis);
+    CHECK_AND_ASSERT_MES(!mis.size(), void(), "internal error, block " << get_block_hash(m_db_blocks[i]->bl) << " [" << i << "] contains missing transactions: " << mis);
+    if (txs.size() != blk_ptr->bl.tx_hashes.size())
+    {
+      LOG_ERROR("some tx's not found");
+      return;
+    }
+    auto coinbase_tx_ptr = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx));
+    if (!coinbase_tx_ptr)
+    {
+      LOG_ERROR("Coinbase not found");
+      return;
+    }
+    if (epee::misc_utils::get_tick_count() - ticks_last_print > 1000)
+    {
+      ticks_last_print = epee::misc_utils::get_tick_count();
+      LOG_PRINT_CYAN("Warming up: " <<  ( ( (current_sz - i) * 100)/ current_sz) << "%, " << (current_sz - i) << " of " << current_sz, LOG_LEVEL_0);
+
+#define PRINT_CONTAINER(cont_name)  strm << #cont_name"[" << cont_name.get_cache_size() << "]:" << (cont_name.get_cacheed_items_count() * 100) / cont_name.get_cache_size() << " %, items: " << cont_name.get_cacheed_items_count() << ENDL
+
+      std::stringstream strm;
+      PRINT_CONTAINER(m_db_blocks_index);
+      PRINT_CONTAINER(m_db_blocks);
+      PRINT_CONTAINER(m_db_blocks_index);
+      PRINT_CONTAINER(m_db_transactions);
+      PRINT_CONTAINER(m_db_spent_keys);
+      PRINT_CONTAINER(m_db_multisig_outs);
+      PRINT_CONTAINER(m_db_solo_options);
+      PRINT_CONTAINER(m_db_aliases);
+      PRINT_CONTAINER(m_db_assets);
+      PRINT_CONTAINER(m_db_addr_to_alias);
+      LOG_PRINT_CYAN("CACHE STATE: " << ENDL << strm.str(), LOG_LEVEL_0);
+    }
+  }
+}
+//------------------------------------------------------------------
 bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase) const
 {
   CRITICAL_REGION_LOCAL(m_read_lock);

src/currency_core/blockchain_storage.h

@@ -508,6 +508,9 @@ namespace currency
     uint64_t get_last_x_block_height(bool pos)const;
     bool is_tx_spendtime_unlocked(uint64_t unlock_time)const;
 
+    //experimental
+    void do_full_db_warm_up() const;
+
   private:
 
     //-------------- DB containers --------------

src/daemon/daemon.cpp

@@ -40,6 +40,7 @@ using namespace epee;
 //TODO: need refactoring here. (template classes can't be used in BOOST_CLASS_VERSION) 
 BOOST_CLASS_VERSION(nodetool::node_server<currency::t_currency_protocol_handler<currency::core> >, CURRENT_P2P_STORAGE_ARCHIVE_VER);
 
+const command_line::arg_descriptor<uint32_t>    arg_rpc_server_threads("rpc-server-threads", "Specify number of RPC server threads. Default: 10", RPC_SERVER_DEFAULT_THREADS_NUM);
 
 namespace po = boost::program_options;
 
@@ -165,7 +166,8 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload);
   command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link);
   command_line::add_arg(desc_cmd_sett, command_line::arg_disable_ntp);
-
+  command_line::add_arg(desc_cmd_sett, arg_rpc_server_threads);
+ 
 
   arg_market_disable.default_value = true;
   arg_market_disable.use_default = true;
@@ -184,6 +186,8 @@ int main(int argc, char* argv[])
   po::options_description desc_options("Allowed options");
   desc_options.add(desc_cmd_only).add(desc_cmd_sett);
 
+  
+
   po::variables_map vm;
   bool exit_requested = false;
   bool r = command_line::handle_error_helper(desc_options, [&]()
@@ -374,8 +378,28 @@ int main(int argc, char* argv[])
     dch.start_handling();
   }
 
+  uint32_t rpc_threads_count = RPC_SERVER_DEFAULT_THREADS_NUM;
+  if (command_line::has_arg(vm, arg_rpc_server_threads))
+  {
+    rpc_threads_count = command_line::get_arg(vm, arg_rpc_server_threads);
+
+    if (rpc_threads_count < 2)
+    {
+      LOG_ERROR("RPC server threads number number is too low: " << rpc_threads_count << "(why would you do that?!)");
+      return false;
+    }
+    LOG_PRINT_L0("RPC server threads number is overridden with " << rpc_threads_count);
+    if (rpc_threads_count < RPC_SERVER_DEFAULT_THREADS_NUM)
+    {
+      LOG_PRINT_RED_L0("Warning: RPC server threads number is overridden with low number: " << rpc_threads_count << "(why would you do that?!)");
+    }
+
+
+  }
+  
+   
   LOG_PRINT_L0("Starting core rpc server...");
-  res = rpc_server.run(2, false);
+  res = rpc_server.run(rpc_threads_count, false);
   CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core rpc server.");
   LOG_PRINT_L0("Core rpc server started ok");
 

src/daemon/daemon_commands_handler.h

@@ -79,6 +79,7 @@ public:
     m_cmd_binder.set_handler("print_tx_outputs_usage", boost::bind(&daemon_commands_handler::print_tx_outputs_usage, this, ph::_1), "Analyse if tx outputs for involved in subsequent transactions");
     m_cmd_binder.set_handler("print_difficulties_of_last_n_blocks", boost::bind(&daemon_commands_handler::print_difficulties_of_last_n_blocks, this, ph::_1), "Print difficulties of last n blocks");
     m_cmd_binder.set_handler("debug_remote_node_mode", boost::bind(&daemon_commands_handler::debug_remote_node_mode, this, ph::_1), "<ip-address> - If node got connected put node into 'debug mode' i.e. no sync process of other communication except ping responses, maintenance secrete key will be requested"); 
+    m_cmd_binder.set_handler("full_db_cache_warmup", boost::bind(&daemon_commands_handler::full_db_cache_warmup, this, ph::_1), "(Experimental) Perform full DB loading to RAM cache(make sense only with big numbers passed to --db-cache-l2 option)");
 #ifdef _DEBUG
     m_cmd_binder.set_handler("debug_set_time_adj", boost::bind(&daemon_commands_handler::debug_set_time_adj, this, ph::_1), "DEBUG: set core time adjustment");
 #endif
@@ -883,6 +884,14 @@ private:
     return true;
   }
   //--------------------------------------------------------------------------------
+  bool full_db_cache_warmup(const std::vector<std::string>& args)
+  {
+    
+    m_srv.get_payload_object().get_core().get_blockchain_storage().do_full_db_warm_up();
+
+    return true;
+  }
+  //--------------------------------------------------------------------------------
   bool print_pool(const std::vector<std::string>& args)
   {
     LOG_PRINT_L0("Pool state: " << ENDL << m_srv.get_payload_object().get_core().print_pool(false));

src/p2p/net_node.h

@@ -36,6 +36,8 @@ ENABLE_CHANNEL_BY_DEFAULT(LOG_DEFAULT_CHANNEL);
 
 #define CURRENT_P2P_STORAGE_ARCHIVE_VER    (CURRENCY_FORMATION_VERSION+13)
 
+#define P2P_SERVER_DEFAULT_THREADS_NUM                                       10
+
 PUSH_VS_WARNINGS
 DISABLE_VS_WARNINGS(4355)
 
@@ -249,8 +251,9 @@ namespace nodetool
     bool m_hide_my_port;
     bool m_offline_mode;
     bool m_debug_requests_enabled;
-    bool m_ip_auto_blocking_enabled;
+    bool m_ip_auto_blocking_enabled = false;
     uint64_t m_startup_time;
+    uint32_t m_threads_count = P2P_SERVER_DEFAULT_THREADS_NUM;
 
 
     //critical_section m_connections_lock;

src/p2p/net_node.inl

@@ -38,6 +38,7 @@ namespace nodetool
     const command_line::arg_descriptor<bool>                      arg_p2p_offline_mode               ( "offline-mode", "Don't connect to any node and reject any connections");
     const command_line::arg_descriptor<bool>                      arg_p2p_disable_debug_reqs         ( "disable-debug-p2p-requests", "Disable p2p debug requests");
     const command_line::arg_descriptor<uint32_t>                  arg_p2p_ip_auto_blocking           ( "p2p-ip-auto-blocking", "Enable (1) or disable (0) peers auto-blocking by IP <0|1>. Default: 0", 1);
+    const command_line::arg_descriptor<uint32_t>                  arg_p2p_server_threads             ( "p2p-server-threads", "Specify number of p2p server threads. Default: 10", P2P_SERVER_DEFAULT_THREADS_NUM);
   }
 
   //-----------------------------------------------------------------------------------
@@ -56,6 +57,7 @@ namespace nodetool
     command_line::add_arg(desc, arg_p2p_disable_debug_reqs);
     command_line::add_arg(desc, arg_p2p_use_only_priority_nodes);
     command_line::add_arg(desc, arg_p2p_ip_auto_blocking);
+    command_line::add_arg(desc, arg_p2p_server_threads);
   }
   //-----------------------------------------------------------------------------------
   template<class t_payload_net_handler>
@@ -195,6 +197,21 @@ namespace nodetool
     m_offline_mode = command_line::get_arg(vm, arg_p2p_offline_mode);
     m_debug_requests_enabled = !command_line::get_arg(vm, arg_p2p_disable_debug_reqs);
     m_ip_auto_blocking_enabled = (command_line::get_arg(vm, arg_p2p_ip_auto_blocking) != 0);
+    
+    if (command_line::has_arg(vm, arg_p2p_server_threads))
+    {
+      m_threads_count = command_line::get_arg(vm, arg_p2p_server_threads);
+      if (m_threads_count < 2)
+      {
+        LOG_ERROR("P2P server threads number number is too low: " << m_threads_count << "(why would you do that?!)");
+        return false;
+      }
+      LOG_PRINT_L0("P2P server threads number is overridden with " << m_threads_count);
+      if (m_threads_count < P2P_SERVER_DEFAULT_THREADS_NUM)
+      {
+        LOG_PRINT_RED_L0("Warning: P2P server threads number is overridden with low number: " << m_threads_count << "(why would you do that?!)");
+      }
+    }
 
     LOG_PRINT_L0("p2p peers auto-blocking is " << (m_ip_auto_blocking_enabled ? "enabled" : "disabled"));
 

src/rpc/core_rpc_server.cpp

@@ -102,12 +102,19 @@ namespace currency
     res.current_blocks_median = m_core.get_blockchain_storage().get_current_comulative_blocksize_limit() / 2;
     res.alias_count = m_core.get_blockchain_storage().get_aliases_count();
     res.current_max_allowed_block_size = m_core.get_blockchain_storage().get_current_comulative_blocksize_limit();
-    if (!res.outgoing_connections_count)
-      res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_connecting;
-    else if (m_p2p.get_payload_object().is_synchronized())
+    if(m_ignore_offline_status)
+    { 
       res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_online;
+    }
     else
-      res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_synchronizing;
+    {
+      if (!res.outgoing_connections_count)
+        res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_connecting;
+      else if (m_p2p.get_payload_object().is_synchronized())
+        res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_online;
+      else
+        res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_synchronizing;
+    }
     res.synchronization_start_height = m_p2p.get_payload_object().get_core_inital_height();
     res.max_net_seen_height = m_p2p.get_payload_object().get_max_seen_height();
     m_p2p.get_maintainers_info(res.mi);
@@ -311,7 +318,7 @@ namespace currency
   bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx)
   {
     CHECK_CORE_READY();
-
+    LOG_PRINT_L2("[on_get_blocks]: Prevalidating....");
     if (req.block_ids.back() != m_core.get_blockchain_storage().get_block_id_by_height(0))
     {
       //genesis mismatch, return specific
@@ -325,7 +332,7 @@ namespace currency
       res.status = API_RETURN_CODE_BAD_ARG;
       return true;
     }
-
+    LOG_PRINT_L2("[on_get_blocks]: find_blockchain_supplement ....");
     blockchain_storage::blocks_direct_container bs;
     if (!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height))
     {
@@ -333,23 +340,24 @@ namespace currency
       return false;
     }
 
+    LOG_PRINT_L2("[on_get_blocks]: Enumerating over blocks ....");
     for (auto& b : bs)
     {
-      res.blocks.resize(res.blocks.size()+1);
+      res.blocks.resize(res.blocks.size() + 1);
       res.blocks.back().block = block_to_blob(b.first->bl);
       CHECK_AND_ASSERT_MES(b.third.get(), false, "Internal error on handling COMMAND_RPC_GET_BLOCKS_FAST: b.third is empty, ie coinbase info is not prepared");
       res.blocks.back().coinbase_global_outs = b.third->m_global_output_indexes;
       res.blocks.back().tx_global_outs.resize(b.second.size());
       size_t i = 0;
-      
-      BOOST_FOREACH(auto& t, b.second)
+
+      BOOST_FOREACH(auto & t, b.second)
       {
         res.blocks.back().txs.push_back(tx_to_blob(t->tx));
         res.blocks.back().tx_global_outs[i].v = t->m_global_output_indexes;
         i++;
       }
     }
-
+    LOG_PRINT_L2("[on_get_blocks]: Finished");
     res.status = API_RETURN_CODE_OK;
     return true;
   }

src/rpc/core_rpc_server.h

@@ -16,7 +16,7 @@
 #include "currency_protocol/currency_protocol_handler.h"
 #include "currency_core/bc_offers_service.h"
 
-
+#define RPC_SERVER_DEFAULT_THREADS_NUM                              10
   
 #undef LOG_DEFAULT_CHANNEL 
 #define LOG_DEFAULT_CHANNEL "rpc"