Merge branch 'libmdbx' into develop

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "contrib/miniupnp"]
 	path = contrib/miniupnp
 	url = https://github.com/miniupnp/miniupnp
+[submodule "contrib\\db\\libmdbx"]
+	path = contrib/db/libmdbx
+	url = https://github.com/leo-yuriev/libmdbx.git

CMakeLists.txt

@@ -37,7 +37,8 @@ endif()
 
 set(USE_PCH FALSE CACHE BOOL "Use shared precompiled headers for MSVC")
 
-include_directories(src contrib/eos_portable_archive contrib/db/liblmdb contrib contrib/epee/include "${CMAKE_BINARY_DIR}/version" "${CMAKE_BINARY_DIR}/contrib/zlib")
+include_directories(src contrib/eos_portable_archive contrib contrib/epee/include "${CMAKE_BINARY_DIR}/version" "${CMAKE_BINARY_DIR}/contrib/zlib")
+
 add_definitions(-DSTATICLIB)
 
 set(TESTNET FALSE CACHE BOOL "Compile for testnet")
@@ -213,7 +214,12 @@ else()
 endif()
 
 set(BUILD_TESTS FALSE CACHE BOOL "Build Zano tests")
+set(DISABLE_MDBX FALSE CACHE BOOL "Exclude mdbx from build(need for a first time)") 
+if(NOT DISABLE_MDBX)
+   add_definitions(-DENABLED_ENGINE_MDBX)
+endif()
 
+  
 add_subdirectory(contrib)
 add_subdirectory(src)
 

contrib/CMakeLists.txt

@@ -7,13 +7,19 @@ add_subdirectory(db)
 add_subdirectory(ethereum)
 
 
-set_property(TARGET upnpc-static PROPERTY FOLDER "contrib/miniupnp")
+set_property(TARGET libminiupnpc-static PROPERTY FOLDER "contrib")
 set_property(TARGET zlibstatic PROPERTY FOLDER "contrib")
+set_property(TARGET mdbx PROPERTY FOLDER "contrib")
 set_property(TARGET lmdb PROPERTY FOLDER "contrib")
+set_property(TARGET upnpc-static mdbx_chk mdbx_copy mdbx_dump mdbx_load mdbx_stat minigzip zlib example PROPERTY FOLDER "unused")
+if(MSVC)
+  set_property(TARGET ntdll_extra_target PROPERTY FOLDER "unused")
+endif()
 
 
 if(MSVC)
-  set_property(TARGET upnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267")
+  set_property(TARGET upnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4244 /wd4267")
+  set_property(TARGET zlibstatic APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4267 /wd4267")
 else()
   set_property(TARGET upnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value -Wno-implicit-fallthrough -Wno-discarded-qualifiers ")
   set_property(TARGET zlibstatic APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value -Wno-implicit-fallthrough -Wno-discarded-qualifiers ")

contrib/db/CMakeLists.txt

@@ -1,7 +1,13 @@
-add_subdirectory(liblmdb)
-if(MSVC)
-   target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172)
-else()
-  # Warnings as used by LMDB itself (LMDB_0.9.23)
-  target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized)
-endif()
+
+  message("DB ENGINE: lmdb")  
+  add_subdirectory(liblmdb)
+  if(MSVC)
+     target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172)
+  else()
+     # Warnings as used by LMDB itself (LMDB_0.9.23)
+     target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized)
+  endif()
+  if(NOT DISABLE_MDBX)
+     message("DB ENGINE: mdbx")  
+     add_subdirectory(libmdbx)
+  endif()

contrib/db/libmdbx

@@ -0,0 +1 @@
+Subproject commit a0ec89e46833e61d9b93850e85157d4bc93db1b2

src/CMakeLists.txt

@@ -116,7 +116,7 @@ add_library(wallet ${WALLET})
 add_dependencies(wallet version ${PCH_LIB_NAME})
 ENABLE_SHARED_PCH(WALLET)
 
-target_link_libraries(currency_core lmdb)
+target_link_libraries(currency_core lmdb mdbx)
 
 add_executable(daemon ${DAEMON} ${P2P} ${CURRENCY_PROTOCOL})
 add_dependencies(daemon version)

src/common/command_line.cpp

@@ -30,4 +30,5 @@ namespace command_line
   const arg_descriptor<bool>        arg_disable_stop_if_time_out_of_sync = { "disable-stop-if-time-out-of-sync", "Do not stop the daemon if serious time synchronization problem is detected", false, true };
   const arg_descriptor<bool>        arg_disable_stop_on_low_free_space   = { "disable-stop-on-low-free-space", "Do not stop the daemon if free space at data dir is critically low", false, true };
   const arg_descriptor<bool>        arg_enable_offers_service = { "enable-offers-service", "Enables marketplace feature", false, false};
+  const arg_descriptor<std::string> arg_db_engine =             { "db-engine", "Specify database engine for storage. May be \"lmdb\"(default) or \"mdbx\"", ARG_DB_ENGINE_LMDB, false };
 }

src/common/command_line.h

@@ -172,6 +172,9 @@ namespace command_line
     return get_arg<bool, false>(vm, arg);
   }
 
+#define ARG_DB_ENGINE_LMDB   "lmdb"
+#define ARG_DB_ENGINE_MDBX   "mdbx"
+
 
   extern const arg_descriptor<bool>        arg_help;
   extern const arg_descriptor<bool>        arg_version;
@@ -188,4 +191,5 @@ namespace command_line
   extern const arg_descriptor<bool>        arg_disable_stop_if_time_out_of_sync;
   extern const arg_descriptor<bool>        arg_disable_stop_on_low_free_space;
   extern const arg_descriptor<bool>        arg_enable_offers_service;
+  extern const arg_descriptor<std::string> arg_db_engine;
 }

src/common/db_abstract_accessor.h

@@ -95,7 +95,7 @@ namespace tools
       {
         close();
       }
-
+      void reset_backend(std::shared_ptr<i_db_backend> backend) { m_backend = backend; }
       performance_data& get_performance_data_for_handle(container_handle h) const  { return m_performance_data_map[h]; }
       performance_data& get_performance_data_global() const { return m_gperformance_data; }
 

src/common/db_backend_base.h

@@ -45,6 +45,7 @@ namespace tools
       virtual bool clear(container_handle h) = 0;
       virtual bool enumerate(container_handle h, i_db_callback* pcb)=0;
       virtual bool get_stat_info(stat_info& si) = 0;
+      virtual const char* name()=0;
       virtual ~i_db_backend(){};
     };
   }

src/common/db_backend_lmdb.cpp

@@ -379,6 +379,10 @@ namespace tools
       }
       return true;
     }
+    const char* lmdb_db_backend::name()
+    {
+      return "lmdb";
+    }
   }
 }
 

src/common/db_backend_lmdb.h

@@ -4,6 +4,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #pragma once
+
 #include  <thread>
 
 #include "include_base_utils.h"
@@ -53,6 +54,7 @@ namespace tools
       bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs);
       bool enumerate(container_handle h, i_db_callback* pcb);
       bool get_stat_info(tools::db::stat_info& si);
+      const char* name();
       //-------------------------------------------------------------------------------------
       bool have_tx();
       MDB_txn* get_current_tx();

src/common/db_backend_mdbx.cpp

@@ -0,0 +1,402 @@
+// Copyright (c) 2014-2019 Zano Project
+// Copyright (c) 2014-2018 The Louisdor Project 
+// Distributed under the MIT/X11 software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#ifdef ENABLED_ENGINE_MDBX
+#include "db_backend_mdbx.h"
+#include "misc_language.h"
+#include "string_coding.h"
+#include "profile_tools.h"
+#include "util.h"
+
+
+#define BUF_SIZE 1024
+
+#define CHECK_AND_ASSERT_MESS_MDBX_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDBX_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess);
+#define CHECK_AND_ASSERT_THROW_MESS_MDBX_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDBX_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess);
+#define ASSERT_MES_AND_THROW_MDBX(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess);
+
+#undef LOG_DEFAULT_CHANNEL 
+#define LOG_DEFAULT_CHANNEL "mdbx"
+// 'mdbx' channel is disabled by default
+
+namespace tools
+{
+  namespace db
+  {
+    mdbx_db_backend::mdbx_db_backend() : m_penv(AUTO_VAL_INIT(m_penv))  
+    {
+
+    }
+    mdbx_db_backend::~mdbx_db_backend()
+    {
+      NESTED_TRY_ENTRY();
+
+      close();
+
+      NESTED_CATCH_ENTRY(__func__);
+    }
+
+    bool mdbx_db_backend::open(const std::string& path_, uint64_t cache_sz)
+    {
+      int res = 0;
+      res = mdbx_env_create(&m_penv);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_create");
+      
+      res = mdbx_env_set_maxdbs(m_penv, 15);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_maxdbs");
+
+      intptr_t size_lower = 0;
+      intptr_t size_now = -1;            //don't change current database size
+      intptr_t size_upper = 0x10000000000;          //don't set db file size limit
+      intptr_t growth_step = 0x40000000; //increment step 1GB
+      intptr_t shrink_threshold = -1;
+      intptr_t pagesize = 0x00001000;   //4kb
+      res = mdbx_env_set_geometry(m_penv, size_lower, size_now, size_upper, growth_step, shrink_threshold, pagesize);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_mapsize");
+      
+      m_path = path_;
+#ifdef WIN32
+      m_path = epee::string_encoding::convert_ansii_to_utf8(m_path);
+#endif
+
+      CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path);
+
+      res = mdbx_env_open(m_penv, m_path.c_str(), MDBX_NORDAHEAD , 0644);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_open, m_path=" << m_path);
+      
+      return true;
+    }
+
+    bool mdbx_db_backend::open_container(const std::string& name, container_handle& h)
+    {
+
+      MDBX_dbi dbi = AUTO_VAL_INIT(dbi);
+      begin_transaction();
+      int res = mdbx_dbi_open(get_current_tx(), name.c_str(), MDBX_CREATE, &dbi);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_dbi_open with container name: " << name);
+      commit_transaction();
+      h = static_cast<container_handle>(dbi);
+      return true;
+    }
+
+    bool mdbx_db_backend::close()
+    {
+      {
+        std::lock_guard<boost::recursive_mutex> lock(m_cs);
+        for (auto& tx_thread : m_txs)
+        {
+          for (auto txe : tx_thread.second)
+          {
+            int res = mdbx_txn_commit(txe.ptx);
+            if (res != MDBX_SUCCESS)
+            {
+              LOG_ERROR("[DB ERROR]: On close tranactions: " << mdbx_strerror(res));
+            }
+          }
+        }
+
+        m_txs.clear();
+      }
+      if (m_penv)
+      {
+        mdbx_env_close(m_penv);
+        m_penv = nullptr;
+      }
+      return true;
+    }
+
+    bool mdbx_db_backend::begin_transaction(bool read_only)
+    {
+      if (!read_only)
+      {
+        LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3);
+        CRITICAL_SECTION_LOCK(m_write_exclusive_lock);
+      }
+      PROFILE_FUNC("mdbx_db_backend::begin_transaction");
+      {
+        std::lock_guard<boost::recursive_mutex> lock(m_cs);
+        CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed");
+        transactions_list& rtxlist = m_txs[std::this_thread::get_id()];
+        MDBX_txn* pparent_tx = nullptr;
+        MDBX_txn* p_new_tx = nullptr;
+        bool parent_read_only = false;
+        if (rtxlist.size())
+        {
+          pparent_tx = rtxlist.back().ptx;
+          parent_read_only = rtxlist.back().read_only;
+        }
+
+
+        if (pparent_tx && read_only)
+        {
+          ++rtxlist.back().count;
+        }
+        else
+        {
+          int res = 0;
+          unsigned int flags = 0;
+          if (read_only)
+            flags += MDBX_RDONLY;
+
+          //don't use parent tx in write transactions if parent tx was read-only (restriction in mdbx) 
+          //see "Nested transactions: Max 1 child, write txns only, no writemap"
+          if (pparent_tx && parent_read_only)
+            pparent_tx = nullptr;
+
+          CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed");
+          res = mdbx_txn_begin(m_penv, pparent_tx, flags, &p_new_tx);
+          if(res != MDBX_SUCCESS)
+          {
+            //Important: if mdbx_txn_begin is failed need to unlock previously locked mutex
+            CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock);
+            //throw exception to avoid regular code execution 
+            ASSERT_MES_AND_THROW_MDBX(res, "Unable to mdbx_txn_begin");
+          }
+
+          rtxlist.push_back(tx_entry());
+          rtxlist.back().count = read_only ? 1 : 0;
+          rtxlist.back().ptx = p_new_tx;
+          rtxlist.back().read_only = read_only;
+        }
+      }
+
+
+      LOG_PRINT_L4("[DB] Transaction started");
+      return true;
+    }
+
+    MDBX_txn* mdbx_db_backend::get_current_tx()
+    {
+      std::lock_guard<boost::recursive_mutex> lock(m_cs);
+      auto& rtxlist = m_txs[std::this_thread::get_id()];
+      CHECK_AND_ASSERT_MES(rtxlist.size(), nullptr, "Unable to find active tx for thread " << std::this_thread::get_id());
+      return rtxlist.back().ptx;
+    }
+
+    bool mdbx_db_backend::pop_tx_entry(tx_entry& txe)
+    {
+      std::lock_guard<boost::recursive_mutex> lock(m_cs);
+      auto it = m_txs.find(std::this_thread::get_id());
+      CHECK_AND_ASSERT_MES(it != m_txs.end(), false, "[DB] Unable to find id cor current thread");
+      CHECK_AND_ASSERT_MES(it->second.size(), false, "[DB] No active tx for current thread");
+
+      txe = it->second.back();
+
+      if (it->second.back().read_only &&  it->second.back().count == 0)
+      {
+        LOG_ERROR("Internal db tx state error: read_only and count readers == 0");
+      }
+
+      if ((it->second.back().read_only && it->second.back().count < 2) || (!it->second.back().read_only && it->second.back().count < 1))
+      {
+        it->second.pop_back();
+        if (!it->second.size())
+          m_txs.erase(it);       
+      }
+      else
+      {
+        --it->second.back().count;
+      }
+      return true;
+    }
+
+    bool mdbx_db_backend::commit_transaction()
+    {
+      PROFILE_FUNC("mdbx_db_backend::commit_transaction");
+      {
+        tx_entry txe = AUTO_VAL_INIT(txe);
+        bool r = pop_tx_entry(txe);
+        CHECK_AND_ASSERT_MES(r, false, "Unable to pop_tx_entry");
+          
+        if (txe.count == 0 || (txe.read_only && txe.count == 1))
+        {
+          int res = 0;
+          res = mdbx_txn_commit(txe.ptx);
+          CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_txn_commit (error " << res << ")");
+          if (!txe.read_only && !txe.count)
+          {
+            CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock);
+            LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED", LOG_LEVEL_3);
+          }
+        } 
+      }
+      LOG_PRINT_L4("[DB] Transaction committed");
+      return true;
+    }
+    
+    void mdbx_db_backend::abort_transaction()
+    {
+      {
+        tx_entry txe = AUTO_VAL_INIT(txe);
+        bool r = pop_tx_entry(txe);
+        CHECK_AND_ASSERT_MES(r, void(), "Unable to pop_tx_entry");
+        if (txe.count == 0 || (txe.read_only && txe.count == 1))
+        {
+          mdbx_txn_abort(txe.ptx);
+          if (!txe.read_only && !txe.count)
+          {
+            CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock);
+            LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED(ABORTED)", LOG_LEVEL_3);
+          }
+        }
+          
+
+      }
+      LOG_PRINT_L4("[DB] Transaction aborted");
+    }
+
+    bool mdbx_db_backend::erase(container_handle h, const char* k, size_t ks)
+    {
+      int res = 0;
+      MDBX_val key = AUTO_VAL_INIT(key);
+      key.iov_base = (void*)k;
+      key.iov_len = ks;
+
+      res = mdbx_del(get_current_tx(), static_cast<MDBX_dbi>(h), &key, nullptr);
+      if (res == MDBX_NOTFOUND)
+        return false;
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_del");
+      return true;
+    }
+
+    bool mdbx_db_backend::have_tx()
+    {
+      std::lock_guard<boost::recursive_mutex> lock(m_cs);
+      auto it = m_txs.find(std::this_thread::get_id());
+      if (it == m_txs.end())
+        return false;
+      return it->second.size() ? true : false;
+    }
+
+    bool mdbx_db_backend::get(container_handle h, const char* k, size_t ks, std::string& res_buff)
+    {
+      PROFILE_FUNC("mdbx_db_backend::get");
+      int res = 0;
+      MDBX_val key = AUTO_VAL_INIT(key);
+      MDBX_val data = AUTO_VAL_INIT(data);
+      key.iov_base = (void*)k;
+      key.iov_len = ks;
+      bool need_to_commit = false;
+      if (!have_tx())
+      {
+        need_to_commit = true;
+        begin_transaction(true);
+      }
+
+      res = mdbx_get(get_current_tx(), static_cast<MDBX_dbi>(h), &key, &data);
+
+      if (need_to_commit)
+        commit_transaction();
+
+      if (res == MDBX_NOTFOUND)
+        return false;
+
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_get, h: " << h << ", ks: " << ks);
+      res_buff.assign((const char*)data.iov_base, data.iov_len);
+      return true;
+    }
+
+    bool mdbx_db_backend::clear(container_handle h)
+    {
+      int res = mdbx_drop(get_current_tx(), static_cast<MDBX_dbi>(h), 0);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_drop");
+      return true;
+    }
+    
+    uint64_t mdbx_db_backend::size(container_handle h)
+    {
+      PROFILE_FUNC("mdbx_db_backend::size");
+      MDBX_stat container_stat = AUTO_VAL_INIT(container_stat);
+      bool need_to_commit = false;
+      if (!have_tx())
+      {
+        need_to_commit = true;
+        begin_transaction(true);
+      }
+      int res = mdbx_dbi_stat(get_current_tx(), static_cast<MDBX_dbi>(h), &container_stat, sizeof(MDBX_stat));
+      if (need_to_commit)
+        commit_transaction();
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_stat");
+      return container_stat.ms_entries;
+    }
+
+    bool mdbx_db_backend::set(container_handle h, const char* k, size_t ks, const char* v, size_t vs)
+    {
+      PROFILE_FUNC("mdbx_db_backend::set");
+      int res = 0;
+      MDBX_val key = AUTO_VAL_INIT(key);
+      MDBX_val data = AUTO_VAL_INIT(data);
+      key.iov_base = (void*)k;
+      key.iov_len = ks;
+      data.iov_base = (void*)v;
+      data.iov_len = vs;
+
+      res = mdbx_put(get_current_tx(), static_cast<MDBX_dbi>(h), &key, &data, 0);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_put");
+      return true;
+    }
+    bool mdbx_db_backend::enumerate(container_handle h, i_db_callback* pcb)
+    {
+      CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate");
+      MDBX_val key = AUTO_VAL_INIT(key);
+      MDBX_val data = AUTO_VAL_INIT(data);
+
+      bool need_to_commit = false;
+      if (!have_tx())
+      {
+        need_to_commit = true;
+        begin_transaction(true);
+      }
+      MDBX_cursor* cursor_ptr = nullptr;
+      int res = mdbx_cursor_open(get_current_tx(), static_cast<MDBX_dbi>(h), &cursor_ptr);
+      CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_cursor_open");
+      CHECK_AND_ASSERT_MES(cursor_ptr, false, "cursor_ptr is null after mdbx_cursor_open");
+
+      uint64_t count = 0;
+      do
+      {
+        int res = mdbx_cursor_get(cursor_ptr, &key, &data, MDBX_NEXT);
+        if (res == MDBX_NOTFOUND)
+          break;
+        if (!pcb->on_enum_item(count, key.iov_base, key.iov_len, data.iov_base, data.iov_len))
+          break;
+        count++;
+      } while (cursor_ptr);
+
+      mdbx_cursor_close(cursor_ptr);
+      if (need_to_commit)
+        commit_transaction();
+      return true;
+    }
+
+    bool mdbx_db_backend::get_stat_info(tools::db::stat_info& si)
+    {
+      si = AUTO_VAL_INIT_T(tools::db::stat_info);
+
+      MDBX_envinfo ei = AUTO_VAL_INIT(ei);
+      mdbx_env_info(m_penv, &ei, sizeof(MDBX_envinfo));
+      si.map_size = ei.mi_mapsize;
+      
+      std::lock_guard<boost::recursive_mutex> lock(m_cs);
+      for (auto& e : m_txs)
+      {
+        for (auto& pr : e.second)
+        {
+          ++si.tx_count;
+          if(!pr.read_only)
+            ++si.write_tx_count;
+        }
+      }
+      return true;
+    }
+    const char* mdbx_db_backend::name()
+    {
+      return "mdbx";
+    }
+  }
+}
+
+#undef LOG_DEFAULT_CHANNEL 
+#define LOG_DEFAULT_CHANNEL NULL
+#endif

src/common/db_backend_mdbx.h

@@ -0,0 +1,66 @@
+// Copyright (c) 2014-2019 Zano Project
+// Copyright (c) 2014-2018 The Louisdor Project 
+// Distributed under the MIT/X11 software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#pragma once
+#ifdef ENABLED_ENGINE_MDBX
+#include  <thread>
+
+#include "include_base_utils.h"
+
+#include "db_backend_base.h"
+#include "db/libmdbx/mdbx.h"
+
+
+namespace tools
+{
+  namespace db
+  {
+    class mdbx_db_backend : public i_db_backend
+    {
+
+      struct tx_entry
+      {
+        MDBX_txn* ptx;
+        bool read_only; // needed for thread-top transaction, for figure out if we need to unlock exclusive access
+        size_t count;   //count of read-only nested emulated transactions
+      };
+      typedef std::list<tx_entry> transactions_list;
+
+
+      std::string m_path;
+      MDBX_env *m_penv;
+
+      boost::recursive_mutex m_cs;
+      boost::recursive_mutex m_write_exclusive_lock;
+      std::map<std::thread::id, transactions_list> m_txs; // size_t -> count of nested read_only transactions
+      bool pop_tx_entry(tx_entry& txe);
+    public:
+      mdbx_db_backend();
+      ~mdbx_db_backend();
+
+      //----------------- i_db_backend -----------------------------------------------------
+      bool close();
+      bool begin_transaction(bool read_only = false);
+      bool commit_transaction();
+      void abort_transaction();
+      bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE);
+      bool open_container(const std::string& name, container_handle& h);
+      bool erase(container_handle h, const char* k, size_t s);
+      bool get(container_handle h, const char* k, size_t s, std::string& res_buff);
+      bool clear(container_handle h);
+      uint64_t size(container_handle h);
+      bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs);
+      bool enumerate(container_handle h, i_db_callback* pcb);
+      bool get_stat_info(tools::db::stat_info& si);
+      const char* name();
+      //-------------------------------------------------------------------------------------
+      bool have_tx();
+      MDBX_txn* get_current_tx();
+
+    };
+    
+  }
+}
+#endif

src/common/db_backend_selector.h

@@ -0,0 +1,41 @@
+// Copyright (c) 2014-2019 Zano Project
+// Copyright (c) 2014-2018 The Louisdor Project
+// Distributed under the MIT/X11 software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#pragma once
+
+#include "db_backend_lmdb.h"
+#include "db_backend_mdbx.h"
+#include "common/command_line.h"
+#include "common/db_abstract_accessor.h"
+
+namespace tools
+{
+  namespace db
+  {
+    inline
+      bool select_db_engine_from_arg(const boost::program_options::variables_map& vm, tools::db::basic_db_accessor& rdb)
+    {
+      if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_LMDB)
+      {
+        rdb.reset_backend(std::shared_ptr<tools::db::i_db_backend>(new tools::db::lmdb_db_backend));
+      }
+      else if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_MDBX)
+      {
+#ifdef ENABLED_ENGINE_MDBX
+        rdb.reset_backend(std::shared_ptr<tools::db::i_db_backend>(new tools::db::mdbx_db_backend));
+#else
+        LOG_PRINT_L0(" DB ENGINE: " << ARG_DB_ENGINE_MDBX << " is not suported by this build(see DISABLE_MDBX cmake option), STOPPING");
+        return false;
+#endif 
+      }
+      else
+      {
+        LOG_PRINT_RED_L0(" UNKNOWN DB ENGINE: " << command_line::get_arg(vm, command_line::arg_db_engine) << ", STOPPING");
+        return false;
+      }
+      return true;
+    }
+  }
+}

src/common/ntp.cpp

@@ -201,7 +201,7 @@ namespace tools
 
       udp_blocking_client ubc(sender_endpoint, socket, io_service);
       boost::system::error_code ec;
-      size_t len = ubc.receive(boost::asio::buffer(&packet_received, sizeof packet_received), boost::posix_time::seconds(timeout_sec), ec);
+      size_t len = ubc.receive(boost::asio::buffer(&packet_received, sizeof packet_received), boost::posix_time::seconds(static_cast<long>(timeout_sec)), ec);
       if (ec)
       {
         LOG_PRINT_L3("NTP: get_ntp_time(" << host_name << "): boost error: " << ec.message());

src/currency_core/bc_attachments_service_manager.h

@@ -27,7 +27,7 @@ namespace currency
   class bc_attachment_services_manager
   {
   public:
-    bc_attachment_services_manager(i_core_event_handler* pcore_event_handler) : m_pcore_event_handler(pcore_event_handler), m_core_runtime_config(get_default_core_runtime_config())
+    bc_attachment_services_manager(/* i_core_event_handler* pcore_event_handler*/) : /*m_pcore_event_handler(pcore_event_handler),*/ m_core_runtime_config(get_default_core_runtime_config())
     {}
   
 
@@ -43,7 +43,7 @@ namespace currency
   
   private:
     std::map<std::string, i_bc_service*> m_services;
-    i_core_event_handler* m_pcore_event_handler;
+    //i_core_event_handler* m_pcore_event_handler;
     core_runtime_config m_core_runtime_config;
 
   };

src/currency_core/blockchain_storage.cpp

@@ -14,7 +14,7 @@
 
 #include "include_base_utils.h"
 
-#include "common/db_backend_lmdb.h"
+#include "common/db_backend_selector.h"
 #include "common/command_line.h"
 
 #include "blockchain_storage.h"
@@ -30,7 +30,6 @@
 #include "crypto/hash.h"
 #include "miner_common.h"
 #include "storages/portable_storage_template_helper.h"
-#include "common/db_backend_lmdb.h"
 #include "basic_pow_helpers.h"
 #include "version.h"
 
@@ -70,7 +69,7 @@ using namespace currency;
 #endif
 #define BLOCK_POS_STRICT_SEQUENCE_LIMIT                               20
 
-
+#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX                     "_v1"
 
 DISABLE_VS_WARNINGS(4267)
 
@@ -81,7 +80,7 @@ namespace
 }
 
 //------------------------------------------------------------------
-blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::shared_ptr<tools::db::i_db_backend>(new tools::db::lmdb_db_backend), m_rw_lock),
+blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m_rw_lock),
                                                                  m_db_blocks(m_db),
                                                                  m_db_blocks_index(m_db),
                                                                  m_db_transactions(m_db),
@@ -104,7 +103,6 @@ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::share
                                                                  m_core_runtime_config(get_default_core_runtime_config()),
                                                                  //m_bei_stub(AUTO_VAL_INIT(m_bei_stub)),
                                                                  m_event_handler(&m_event_handler_stub), 
-                                                                 m_services_mgr(nullptr), 
                                                                  m_interprocess_locker_file(0), 
                                                                  m_current_fee_median(0), 
                                                                  m_current_fee_median_effective_index(0), 
@@ -207,7 +205,13 @@ bool blockchain_storage::validate_instance(const std::string& path)
 bool blockchain_storage::init(const std::string& config_folder, const boost::program_options::variables_map& vm)
 {
 //  CRITICAL_REGION_LOCAL(m_read_lock);
-
+  if (!select_db_engine_from_arg(vm, m_db))
+  {
+    LOG_PRINT_RED_L0("Failed to select db engine");
+    return false;
+  }
+  LOG_PRINT_L0("DB ENGINE USED BY CORE: " << m_db.get_backend()->name());
+  
   if (!validate_instance(config_folder))
   {
     LOG_ERROR("Failed to initialize instance");
@@ -230,8 +234,8 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro
     LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0);
     boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path));
   }
-
-  const std::string db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME;
+  ;
+  const std::string db_folder_path = m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + m_db.get_backend()->name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX;
   LOG_PRINT_L0("Loading blockchain from " << db_folder_path);
 
   bool db_opened_okay = false;

src/currency_core/currency_config.h

@@ -189,8 +189,11 @@
 
 #define CURRENCY_POOLDATA_FOLDERNAME_OLD                "poolstate"
 #define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD          "blockchain"
-#define CURRENCY_POOLDATA_FOLDERNAME                    "poolstate_lmdb_v1"
-#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME              "blockchain_lmdb_v1"
+
+
+#define CURRENCY_POOLDATA_FOLDERNAME_PREFIX                    "poolstate_"
+#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX              "blockchain_" 
+
 #define P2P_NET_DATA_FILENAME                           "p2pstate.bin"
 #define MINER_CONFIG_FILENAME                           "miner_conf.json"
 #define GUI_SECURE_CONFIG_FILENAME                      "gui_secure_conf.bin"

src/currency_core/currency_core.cpp

@@ -142,7 +142,7 @@ namespace currency
     uint64_t available_space = 0;
     CHECK_AND_ASSERT_MES(!check_if_free_space_critically_low(&available_space), false, "free space in data folder is critically low: " << std::fixed << available_space / (1024 * 1024) << " MB");
 
-    r = m_mempool.init(m_config_folder);
+    r = m_mempool.init(m_config_folder, vm);
     CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool");
 
     r = m_blockchain_storage.init(m_config_folder, vm);

src/currency_core/miner.cpp

@@ -41,7 +41,7 @@ namespace currency
 
 
   miner::miner(i_miner_handler* phandler, blockchain_storage& bc):m_stop(1),
-    m_bc(bc),
+    //m_bc(bc),
     m_template(boost::value_initialized<block>()),
     m_template_no(0),
     m_diffic(0),

src/currency_core/miner.h

@@ -105,7 +105,7 @@ namespace currency
     std::list<boost::thread> m_threads;
     ::critical_section m_threads_lock;
     i_miner_handler* m_phandler;
-    blockchain_storage& m_bc;
+    //blockchain_storage& m_bc;
     account_public_address m_mine_address;
     math_helper::once_a_time_seconds<5> m_update_block_template_interval;
     math_helper::once_a_time_seconds<2> m_update_merge_hr_interval;

src/currency_core/tx_pool.cpp

@@ -9,7 +9,7 @@
 #include <unordered_set>
 #include <vector>
 
-#include "common/db_backend_lmdb.h"
+#include "common/db_backend_selector.h"
 #include "tx_pool.h"
 #include "currency_boost_serialization.h"
 #include "currency_core/currency_config.h"
@@ -32,6 +32,8 @@ DISABLE_VS_WARNINGS(4244 4345 4503) //'boost::foreach_detail_::or_' : decorated
 #define TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 92 // DON'T CHANGE THIS, if you need to resync db! Change TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION instead!
 #define TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION      BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION + 1
 
+#define CURRENCY_POOLDATA_FOLDERNAME_SUFFIX               "_v1"
+
 #define CONFLICT_KEY_IMAGE_SPENT_DEPTH_TO_REMOVE_TX_FROM_POOL 50 // if there's a conflict in key images between tx in the pool and in the blockchain this much depth in required to remove correspongin tx from pool
 
 #undef LOG_DEFAULT_CHANNEL 
@@ -44,7 +46,7 @@ namespace currency
   tx_memory_pool::tx_memory_pool(blockchain_storage& bchs, i_currency_protocol* pprotocol) :
     m_blockchain(bchs),
     m_pprotocol(pprotocol),
-    m_db(std::shared_ptr<tools::db::i_db_backend>(new tools::db::lmdb_db_backend), m_dummy_rw_lock),
+    m_db(nullptr, m_dummy_rw_lock),
     m_db_transactions(m_db),
     m_db_black_tx_list(m_db),
     m_db_solo_options(m_db), 
@@ -1141,8 +1143,15 @@ namespace currency
     m_db.commit_transaction();
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::init(const std::string& config_folder)
+  bool tx_memory_pool::init(const std::string& config_folder, const boost::program_options::variables_map& vm)
   {
+    if (!select_db_engine_from_arg(vm, m_db))
+    {
+      LOG_PRINT_RED_L0("Failed to select db engine");
+      return false;
+    }
+    LOG_PRINT_L0("DB ENGINE USED BY POOL: " << m_db.get_backend()->name());
+
     m_config_folder = config_folder;
 
     uint64_t cache_size_l1 = CACHE_SIZE;
@@ -1156,7 +1165,8 @@ namespace currency
       boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path));
     }
 
-    const std::string db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME;
+    const std::string db_folder_path = m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + m_db.get_backend()->name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX;
+    
     LOG_PRINT_L0("Loading blockchain from " << db_folder_path << "...");
 
     bool db_opened_okay = false;

src/currency_core/tx_pool.h

@@ -20,6 +20,7 @@ using namespace epee;
 #include "math_helper.h"
 
 #include "common/db_abstract_accessor.h"
+#include "common/command_line.h"
 
 #include "currency_format_utils.h"
 #include "verification_context.h"
@@ -114,7 +115,7 @@ namespace currency
     void clear();
 
     // load/store operations
-    bool init(const std::string& config_folder);
+    bool init(const std::string& config_folder, const boost::program_options::variables_map& vm);
     bool deinit();
     bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height);
     bool get_transactions(std::list<transaction>& txs) const;

src/currency_protocol/currency_protocol_handler.inl

@@ -800,7 +800,7 @@ namespace currency
     std::vector<int64_t> time_deltas_copy(m_time_deltas.begin(), m_time_deltas.end());
 
     m_last_median2local_time_difference = epee::misc_utils::median(time_deltas_copy);
-    LOG_PRINT_MAGENTA("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << ")", m_last_median2local_time_difference >= 3 ? LOG_LEVEL_2 : LOG_LEVEL_3);
+    LOG_PRINT_MAGENTA("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << ")", ((m_last_median2local_time_difference >= 3) ? LOG_LEVEL_2 : LOG_LEVEL_3));
     if (std::abs(m_last_median2local_time_difference) > TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE)
     {
       int64_t ntp_time = tools::get_ntp_time();

src/daemon/daemon.cpp

@@ -147,6 +147,7 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, command_line::arg_disable_stop_if_time_out_of_sync);
   command_line::add_arg(desc_cmd_sett, command_line::arg_disable_stop_on_low_free_space);
   command_line::add_arg(desc_cmd_sett, command_line::arg_enable_offers_service);
+  command_line::add_arg(desc_cmd_sett, command_line::arg_db_engine);  
 
 
   arg_market_disable.default_value = true;

src/gui/qt-daemon/application/daemon_backend.cpp

@@ -120,6 +120,7 @@ bool daemon_backend::init(int argc, char* argv[], view::i_view* pview_handler)
   command_line::add_arg(desc_cmd_sett, command_line::arg_log_level);
   command_line::add_arg(desc_cmd_sett, command_line::arg_console);
   command_line::add_arg(desc_cmd_sett, command_line::arg_show_details);
+  command_line::add_arg(desc_cmd_sett, command_line::arg_db_engine);
   command_line::add_arg(desc_cmd_sett, arg_alloc_win_console);
   command_line::add_arg(desc_cmd_sett, arg_html_folder);
   command_line::add_arg(desc_cmd_sett, arg_xcode_stub);