diff --git a/.gitmodules b/.gitmodules index 349bf872..63349027 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "contrib/miniupnp"] path = contrib/miniupnp url = https://github.com/miniupnp/miniupnp +[submodule "contrib\\db\\libmdbx"] + path = contrib/db/libmdbx + url = https://github.com/leo-yuriev/libmdbx.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e2a4a79..76b5f69b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,8 @@ endif() set(USE_PCH FALSE CACHE BOOL "Use shared precompiled headers for MSVC") -include_directories(src contrib/eos_portable_archive contrib/db/liblmdb contrib contrib/epee/include "${CMAKE_BINARY_DIR}/version" "${CMAKE_BINARY_DIR}/contrib/zlib") +include_directories(src contrib/eos_portable_archive contrib contrib/epee/include "${CMAKE_BINARY_DIR}/version" "${CMAKE_BINARY_DIR}/contrib/zlib") + add_definitions(-DSTATICLIB) set(TESTNET FALSE CACHE BOOL "Compile for testnet") @@ -213,7 +214,12 @@ else() endif() set(BUILD_TESTS FALSE CACHE BOOL "Build Zano tests") +set(DISABLE_MDBX FALSE CACHE BOOL "Exclude mdbx from build(need for a first time)") +if(NOT DISABLE_MDBX) + add_definitions(-DENABLED_ENGINE_MDBX) +endif() + add_subdirectory(contrib) add_subdirectory(src) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index f691d096..df753f9a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -7,13 +7,19 @@ add_subdirectory(db) add_subdirectory(ethereum) -set_property(TARGET upnpc-static PROPERTY FOLDER "contrib/miniupnp") +set_property(TARGET libminiupnpc-static PROPERTY FOLDER "contrib") set_property(TARGET zlibstatic PROPERTY FOLDER "contrib") +set_property(TARGET mdbx PROPERTY FOLDER "contrib") set_property(TARGET lmdb PROPERTY FOLDER "contrib") +set_property(TARGET upnpc-static mdbx_chk mdbx_copy mdbx_dump mdbx_load mdbx_stat minigzip zlib example PROPERTY FOLDER "unused") +if(MSVC) + set_property(TARGET ntdll_extra_target PROPERTY FOLDER "unused") +endif() if(MSVC) - set_property(TARGET upnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267") + set_property(TARGET upnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4244 /wd4267") + set_property(TARGET zlibstatic APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4267 /wd4267") else() set_property(TARGET upnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value -Wno-implicit-fallthrough -Wno-discarded-qualifiers ") set_property(TARGET zlibstatic APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value -Wno-implicit-fallthrough -Wno-discarded-qualifiers ") diff --git a/contrib/db/CMakeLists.txt b/contrib/db/CMakeLists.txt index eb5c8415..9497760e 100644 --- a/contrib/db/CMakeLists.txt +++ b/contrib/db/CMakeLists.txt @@ -1,7 +1,13 @@ -add_subdirectory(liblmdb) -if(MSVC) - target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172) -else() - # Warnings as used by LMDB itself (LMDB_0.9.23) - target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized) -endif() + + message("DB ENGINE: lmdb") + add_subdirectory(liblmdb) + if(MSVC) + target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172) + else() + # Warnings as used by LMDB itself (LMDB_0.9.23) + target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized) + endif() + if(NOT DISABLE_MDBX) + message("DB ENGINE: mdbx") + add_subdirectory(libmdbx) + endif() diff --git a/contrib/db/libmdbx b/contrib/db/libmdbx new file mode 160000 index 00000000..a0ec89e4 --- /dev/null +++ b/contrib/db/libmdbx @@ -0,0 +1 @@ +Subproject commit a0ec89e46833e61d9b93850e85157d4bc93db1b2 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d6f96312..ba358dd6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -116,7 +116,7 @@ add_library(wallet ${WALLET}) add_dependencies(wallet version ${PCH_LIB_NAME}) ENABLE_SHARED_PCH(WALLET) -target_link_libraries(currency_core lmdb) +target_link_libraries(currency_core lmdb mdbx) add_executable(daemon ${DAEMON} ${P2P} ${CURRENCY_PROTOCOL}) add_dependencies(daemon version) diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index e6fd1c44..e3407412 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -30,4 +30,5 @@ namespace command_line const arg_descriptor arg_disable_stop_if_time_out_of_sync = { "disable-stop-if-time-out-of-sync", "Do not stop the daemon if serious time synchronization problem is detected", false, true }; const arg_descriptor arg_disable_stop_on_low_free_space = { "disable-stop-on-low-free-space", "Do not stop the daemon if free space at data dir is critically low", false, true }; const arg_descriptor arg_enable_offers_service = { "enable-offers-service", "Enables marketplace feature", false, false}; + const arg_descriptor arg_db_engine = { "db-engine", "Specify database engine for storage. May be \"lmdb\"(default) or \"mdbx\"", ARG_DB_ENGINE_LMDB, false }; } diff --git a/src/common/command_line.h b/src/common/command_line.h index 65edaa39..eb849101 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -172,6 +172,9 @@ namespace command_line return get_arg(vm, arg); } +#define ARG_DB_ENGINE_LMDB "lmdb" +#define ARG_DB_ENGINE_MDBX "mdbx" + extern const arg_descriptor arg_help; extern const arg_descriptor arg_version; @@ -188,4 +191,5 @@ namespace command_line extern const arg_descriptor arg_disable_stop_if_time_out_of_sync; extern const arg_descriptor arg_disable_stop_on_low_free_space; extern const arg_descriptor arg_enable_offers_service; + extern const arg_descriptor arg_db_engine; } diff --git a/src/common/db_abstract_accessor.h b/src/common/db_abstract_accessor.h index 56e517cb..596c7bdd 100644 --- a/src/common/db_abstract_accessor.h +++ b/src/common/db_abstract_accessor.h @@ -95,7 +95,7 @@ namespace tools { close(); } - + void reset_backend(std::shared_ptr backend) { m_backend = backend; } performance_data& get_performance_data_for_handle(container_handle h) const { return m_performance_data_map[h]; } performance_data& get_performance_data_global() const { return m_gperformance_data; } diff --git a/src/common/db_backend_base.h b/src/common/db_backend_base.h index 538abd62..c71ec2a2 100644 --- a/src/common/db_backend_base.h +++ b/src/common/db_backend_base.h @@ -45,6 +45,7 @@ namespace tools virtual bool clear(container_handle h) = 0; virtual bool enumerate(container_handle h, i_db_callback* pcb)=0; virtual bool get_stat_info(stat_info& si) = 0; + virtual const char* name()=0; virtual ~i_db_backend(){}; }; } diff --git a/src/common/db_backend_lmdb.cpp b/src/common/db_backend_lmdb.cpp index 6c913f34..b02a8fe7 100644 --- a/src/common/db_backend_lmdb.cpp +++ b/src/common/db_backend_lmdb.cpp @@ -379,6 +379,10 @@ namespace tools } return true; } + const char* lmdb_db_backend::name() + { + return "lmdb"; + } } } diff --git a/src/common/db_backend_lmdb.h b/src/common/db_backend_lmdb.h index 95c9d8a9..ebe7cf4e 100644 --- a/src/common/db_backend_lmdb.h +++ b/src/common/db_backend_lmdb.h @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once + #include #include "include_base_utils.h" @@ -53,6 +54,7 @@ namespace tools bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs); bool enumerate(container_handle h, i_db_callback* pcb); bool get_stat_info(tools::db::stat_info& si); + const char* name(); //------------------------------------------------------------------------------------- bool have_tx(); MDB_txn* get_current_tx(); diff --git a/src/common/db_backend_mdbx.cpp b/src/common/db_backend_mdbx.cpp new file mode 100644 index 00000000..9ee58ef8 --- /dev/null +++ b/src/common/db_backend_mdbx.cpp @@ -0,0 +1,402 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifdef ENABLED_ENGINE_MDBX +#include "db_backend_mdbx.h" +#include "misc_language.h" +#include "string_coding.h" +#include "profile_tools.h" +#include "util.h" + + +#define BUF_SIZE 1024 + +#define CHECK_AND_ASSERT_MESS_MDBX_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDBX_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); +#define CHECK_AND_ASSERT_THROW_MESS_MDBX_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDBX_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); +#define ASSERT_MES_AND_THROW_MDBX(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); + +#undef LOG_DEFAULT_CHANNEL +#define LOG_DEFAULT_CHANNEL "mdbx" +// 'mdbx' channel is disabled by default + +namespace tools +{ + namespace db + { + mdbx_db_backend::mdbx_db_backend() : m_penv(AUTO_VAL_INIT(m_penv)) + { + + } + mdbx_db_backend::~mdbx_db_backend() + { + NESTED_TRY_ENTRY(); + + close(); + + NESTED_CATCH_ENTRY(__func__); + } + + bool mdbx_db_backend::open(const std::string& path_, uint64_t cache_sz) + { + int res = 0; + res = mdbx_env_create(&m_penv); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_create"); + + res = mdbx_env_set_maxdbs(m_penv, 15); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_maxdbs"); + + intptr_t size_lower = 0; + intptr_t size_now = -1; //don't change current database size + intptr_t size_upper = 0x10000000000; //don't set db file size limit + intptr_t growth_step = 0x40000000; //increment step 1GB + intptr_t shrink_threshold = -1; + intptr_t pagesize = 0x00001000; //4kb + res = mdbx_env_set_geometry(m_penv, size_lower, size_now, size_upper, growth_step, shrink_threshold, pagesize); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_mapsize"); + + m_path = path_; +#ifdef WIN32 + m_path = epee::string_encoding::convert_ansii_to_utf8(m_path); +#endif + + CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path); + + res = mdbx_env_open(m_penv, m_path.c_str(), MDBX_NORDAHEAD , 0644); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_open, m_path=" << m_path); + + return true; + } + + bool mdbx_db_backend::open_container(const std::string& name, container_handle& h) + { + + MDBX_dbi dbi = AUTO_VAL_INIT(dbi); + begin_transaction(); + int res = mdbx_dbi_open(get_current_tx(), name.c_str(), MDBX_CREATE, &dbi); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_dbi_open with container name: " << name); + commit_transaction(); + h = static_cast(dbi); + return true; + } + + bool mdbx_db_backend::close() + { + { + std::lock_guard lock(m_cs); + for (auto& tx_thread : m_txs) + { + for (auto txe : tx_thread.second) + { + int res = mdbx_txn_commit(txe.ptx); + if (res != MDBX_SUCCESS) + { + LOG_ERROR("[DB ERROR]: On close tranactions: " << mdbx_strerror(res)); + } + } + } + + m_txs.clear(); + } + if (m_penv) + { + mdbx_env_close(m_penv); + m_penv = nullptr; + } + return true; + } + + bool mdbx_db_backend::begin_transaction(bool read_only) + { + if (!read_only) + { + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3); + CRITICAL_SECTION_LOCK(m_write_exclusive_lock); + } + PROFILE_FUNC("mdbx_db_backend::begin_transaction"); + { + std::lock_guard lock(m_cs); + CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed"); + transactions_list& rtxlist = m_txs[std::this_thread::get_id()]; + MDBX_txn* pparent_tx = nullptr; + MDBX_txn* p_new_tx = nullptr; + bool parent_read_only = false; + if (rtxlist.size()) + { + pparent_tx = rtxlist.back().ptx; + parent_read_only = rtxlist.back().read_only; + } + + + if (pparent_tx && read_only) + { + ++rtxlist.back().count; + } + else + { + int res = 0; + unsigned int flags = 0; + if (read_only) + flags += MDBX_RDONLY; + + //don't use parent tx in write transactions if parent tx was read-only (restriction in mdbx) + //see "Nested transactions: Max 1 child, write txns only, no writemap" + if (pparent_tx && parent_read_only) + pparent_tx = nullptr; + + CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed"); + res = mdbx_txn_begin(m_penv, pparent_tx, flags, &p_new_tx); + if(res != MDBX_SUCCESS) + { + //Important: if mdbx_txn_begin is failed need to unlock previously locked mutex + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + //throw exception to avoid regular code execution + ASSERT_MES_AND_THROW_MDBX(res, "Unable to mdbx_txn_begin"); + } + + rtxlist.push_back(tx_entry()); + rtxlist.back().count = read_only ? 1 : 0; + rtxlist.back().ptx = p_new_tx; + rtxlist.back().read_only = read_only; + } + } + + + LOG_PRINT_L4("[DB] Transaction started"); + return true; + } + + MDBX_txn* mdbx_db_backend::get_current_tx() + { + std::lock_guard lock(m_cs); + auto& rtxlist = m_txs[std::this_thread::get_id()]; + CHECK_AND_ASSERT_MES(rtxlist.size(), nullptr, "Unable to find active tx for thread " << std::this_thread::get_id()); + return rtxlist.back().ptx; + } + + bool mdbx_db_backend::pop_tx_entry(tx_entry& txe) + { + std::lock_guard lock(m_cs); + auto it = m_txs.find(std::this_thread::get_id()); + CHECK_AND_ASSERT_MES(it != m_txs.end(), false, "[DB] Unable to find id cor current thread"); + CHECK_AND_ASSERT_MES(it->second.size(), false, "[DB] No active tx for current thread"); + + txe = it->second.back(); + + if (it->second.back().read_only && it->second.back().count == 0) + { + LOG_ERROR("Internal db tx state error: read_only and count readers == 0"); + } + + if ((it->second.back().read_only && it->second.back().count < 2) || (!it->second.back().read_only && it->second.back().count < 1)) + { + it->second.pop_back(); + if (!it->second.size()) + m_txs.erase(it); + } + else + { + --it->second.back().count; + } + return true; + } + + bool mdbx_db_backend::commit_transaction() + { + PROFILE_FUNC("mdbx_db_backend::commit_transaction"); + { + tx_entry txe = AUTO_VAL_INIT(txe); + bool r = pop_tx_entry(txe); + CHECK_AND_ASSERT_MES(r, false, "Unable to pop_tx_entry"); + + if (txe.count == 0 || (txe.read_only && txe.count == 1)) + { + int res = 0; + res = mdbx_txn_commit(txe.ptx); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_txn_commit (error " << res << ")"); + if (!txe.read_only && !txe.count) + { + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED", LOG_LEVEL_3); + } + } + } + LOG_PRINT_L4("[DB] Transaction committed"); + return true; + } + + void mdbx_db_backend::abort_transaction() + { + { + tx_entry txe = AUTO_VAL_INIT(txe); + bool r = pop_tx_entry(txe); + CHECK_AND_ASSERT_MES(r, void(), "Unable to pop_tx_entry"); + if (txe.count == 0 || (txe.read_only && txe.count == 1)) + { + mdbx_txn_abort(txe.ptx); + if (!txe.read_only && !txe.count) + { + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED(ABORTED)", LOG_LEVEL_3); + } + } + + + } + LOG_PRINT_L4("[DB] Transaction aborted"); + } + + bool mdbx_db_backend::erase(container_handle h, const char* k, size_t ks) + { + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + key.iov_base = (void*)k; + key.iov_len = ks; + + res = mdbx_del(get_current_tx(), static_cast(h), &key, nullptr); + if (res == MDBX_NOTFOUND) + return false; + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_del"); + return true; + } + + bool mdbx_db_backend::have_tx() + { + std::lock_guard lock(m_cs); + auto it = m_txs.find(std::this_thread::get_id()); + if (it == m_txs.end()) + return false; + return it->second.size() ? true : false; + } + + bool mdbx_db_backend::get(container_handle h, const char* k, size_t ks, std::string& res_buff) + { + PROFILE_FUNC("mdbx_db_backend::get"); + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + key.iov_base = (void*)k; + key.iov_len = ks; + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + + res = mdbx_get(get_current_tx(), static_cast(h), &key, &data); + + if (need_to_commit) + commit_transaction(); + + if (res == MDBX_NOTFOUND) + return false; + + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_get, h: " << h << ", ks: " << ks); + res_buff.assign((const char*)data.iov_base, data.iov_len); + return true; + } + + bool mdbx_db_backend::clear(container_handle h) + { + int res = mdbx_drop(get_current_tx(), static_cast(h), 0); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_drop"); + return true; + } + + uint64_t mdbx_db_backend::size(container_handle h) + { + PROFILE_FUNC("mdbx_db_backend::size"); + MDBX_stat container_stat = AUTO_VAL_INIT(container_stat); + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + int res = mdbx_dbi_stat(get_current_tx(), static_cast(h), &container_stat, sizeof(MDBX_stat)); + if (need_to_commit) + commit_transaction(); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_stat"); + return container_stat.ms_entries; + } + + bool mdbx_db_backend::set(container_handle h, const char* k, size_t ks, const char* v, size_t vs) + { + PROFILE_FUNC("mdbx_db_backend::set"); + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + key.iov_base = (void*)k; + key.iov_len = ks; + data.iov_base = (void*)v; + data.iov_len = vs; + + res = mdbx_put(get_current_tx(), static_cast(h), &key, &data, 0); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_put"); + return true; + } + bool mdbx_db_backend::enumerate(container_handle h, i_db_callback* pcb) + { + CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate"); + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + MDBX_cursor* cursor_ptr = nullptr; + int res = mdbx_cursor_open(get_current_tx(), static_cast(h), &cursor_ptr); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_cursor_open"); + CHECK_AND_ASSERT_MES(cursor_ptr, false, "cursor_ptr is null after mdbx_cursor_open"); + + uint64_t count = 0; + do + { + int res = mdbx_cursor_get(cursor_ptr, &key, &data, MDBX_NEXT); + if (res == MDBX_NOTFOUND) + break; + if (!pcb->on_enum_item(count, key.iov_base, key.iov_len, data.iov_base, data.iov_len)) + break; + count++; + } while (cursor_ptr); + + mdbx_cursor_close(cursor_ptr); + if (need_to_commit) + commit_transaction(); + return true; + } + + bool mdbx_db_backend::get_stat_info(tools::db::stat_info& si) + { + si = AUTO_VAL_INIT_T(tools::db::stat_info); + + MDBX_envinfo ei = AUTO_VAL_INIT(ei); + mdbx_env_info(m_penv, &ei, sizeof(MDBX_envinfo)); + si.map_size = ei.mi_mapsize; + + std::lock_guard lock(m_cs); + for (auto& e : m_txs) + { + for (auto& pr : e.second) + { + ++si.tx_count; + if(!pr.read_only) + ++si.write_tx_count; + } + } + return true; + } + const char* mdbx_db_backend::name() + { + return "mdbx"; + } + } +} + +#undef LOG_DEFAULT_CHANNEL +#define LOG_DEFAULT_CHANNEL NULL +#endif \ No newline at end of file diff --git a/src/common/db_backend_mdbx.h b/src/common/db_backend_mdbx.h new file mode 100644 index 00000000..443cbe5a --- /dev/null +++ b/src/common/db_backend_mdbx.h @@ -0,0 +1,66 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#ifdef ENABLED_ENGINE_MDBX +#include + +#include "include_base_utils.h" + +#include "db_backend_base.h" +#include "db/libmdbx/mdbx.h" + + +namespace tools +{ + namespace db + { + class mdbx_db_backend : public i_db_backend + { + + struct tx_entry + { + MDBX_txn* ptx; + bool read_only; // needed for thread-top transaction, for figure out if we need to unlock exclusive access + size_t count; //count of read-only nested emulated transactions + }; + typedef std::list transactions_list; + + + std::string m_path; + MDBX_env *m_penv; + + boost::recursive_mutex m_cs; + boost::recursive_mutex m_write_exclusive_lock; + std::map m_txs; // size_t -> count of nested read_only transactions + bool pop_tx_entry(tx_entry& txe); + public: + mdbx_db_backend(); + ~mdbx_db_backend(); + + //----------------- i_db_backend ----------------------------------------------------- + bool close(); + bool begin_transaction(bool read_only = false); + bool commit_transaction(); + void abort_transaction(); + bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE); + bool open_container(const std::string& name, container_handle& h); + bool erase(container_handle h, const char* k, size_t s); + bool get(container_handle h, const char* k, size_t s, std::string& res_buff); + bool clear(container_handle h); + uint64_t size(container_handle h); + bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs); + bool enumerate(container_handle h, i_db_callback* pcb); + bool get_stat_info(tools::db::stat_info& si); + const char* name(); + //------------------------------------------------------------------------------------- + bool have_tx(); + MDBX_txn* get_current_tx(); + + }; + + } +} +#endif \ No newline at end of file diff --git a/src/common/db_backend_selector.h b/src/common/db_backend_selector.h new file mode 100644 index 00000000..afaee01a --- /dev/null +++ b/src/common/db_backend_selector.h @@ -0,0 +1,41 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "db_backend_lmdb.h" +#include "db_backend_mdbx.h" +#include "common/command_line.h" +#include "common/db_abstract_accessor.h" + +namespace tools +{ + namespace db + { + inline + bool select_db_engine_from_arg(const boost::program_options::variables_map& vm, tools::db::basic_db_accessor& rdb) + { + if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_LMDB) + { + rdb.reset_backend(std::shared_ptr(new tools::db::lmdb_db_backend)); + } + else if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_MDBX) + { +#ifdef ENABLED_ENGINE_MDBX + rdb.reset_backend(std::shared_ptr(new tools::db::mdbx_db_backend)); +#else + LOG_PRINT_L0(" DB ENGINE: " << ARG_DB_ENGINE_MDBX << " is not suported by this build(see DISABLE_MDBX cmake option), STOPPING"); + return false; +#endif + } + else + { + LOG_PRINT_RED_L0(" UNKNOWN DB ENGINE: " << command_line::get_arg(vm, command_line::arg_db_engine) << ", STOPPING"); + return false; + } + return true; + } + } +} \ No newline at end of file diff --git a/src/common/ntp.cpp b/src/common/ntp.cpp index 84a23b0a..84b40f8d 100644 --- a/src/common/ntp.cpp +++ b/src/common/ntp.cpp @@ -201,7 +201,7 @@ namespace tools udp_blocking_client ubc(sender_endpoint, socket, io_service); boost::system::error_code ec; - size_t len = ubc.receive(boost::asio::buffer(&packet_received, sizeof packet_received), boost::posix_time::seconds(timeout_sec), ec); + size_t len = ubc.receive(boost::asio::buffer(&packet_received, sizeof packet_received), boost::posix_time::seconds(static_cast(timeout_sec)), ec); if (ec) { LOG_PRINT_L3("NTP: get_ntp_time(" << host_name << "): boost error: " << ec.message()); diff --git a/src/currency_core/bc_attachments_service_manager.h b/src/currency_core/bc_attachments_service_manager.h index bd4f2640..632126a9 100644 --- a/src/currency_core/bc_attachments_service_manager.h +++ b/src/currency_core/bc_attachments_service_manager.h @@ -27,7 +27,7 @@ namespace currency class bc_attachment_services_manager { public: - bc_attachment_services_manager(i_core_event_handler* pcore_event_handler) : m_pcore_event_handler(pcore_event_handler), m_core_runtime_config(get_default_core_runtime_config()) + bc_attachment_services_manager(/* i_core_event_handler* pcore_event_handler*/) : /*m_pcore_event_handler(pcore_event_handler),*/ m_core_runtime_config(get_default_core_runtime_config()) {} @@ -43,7 +43,7 @@ namespace currency private: std::map m_services; - i_core_event_handler* m_pcore_event_handler; + //i_core_event_handler* m_pcore_event_handler; core_runtime_config m_core_runtime_config; }; diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 67ca688a..c3538879 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -14,7 +14,7 @@ #include "include_base_utils.h" -#include "common/db_backend_lmdb.h" +#include "common/db_backend_selector.h" #include "common/command_line.h" #include "blockchain_storage.h" @@ -30,7 +30,6 @@ #include "crypto/hash.h" #include "miner_common.h" #include "storages/portable_storage_template_helper.h" -#include "common/db_backend_lmdb.h" #include "basic_pow_helpers.h" #include "version.h" @@ -70,7 +69,7 @@ using namespace currency; #endif #define BLOCK_POS_STRICT_SEQUENCE_LIMIT 20 - +#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX "_v1" DISABLE_VS_WARNINGS(4267) @@ -81,7 +80,7 @@ namespace } //------------------------------------------------------------------ -blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::shared_ptr(new tools::db::lmdb_db_backend), m_rw_lock), +blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m_rw_lock), m_db_blocks(m_db), m_db_blocks_index(m_db), m_db_transactions(m_db), @@ -104,7 +103,6 @@ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::share m_core_runtime_config(get_default_core_runtime_config()), //m_bei_stub(AUTO_VAL_INIT(m_bei_stub)), m_event_handler(&m_event_handler_stub), - m_services_mgr(nullptr), m_interprocess_locker_file(0), m_current_fee_median(0), m_current_fee_median_effective_index(0), @@ -207,7 +205,13 @@ bool blockchain_storage::validate_instance(const std::string& path) bool blockchain_storage::init(const std::string& config_folder, const boost::program_options::variables_map& vm) { // CRITICAL_REGION_LOCAL(m_read_lock); - + if (!select_db_engine_from_arg(vm, m_db)) + { + LOG_PRINT_RED_L0("Failed to select db engine"); + return false; + } + LOG_PRINT_L0("DB ENGINE USED BY CORE: " << m_db.get_backend()->name()); + if (!validate_instance(config_folder)) { LOG_ERROR("Failed to initialize instance"); @@ -230,8 +234,8 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0); boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path)); } - - const std::string db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME; + ; + const std::string db_folder_path = m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + m_db.get_backend()->name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; LOG_PRINT_L0("Loading blockchain from " << db_folder_path); bool db_opened_okay = false; diff --git a/src/currency_core/currency_config.h b/src/currency_core/currency_config.h index fce54784..c6b210cd 100644 --- a/src/currency_core/currency_config.h +++ b/src/currency_core/currency_config.h @@ -189,8 +189,11 @@ #define CURRENCY_POOLDATA_FOLDERNAME_OLD "poolstate" #define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD "blockchain" -#define CURRENCY_POOLDATA_FOLDERNAME "poolstate_lmdb_v1" -#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain_lmdb_v1" + + +#define CURRENCY_POOLDATA_FOLDERNAME_PREFIX "poolstate_" +#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX "blockchain_" + #define P2P_NET_DATA_FILENAME "p2pstate.bin" #define MINER_CONFIG_FILENAME "miner_conf.json" #define GUI_SECURE_CONFIG_FILENAME "gui_secure_conf.bin" diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index a79dfb2a..20e81fed 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -142,7 +142,7 @@ namespace currency uint64_t available_space = 0; CHECK_AND_ASSERT_MES(!check_if_free_space_critically_low(&available_space), false, "free space in data folder is critically low: " << std::fixed << available_space / (1024 * 1024) << " MB"); - r = m_mempool.init(m_config_folder); + r = m_mempool.init(m_config_folder, vm); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); r = m_blockchain_storage.init(m_config_folder, vm); diff --git a/src/currency_core/miner.cpp b/src/currency_core/miner.cpp index afb77055..ff2579e2 100644 --- a/src/currency_core/miner.cpp +++ b/src/currency_core/miner.cpp @@ -41,7 +41,7 @@ namespace currency miner::miner(i_miner_handler* phandler, blockchain_storage& bc):m_stop(1), - m_bc(bc), + //m_bc(bc), m_template(boost::value_initialized()), m_template_no(0), m_diffic(0), diff --git a/src/currency_core/miner.h b/src/currency_core/miner.h index 1e9d030e..c1bd83f0 100644 --- a/src/currency_core/miner.h +++ b/src/currency_core/miner.h @@ -105,7 +105,7 @@ namespace currency std::list m_threads; ::critical_section m_threads_lock; i_miner_handler* m_phandler; - blockchain_storage& m_bc; + //blockchain_storage& m_bc; account_public_address m_mine_address; math_helper::once_a_time_seconds<5> m_update_block_template_interval; math_helper::once_a_time_seconds<2> m_update_merge_hr_interval; diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index e773ad31..ae8737a8 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -9,7 +9,7 @@ #include #include -#include "common/db_backend_lmdb.h" +#include "common/db_backend_selector.h" #include "tx_pool.h" #include "currency_boost_serialization.h" #include "currency_core/currency_config.h" @@ -32,6 +32,8 @@ DISABLE_VS_WARNINGS(4244 4345 4503) //'boost::foreach_detail_::or_' : decorated #define TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 92 // DON'T CHANGE THIS, if you need to resync db! Change TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION instead! #define TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION + 1 +#define CURRENCY_POOLDATA_FOLDERNAME_SUFFIX "_v1" + #define CONFLICT_KEY_IMAGE_SPENT_DEPTH_TO_REMOVE_TX_FROM_POOL 50 // if there's a conflict in key images between tx in the pool and in the blockchain this much depth in required to remove correspongin tx from pool #undef LOG_DEFAULT_CHANNEL @@ -44,7 +46,7 @@ namespace currency tx_memory_pool::tx_memory_pool(blockchain_storage& bchs, i_currency_protocol* pprotocol) : m_blockchain(bchs), m_pprotocol(pprotocol), - m_db(std::shared_ptr(new tools::db::lmdb_db_backend), m_dummy_rw_lock), + m_db(nullptr, m_dummy_rw_lock), m_db_transactions(m_db), m_db_black_tx_list(m_db), m_db_solo_options(m_db), @@ -1141,8 +1143,15 @@ namespace currency m_db.commit_transaction(); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::init(const std::string& config_folder) + bool tx_memory_pool::init(const std::string& config_folder, const boost::program_options::variables_map& vm) { + if (!select_db_engine_from_arg(vm, m_db)) + { + LOG_PRINT_RED_L0("Failed to select db engine"); + return false; + } + LOG_PRINT_L0("DB ENGINE USED BY POOL: " << m_db.get_backend()->name()); + m_config_folder = config_folder; uint64_t cache_size_l1 = CACHE_SIZE; @@ -1156,7 +1165,8 @@ namespace currency boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path)); } - const std::string db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME; + const std::string db_folder_path = m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + m_db.get_backend()->name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX; + LOG_PRINT_L0("Loading blockchain from " << db_folder_path << "..."); bool db_opened_okay = false; diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index d546a4a8..6d9bad9d 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -20,6 +20,7 @@ using namespace epee; #include "math_helper.h" #include "common/db_abstract_accessor.h" +#include "common/command_line.h" #include "currency_format_utils.h" #include "verification_context.h" @@ -114,7 +115,7 @@ namespace currency void clear(); // load/store operations - bool init(const std::string& config_folder); + bool init(const std::string& config_folder, const boost::program_options::variables_map& vm); bool deinit(); bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height); bool get_transactions(std::list& txs) const; diff --git a/src/currency_protocol/currency_protocol_handler.inl b/src/currency_protocol/currency_protocol_handler.inl index 7ff9a340..2566bd24 100644 --- a/src/currency_protocol/currency_protocol_handler.inl +++ b/src/currency_protocol/currency_protocol_handler.inl @@ -800,7 +800,7 @@ namespace currency std::vector time_deltas_copy(m_time_deltas.begin(), m_time_deltas.end()); m_last_median2local_time_difference = epee::misc_utils::median(time_deltas_copy); - LOG_PRINT_MAGENTA("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << ")", m_last_median2local_time_difference >= 3 ? LOG_LEVEL_2 : LOG_LEVEL_3); + LOG_PRINT_MAGENTA("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << ")", ((m_last_median2local_time_difference >= 3) ? LOG_LEVEL_2 : LOG_LEVEL_3)); if (std::abs(m_last_median2local_time_difference) > TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE) { int64_t ntp_time = tools::get_ntp_time(); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index f81d1a5a..c168ad63 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -147,6 +147,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, command_line::arg_disable_stop_if_time_out_of_sync); command_line::add_arg(desc_cmd_sett, command_line::arg_disable_stop_on_low_free_space); command_line::add_arg(desc_cmd_sett, command_line::arg_enable_offers_service); + command_line::add_arg(desc_cmd_sett, command_line::arg_db_engine); arg_market_disable.default_value = true; diff --git a/src/gui/qt-daemon/application/daemon_backend.cpp b/src/gui/qt-daemon/application/daemon_backend.cpp index dab77d8b..36ac093e 100644 --- a/src/gui/qt-daemon/application/daemon_backend.cpp +++ b/src/gui/qt-daemon/application/daemon_backend.cpp @@ -120,6 +120,7 @@ bool daemon_backend::init(int argc, char* argv[], view::i_view* pview_handler) command_line::add_arg(desc_cmd_sett, command_line::arg_log_level); command_line::add_arg(desc_cmd_sett, command_line::arg_console); command_line::add_arg(desc_cmd_sett, command_line::arg_show_details); + command_line::add_arg(desc_cmd_sett, command_line::arg_db_engine); command_line::add_arg(desc_cmd_sett, arg_alloc_win_console); command_line::add_arg(desc_cmd_sett, arg_html_folder); command_line::add_arg(desc_cmd_sett, arg_xcode_stub);