diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 7acb5249..bcf3ed09 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -9,7 +9,7 @@ add_subdirectory(ethereum) set_property(TARGET upnpc-static PROPERTY FOLDER "contrib/miniupnp") set_property(TARGET zlibstatic PROPERTY FOLDER "contrib") -# set_property(TARGET lmdb PROPERTY FOLDER "contrib") +set_property(TARGET lmdb PROPERTY FOLDER "contrib") set_property(TARGET mdbx PROPERTY FOLDER "contrib") diff --git a/contrib/db/CMakeLists.txt b/contrib/db/CMakeLists.txt index 3ff49da1..57961a93 100644 --- a/contrib/db/CMakeLists.txt +++ b/contrib/db/CMakeLists.txt @@ -1,8 +1,8 @@ -# add_subdirectory(liblmdb) +add_subdirectory(liblmdb) add_subdirectory(libmdbx) if(MSVC) - # target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172) + target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172) else() - # Warnings as used by LMDB itself (LMDB_0.9.23) - # target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized) + # Warnings as used by LMDB itself (LMDB_0.9.23) + target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized) endif() diff --git a/contrib/db/libmdbx/CMakeLists.txt b/contrib/db/libmdbx/CMakeLists.txt index b44b8002..276dd96b 100644 --- a/contrib/db/libmdbx/CMakeLists.txt +++ b/contrib/db/libmdbx/CMakeLists.txt @@ -1,5 +1,5 @@ -set (mdbx_sources src/mdbx.c src/osal.c) +set (mdbx_sources src/mdbx.c src/osal.c src/lck-windows.c) include_directories("${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fa8a230c..772a4535 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -116,7 +116,7 @@ add_library(wallet ${WALLET}) add_dependencies(wallet version ${PCH_LIB_NAME}) ENABLE_SHARED_PCH(WALLET) -target_link_libraries(currency_core mdbx) +target_link_libraries(currency_core mdbx lmdb) add_executable(daemon ${DAEMON} ${P2P} ${CURRENCY_PROTOCOL}) add_dependencies(daemon version) diff --git a/src/common/db_backend_mdbx.cpp b/src/common/db_backend_mdbx.cpp new file mode 100644 index 00000000..66d2ed10 --- /dev/null +++ b/src/common/db_backend_mdbx.cpp @@ -0,0 +1,396 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "db_backend_mdbx.h" +#include "misc_language.h" +#include "string_coding.h" +#include "profile_tools.h" +#include "util.h" + +#define BUF_SIZE 1024 + +#define CHECK_AND_ASSERT_MESS_MDBX_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDBX_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); +#define CHECK_AND_ASSERT_THROW_MESS_MDBX_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDBX_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); +#define ASSERT_MES_AND_THROW_MDBX(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); + +#undef LOG_DEFAULT_CHANNEL +#define LOG_DEFAULT_CHANNEL "mdbx" +// 'mdbx' channel is disabled by default + +namespace tools +{ + namespace db + { + mdbx_db_backend::mdbx_db_backend() : m_penv(AUTO_VAL_INIT(m_penv)) + { + + } + mdbx_db_backend::~mdbx_db_backend() + { + NESTED_TRY_ENTRY(); + + close(); + + NESTED_CATCH_ENTRY(__func__); + } + + bool mdbx_db_backend::open(const std::string& path_, uint64_t cache_sz) + { + int res = 0; + res = mdbx_env_create(&m_penv); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_create"); + + res = mdbx_env_set_maxdbs(m_penv, 15); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_maxdbs"); + + intptr_t size_lower = 0; + intptr_t size_now = -1; //don't change current database size + intptr_t size_upper = -1; //don't set db file size limit + intptr_t growth_step = 0x40000000; //increment step 1GB + intptr_t shrink_threshold = -1; + intptr_t pagesize = 0x00001000; //4kb + res = mdbx_env_set_geometry(m_penv, size_lower, size_now, size_upper, growth_step, shrink_threshold, pagesize); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_mapsize"); + + m_path = path_; +#ifdef WIN32 + m_path = epee::string_encoding::convert_ansii_to_utf8(m_path); +#endif + + CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path); + + res = mdbx_env_open(m_penv, m_path.c_str(), MDBX_NORDAHEAD , 0644); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_open, m_path=" << m_path); + + return true; + } + + bool mdbx_db_backend::open_container(const std::string& name, container_handle& h) + { + + MDBX_dbi dbi = AUTO_VAL_INIT(dbi); + begin_transaction(); + int res = mdbx_dbi_open(get_current_tx(), name.c_str(), MDBX_CREATE, &dbi); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_dbi_open with container name: " << name); + commit_transaction(); + h = static_cast(dbi); + return true; + } + + bool mdbx_db_backend::close() + { + { + std::lock_guard lock(m_cs); + for (auto& tx_thread : m_txs) + { + for (auto txe : tx_thread.second) + { + int res = mdbx_txn_commit(txe.ptx); + if (res != MDBX_SUCCESS) + { + LOG_ERROR("[DB ERROR]: On close tranactions: " << mdbx_strerror(res)); + } + } + } + + m_txs.clear(); + } + if (m_penv) + { + mdbx_env_close(m_penv); + m_penv = nullptr; + } + return true; + } + + bool mdbx_db_backend::begin_transaction(bool read_only) + { + if (!read_only) + { + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3); + CRITICAL_SECTION_LOCK(m_write_exclusive_lock); + } + PROFILE_FUNC("mdbx_db_backend::begin_transaction"); + { + std::lock_guard lock(m_cs); + CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed"); + transactions_list& rtxlist = m_txs[std::this_thread::get_id()]; + MDBX_txn* pparent_tx = nullptr; + MDBX_txn* p_new_tx = nullptr; + bool parent_read_only = false; + if (rtxlist.size()) + { + pparent_tx = rtxlist.back().ptx; + parent_read_only = rtxlist.back().read_only; + } + + + if (pparent_tx && read_only) + { + ++rtxlist.back().count; + } + else + { + int res = 0; + unsigned int flags = 0; + if (read_only) + flags += MDBX_RDONLY; + + //don't use parent tx in write transactions if parent tx was read-only (restriction in mdbx) + //see "Nested transactions: Max 1 child, write txns only, no writemap" + if (pparent_tx && parent_read_only) + pparent_tx = nullptr; + + CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed"); + res = mdbx_txn_begin(m_penv, pparent_tx, flags, &p_new_tx); + if(res != MDBX_SUCCESS) + { + //Important: if mdbx_txn_begin is failed need to unlock previously locked mutex + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + //throw exception to avoid regular code execution + ASSERT_MES_AND_THROW_MDBX(res, "Unable to mdbx_txn_begin"); + } + + rtxlist.push_back(tx_entry()); + rtxlist.back().count = read_only ? 1 : 0; + rtxlist.back().ptx = p_new_tx; + rtxlist.back().read_only = read_only; + } + } + + + LOG_PRINT_L4("[DB] Transaction started"); + return true; + } + + MDBX_txn* mdbx_db_backend::get_current_tx() + { + std::lock_guard lock(m_cs); + auto& rtxlist = m_txs[std::this_thread::get_id()]; + CHECK_AND_ASSERT_MES(rtxlist.size(), nullptr, "Unable to find active tx for thread " << std::this_thread::get_id()); + return rtxlist.back().ptx; + } + + bool mdbx_db_backend::pop_tx_entry(tx_entry& txe) + { + std::lock_guard lock(m_cs); + auto it = m_txs.find(std::this_thread::get_id()); + CHECK_AND_ASSERT_MES(it != m_txs.end(), false, "[DB] Unable to find id cor current thread"); + CHECK_AND_ASSERT_MES(it->second.size(), false, "[DB] No active tx for current thread"); + + txe = it->second.back(); + + if (it->second.back().read_only && it->second.back().count == 0) + { + LOG_ERROR("Internal db tx state error: read_only and count readers == 0"); + } + + if ((it->second.back().read_only && it->second.back().count < 2) || (!it->second.back().read_only && it->second.back().count < 1)) + { + it->second.pop_back(); + if (!it->second.size()) + m_txs.erase(it); + } + else + { + --it->second.back().count; + } + return true; + } + + bool mdbx_db_backend::commit_transaction() + { + PROFILE_FUNC("mdbx_db_backend::commit_transaction"); + { + tx_entry txe = AUTO_VAL_INIT(txe); + bool r = pop_tx_entry(txe); + CHECK_AND_ASSERT_MES(r, false, "Unable to pop_tx_entry"); + + if (txe.count == 0 || (txe.read_only && txe.count == 1)) + { + int res = 0; + res = mdbx_txn_commit(txe.ptx); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_txn_commit (error " << res << ")"); + if (!txe.read_only && !txe.count) + { + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED", LOG_LEVEL_3); + } + } + } + LOG_PRINT_L4("[DB] Transaction committed"); + return true; + } + + void mdbx_db_backend::abort_transaction() + { + { + tx_entry txe = AUTO_VAL_INIT(txe); + bool r = pop_tx_entry(txe); + CHECK_AND_ASSERT_MES(r, void(), "Unable to pop_tx_entry"); + if (txe.count == 0 || (txe.read_only && txe.count == 1)) + { + mdbx_txn_abort(txe.ptx); + if (!txe.read_only && !txe.count) + { + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED(ABORTED)", LOG_LEVEL_3); + } + } + + + } + LOG_PRINT_L4("[DB] Transaction aborted"); + } + + bool mdbx_db_backend::erase(container_handle h, const char* k, size_t ks) + { + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + key.iov_base = (void*)k; + key.iov_len = ks; + + res = mdbx_del(get_current_tx(), static_cast(h), &key, nullptr); + if (res == MDBX_NOTFOUND) + return false; + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_del"); + return true; + } + + bool mdbx_db_backend::have_tx() + { + std::lock_guard lock(m_cs); + auto it = m_txs.find(std::this_thread::get_id()); + if (it == m_txs.end()) + return false; + return it->second.size() ? true : false; + } + + bool mdbx_db_backend::get(container_handle h, const char* k, size_t ks, std::string& res_buff) + { + PROFILE_FUNC("mdbx_db_backend::get"); + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + key.iov_base = (void*)k; + key.iov_len = ks; + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + + res = mdbx_get(get_current_tx(), static_cast(h), &key, &data); + + if (need_to_commit) + commit_transaction(); + + if (res == MDBX_NOTFOUND) + return false; + + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_get, h: " << h << ", ks: " << ks); + res_buff.assign((const char*)data.iov_base, data.iov_len); + return true; + } + + bool mdbx_db_backend::clear(container_handle h) + { + int res = mdbx_drop(get_current_tx(), static_cast(h), 0); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_drop"); + return true; + } + + uint64_t mdbx_db_backend::size(container_handle h) + { + PROFILE_FUNC("mdbx_db_backend::size"); + MDBX_stat container_stat = AUTO_VAL_INIT(container_stat); + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + int res = mdbx_dbi_stat(get_current_tx(), static_cast(h), &container_stat, sizeof(MDBX_stat)); + if (need_to_commit) + commit_transaction(); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_stat"); + return container_stat.ms_entries; + } + + bool mdbx_db_backend::set(container_handle h, const char* k, size_t ks, const char* v, size_t vs) + { + PROFILE_FUNC("mdbx_db_backend::set"); + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + key.iov_base = (void*)k; + key.iov_len = ks; + data.iov_base = (void*)v; + data.iov_len = vs; + + res = mdbx_put(get_current_tx(), static_cast(h), &key, &data, 0); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_put"); + return true; + } + bool mdbx_db_backend::enumerate(container_handle h, i_db_callback* pcb) + { + CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate"); + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + MDBX_cursor* cursor_ptr = nullptr; + int res = mdbx_cursor_open(get_current_tx(), static_cast(h), &cursor_ptr); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_cursor_open"); + CHECK_AND_ASSERT_MES(cursor_ptr, false, "cursor_ptr is null after mdbx_cursor_open"); + + uint64_t count = 0; + do + { + int res = mdbx_cursor_get(cursor_ptr, &key, &data, MDBX_NEXT); + if (res == MDBX_NOTFOUND) + break; + if (!pcb->on_enum_item(count, key.iov_base, key.iov_len, data.iov_base, data.iov_len)) + break; + count++; + } while (cursor_ptr); + + mdbx_cursor_close(cursor_ptr); + if (need_to_commit) + commit_transaction(); + return true; + } + + bool mdbx_db_backend::get_stat_info(tools::db::stat_info& si) + { + si = AUTO_VAL_INIT_T(tools::db::stat_info); + + MDBX_envinfo ei = AUTO_VAL_INIT(ei); + mdbx_env_info(m_penv, &ei, sizeof(MDBX_envinfo)); + si.map_size = ei.mi_mapsize; + + std::lock_guard lock(m_cs); + for (auto& e : m_txs) + { + for (auto& pr : e.second) + { + ++si.tx_count; + if(!pr.read_only) + ++si.write_tx_count; + } + } + return true; + } + } +} + +#undef LOG_DEFAULT_CHANNEL +#define LOG_DEFAULT_CHANNEL NULL diff --git a/src/common/db_backend_mdbx.h b/src/common/db_backend_mdbx.h new file mode 100644 index 00000000..f60e4028 --- /dev/null +++ b/src/common/db_backend_mdbx.h @@ -0,0 +1,62 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#include + +#include "include_base_utils.h" + +#include "db_backend_base.h" +#include "db/libmdbx/mdbx.h" + + +namespace tools +{ + namespace db + { + class mdbx_db_backend : public i_db_backend + { + + struct tx_entry + { + MDBX_txn* ptx; + bool read_only; // needed for thread-top transaction, for figure out if we need to unlock exclusive access + size_t count; //count of read-only nested emulated transactions + }; + typedef std::list transactions_list; + + + std::string m_path; + MDBX_env *m_penv; + + boost::recursive_mutex m_cs; + boost::recursive_mutex m_write_exclusive_lock; + std::map m_txs; // size_t -> count of nested read_only transactions + bool pop_tx_entry(tx_entry& txe); + public: + mdbx_db_backend(); + ~mdbx_db_backend(); + + //----------------- i_db_backend ----------------------------------------------------- + bool close(); + bool begin_transaction(bool read_only = false); + bool commit_transaction(); + void abort_transaction(); + bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE); + bool open_container(const std::string& name, container_handle& h); + bool erase(container_handle h, const char* k, size_t s); + bool get(container_handle h, const char* k, size_t s, std::string& res_buff); + bool clear(container_handle h); + uint64_t size(container_handle h); + bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs); + bool enumerate(container_handle h, i_db_callback* pcb); + bool get_stat_info(tools::db::stat_info& si); + //------------------------------------------------------------------------------------- + bool have_tx(); + MDBX_txn* get_current_tx(); + + }; + } +} diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 827adc65..a7218a93 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -15,6 +15,7 @@ #include "include_base_utils.h" #include "common/db_backend_lmdb.h" +#include "common/db_backend_mdbx.h" #include "common/command_line.h" #include "blockchain_storage.h" @@ -30,7 +31,6 @@ #include "crypto/hash.h" #include "miner_common.h" #include "storages/portable_storage_template_helper.h" -#include "common/db_backend_lmdb.h" #include "basic_pow_helpers.h" #include "version.h" @@ -81,7 +81,7 @@ namespace } //------------------------------------------------------------------ -blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::shared_ptr(new tools::db::lmdb_db_backend), m_rw_lock), +blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::shared_ptr(new tools::db::mdbx_db_backend), m_rw_lock), m_db_blocks(m_db), m_db_blocks_index(m_db), m_db_transactions(m_db), diff --git a/src/currency_core/currency_config.h b/src/currency_core/currency_config.h index 11d2ec06..10e32087 100644 --- a/src/currency_core/currency_config.h +++ b/src/currency_core/currency_config.h @@ -189,8 +189,10 @@ #define CURRENCY_POOLDATA_FOLDERNAME_OLD "poolstate" #define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD "blockchain" -#define CURRENCY_POOLDATA_FOLDERNAME "poolstate_lmdb_v1" -#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain_lmdb_v1" +//#define CURRENCY_POOLDATA_FOLDERNAME "poolstate_lmdb_v1" +//#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain_lmdb_v1" +#define CURRENCY_POOLDATA_FOLDERNAME "poolstate_mdbx_v1" +#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain_mdbx_v1" #define P2P_NET_DATA_FILENAME "p2pstate.bin" #define MINER_CONFIG_FILENAME "miner_conf.json" #define GUI_SECURE_CONFIG_FILENAME "gui_secure_conf.bin"