From 972acf2632821ea8bd0079bb7aac0240df6f3a3f Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 12:21:05 +0300 Subject: [PATCH] lmdb back end reverted --- src/common/db_backend_lmdb.cpp | 69 ++++------------------------------ src/common/db_backend_lmdb.h | 15 ++------ 2 files changed, 10 insertions(+), 74 deletions(-) diff --git a/src/common/db_backend_lmdb.cpp b/src/common/db_backend_lmdb.cpp index ceb46796..fdb289ff 100644 --- a/src/common/db_backend_lmdb.cpp +++ b/src/common/db_backend_lmdb.cpp @@ -10,10 +10,6 @@ #include "util.h" #define BUF_SIZE 1024 -#define DB_RESIZE_MIN_FREE_SIZE (100 * 1024 * 1024) // DB map size will grow if that much space left on DB -#define DB_RESIZE_MIN_MAX_SIZE (50 * 1024 * 1024) // Minimum DB map size (starting size) -#define DB_RESIZE_INCREMENT_SIZE (100 * 1024 * 1024) // Grow step size -#define DB_RESIZE_COMMITS_TO_CHECK 50 #define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); #define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); @@ -27,13 +23,10 @@ namespace tools { namespace db { - lmdb_db_backend::lmdb_db_backend() - : m_penv(AUTO_VAL_INIT(m_penv)) - , m_commits_count(0) + lmdb_db_backend::lmdb_db_backend() : m_penv(AUTO_VAL_INIT(m_penv)) { } - lmdb_db_backend::~lmdb_db_backend() { NESTED_TRY_ENTRY(); @@ -43,16 +36,18 @@ namespace tools NESTED_CATCH_ENTRY(__func__); } - bool lmdb_db_backend::open(const std::string& path_, uint64_t /* flags -- unused atm */) + bool lmdb_db_backend::open(const std::string& path_, uint64_t cache_sz) { int res = 0; res = mdb_env_create(&m_penv); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_create"); - MDB_dbi max_dbs = 15; - res = mdb_env_set_maxdbs(m_penv, max_dbs); + res = mdb_env_set_maxdbs(m_penv, 15); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_maxdbs"); + res = mdb_env_set_mapsize(m_penv, cache_sz); + CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize"); + m_path = path_; #ifdef WIN32 m_path = epee::string_encoding::convert_ansii_to_utf8(m_path); @@ -60,14 +55,8 @@ namespace tools CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path); - // TODO: convert flags to lmdb_flags (implement fast lmdb modes) - unsigned int lmdb_flags = MDB_NORDAHEAD /*| MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC*/; - mdb_mode_t lmdb_mode = 0644; - - res = mdb_env_open(m_penv, m_path.c_str(), lmdb_flags, lmdb_mode); + res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD , 0644); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_open, m_path=" << m_path); - - resize_if_needed(); return true; } @@ -116,13 +105,6 @@ namespace tools { LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3); CRITICAL_SECTION_LOCK(m_write_exclusive_lock); - - if (m_commits_count.fetch_add(1, std::memory_order_relaxed) % DB_RESIZE_COMMITS_TO_CHECK == DB_RESIZE_COMMITS_TO_CHECK - 1) - { - if (!resize_if_needed()) - m_commits_count.store(DB_RESIZE_COMMITS_TO_CHECK - 1, std::memory_order_relaxed); // if failed, try again on next commit - } - } PROFILE_FUNC("lmdb_db_backend::begin_transaction"); { @@ -347,7 +329,6 @@ namespace tools CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put"); return true; } - bool lmdb_db_backend::enumerate(container_handle h, i_db_callback* pcb) { CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate"); @@ -402,42 +383,6 @@ namespace tools } return true; } - - bool lmdb_db_backend::resize_if_needed() - { - LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED in resize_if_needed()", LOG_LEVEL_3); - CRITICAL_REGION_LOCAL(m_write_exclusive_lock); - - if (have_tx()) - { - LOG_PRINT_RED("[DB " << m_path << "] : resize_if_needed(): Have txs on stack, unable to resize!", LOG_LEVEL_0); - return false; - } - - MDB_stat st = AUTO_VAL_INIT(st); - mdb_env_stat(m_penv, &st); - MDB_envinfo ei = AUTO_VAL_INIT(ei); - mdb_env_info(m_penv, &ei); - - uint64_t dirty_size = ei.me_last_pgno * st.ms_psize; - int64_t size_diff = ei.me_mapsize - dirty_size; - if (size_diff >= DB_RESIZE_MIN_FREE_SIZE && ei.me_mapsize >= DB_RESIZE_MIN_MAX_SIZE) - return true; // resize is not needed - - double gigabyte = 1024 * 1024 * 1024; - const uint64_t increment_size_pg_aligned = DB_RESIZE_INCREMENT_SIZE - (DB_RESIZE_INCREMENT_SIZE % st.ms_psize); - - // need to resize DB - uint64_t new_size = ei.me_mapsize - (ei.me_mapsize % increment_size_pg_aligned) + increment_size_pg_aligned; - - int res = mdb_env_set_mapsize(m_penv, new_size); - CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize"); - - LOG_PRINT_CYAN("[DB " << m_path << "] has grown: " << std::fixed << std::setprecision(2) << ei.me_mapsize / gigabyte << " GiB -> " << std::fixed << std::setprecision(2) << new_size / gigabyte << " GiB", LOG_LEVEL_0); - - return true; - } - } } diff --git a/src/common/db_backend_lmdb.h b/src/common/db_backend_lmdb.h index 84c441d3..95c9d8a9 100644 --- a/src/common/db_backend_lmdb.h +++ b/src/common/db_backend_lmdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,6 @@ #pragma once #include - #include "include_base_utils.h" #include "db_backend_base.h" @@ -17,9 +16,6 @@ namespace tools { namespace db { - - - class lmdb_db_backend : public i_db_backend { @@ -38,11 +34,7 @@ namespace tools boost::recursive_mutex m_cs; boost::recursive_mutex m_write_exclusive_lock; std::map m_txs; // size_t -> count of nested read_only transactions - std::atomic m_commits_count; - bool pop_tx_entry(tx_entry& txe); - - public: lmdb_db_backend(); ~lmdb_db_backend(); @@ -52,7 +44,7 @@ namespace tools bool begin_transaction(bool read_only = false); bool commit_transaction(); void abort_transaction(); - bool open(const std::string& path, uint64_t flags = 0); + bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE); bool open_container(const std::string& name, container_handle& h); bool erase(container_handle h, const char* k, size_t s); bool get(container_handle h, const char* k, size_t s, std::string& res_buff); @@ -64,8 +56,7 @@ namespace tools //------------------------------------------------------------------------------------- bool have_tx(); MDB_txn* get_current_tx(); - bool resize_if_needed(); }; } -} \ No newline at end of file +}