Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity

lmdb back end reverted

Browse source
This commit is contained in:
sowle 2019-08-29 12:21:05 +03:00
parent fb66630676
commit 972acf2632
No known key found for this signature in database
GPG key ID: C07A24B2D89D49FC
2 changed files with 10 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
src/common/db_backend_lmdb.cpp
View file

@ -10,10 +10,6 @@
#include "util.h"
#define BUF_SIZE 1024
#define DB_RESIZE_MIN_FREE_SIZE (100 * 1024 * 1024) // DB map size will grow if that much space left on DB
#define DB_RESIZE_MIN_MAX_SIZE (50 * 1024 * 1024) // Minimum DB map size (starting size)
#define DB_RESIZE_INCREMENT_SIZE (100 * 1024 * 1024) // Grow step size
#define DB_RESIZE_COMMITS_TO_CHECK 50
#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
#define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
@ -27,13 +23,10 @@ namespace tools
{
namespace db
{
lmdb_db_backend::lmdb_db_backend()
: m_penv(AUTO_VAL_INIT(m_penv))
, m_commits_count(0)
lmdb_db_backend::lmdb_db_backend() : m_penv(AUTO_VAL_INIT(m_penv))
{
}
lmdb_db_backend::~lmdb_db_backend()
{
NESTED_TRY_ENTRY();
@ -43,16 +36,18 @@ namespace tools
NESTED_CATCH_ENTRY(__func__);
}
bool lmdb_db_backend::open(const std::string& path_, uint64_t /* flags -- unused atm */)
bool lmdb_db_backend::open(const std::string& path_, uint64_t cache_sz)
{
int res = 0;
res = mdb_env_create(&m_penv);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_create");
MDB_dbi max_dbs = 15;
res = mdb_env_set_maxdbs(m_penv, max_dbs);
res = mdb_env_set_maxdbs(m_penv, 15);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_maxdbs");
res = mdb_env_set_mapsize(m_penv, cache_sz);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize");
m_path = path_;
#ifdef WIN32
m_path = epee::string_encoding::convert_ansii_to_utf8(m_path);
@ -60,14 +55,8 @@ namespace tools
CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path);
// TODO: convert flags to lmdb_flags (implement fast lmdb modes)
unsigned int lmdb_flags = MDB_NORDAHEAD /*| MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC*/;
mdb_mode_t lmdb_mode = 0644;
res = mdb_env_open(m_penv, m_path.c_str(), lmdb_flags, lmdb_mode);
res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD , 0644);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_open, m_path=" << m_path);
resize_if_needed();
return true;
}
@ -116,13 +105,6 @@ namespace tools
{
LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3);
CRITICAL_SECTION_LOCK(m_write_exclusive_lock);
if (m_commits_count.fetch_add(1, std::memory_order_relaxed) % DB_RESIZE_COMMITS_TO_CHECK == DB_RESIZE_COMMITS_TO_CHECK - 1)
{
if (!resize_if_needed())
m_commits_count.store(DB_RESIZE_COMMITS_TO_CHECK - 1, std::memory_order_relaxed); // if failed, try again on next commit
}
}
PROFILE_FUNC("lmdb_db_backend::begin_transaction");
{
@ -347,7 +329,6 @@ namespace tools
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put");
return true;
}
bool lmdb_db_backend::enumerate(container_handle h, i_db_callback* pcb)
{
CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate");
@ -402,42 +383,6 @@ namespace tools
}
return true;
}
bool lmdb_db_backend::resize_if_needed()
{
LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED in resize_if_needed()", LOG_LEVEL_3);
CRITICAL_REGION_LOCAL(m_write_exclusive_lock);
if (have_tx())
{
LOG_PRINT_RED("[DB " << m_path << "] : resize_if_needed(): Have txs on stack, unable to resize!", LOG_LEVEL_0);
return false;
}
MDB_stat st = AUTO_VAL_INIT(st);
mdb_env_stat(m_penv, &st);
MDB_envinfo ei = AUTO_VAL_INIT(ei);
mdb_env_info(m_penv, &ei);
uint64_t dirty_size = ei.me_last_pgno * st.ms_psize;
int64_t size_diff = ei.me_mapsize - dirty_size;
if (size_diff >= DB_RESIZE_MIN_FREE_SIZE && ei.me_mapsize >= DB_RESIZE_MIN_MAX_SIZE)
return true; // resize is not needed
double gigabyte = 1024 * 1024 * 1024;
const uint64_t increment_size_pg_aligned = DB_RESIZE_INCREMENT_SIZE - (DB_RESIZE_INCREMENT_SIZE % st.ms_psize);
// need to resize DB
uint64_t new_size = ei.me_mapsize - (ei.me_mapsize % increment_size_pg_aligned) + increment_size_pg_aligned;
int res = mdb_env_set_mapsize(m_penv, new_size);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize");
LOG_PRINT_CYAN("[DB " << m_path << "] has grown: " << std::fixed << std::setprecision(2) << ei.me_mapsize / gigabyte << " GiB -> " << std::fixed << std::setprecision(2) << new_size / gigabyte << " GiB", LOG_LEVEL_0);
return true;
}
}
}

15
src/common/db_backend_lmdb.h
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
@ -6,7 +6,6 @@
#pragma once
#include <thread>
#include "include_base_utils.h"
#include "db_backend_base.h"
@ -17,9 +16,6 @@ namespace tools
{
namespace db
{
class lmdb_db_backend : public i_db_backend
{
@ -38,11 +34,7 @@ namespace tools
boost::recursive_mutex m_cs;
boost::recursive_mutex m_write_exclusive_lock;
std::map<std::thread::id, transactions_list> m_txs; // size_t -> count of nested read_only transactions
std::atomic<uint64_t> m_commits_count;
bool pop_tx_entry(tx_entry& txe);
public:
lmdb_db_backend();
~lmdb_db_backend();
@ -52,7 +44,7 @@ namespace tools
bool begin_transaction(bool read_only = false);
bool commit_transaction();
void abort_transaction();
bool open(const std::string& path, uint64_t flags = 0);
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE);
bool open_container(const std::string& name, container_handle& h);
bool erase(container_handle h, const char* k, size_t s);
bool get(container_handle h, const char* k, size_t s, std::string& res_buff);
@ -64,8 +56,7 @@ namespace tools
//-------------------------------------------------------------------------------------
bool have_tx();
MDB_txn* get_current_tx();
bool resize_if_needed();
};
}
}
}