lthn/blockchain
2
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
blockchain/tests/unit_tests/db_tests.cpp

1148 lines
33 KiB
C++
Raw Blame History

// Copyright (c) 2019 Zano Project
// Copyright (c) 2018 The Boolberry developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <memory>
#include <thread>
#include <atomic>
#include <boost/format.hpp>
#include <bitset>
#include "epee/include/include_base_utils.h"
#define USE_INSECURE_RANDOM_RPNG_ROUTINES // turns on random manupulation for tests
#include "crypto/crypto.h"
#include "gtest/gtest.h"
#include "common/db_abstract_accessor.h"
#include "common/db_backend_lmdb.h"
#include "serialization/serialization.h"
#include "common/db_backend_mdbx.h"
using namespace tools;
namespace db_test
{
crypto::hash null_hash = AUTO_VAL_INIT(null_hash);
template<typename T>
T random_t_from_range(T from, T to)
{
if (from >= to)
return from;
T result;
crypto::generate_random_bytes(sizeof result, &result);
return from + result % (to - from + 1);
}
//////////////////////////////////////////////////////////////////////////////
// basic_test
//////////////////////////////////////////////////////////////////////////////
template<typename db_backend_t>
void basic_test()
{
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(backend_ptr, db_lock);
bool r = false;
r = dbb.open("test_lmdb");
ASSERT_TRUE(r);
db::container_handle tid_decapod;
r = backend_ptr->open_container("decapod", tid_decapod);
ASSERT_TRUE(r);
ASSERT_TRUE(backend_ptr->begin_transaction());
ASSERT_TRUE(backend_ptr->clear(tid_decapod));
uint64_t key = 10;
std::string buf = "nxjdu47flrp20soam19e7nfhxbcy48owks03of92sbf31n1oqkanmdb47";
r = backend_ptr->set(tid_decapod, (char*)&key, sizeof key, buf.c_str(), buf.size());
ASSERT_TRUE(r);
ASSERT_TRUE(backend_ptr->commit_transaction());
r = dbb.close();
ASSERT_TRUE(r);
r = dbb.open("test_lmdb");
ASSERT_TRUE(r);
r = backend_ptr->open_container("decapod", tid_decapod);
ASSERT_TRUE(r);
ASSERT_TRUE(backend_ptr->begin_transaction());
std::string out_buffer;
r = backend_ptr->get(tid_decapod, (char*)&key, sizeof key, out_buffer);
ASSERT_TRUE(r);
ASSERT_EQ(buf, out_buffer);
ASSERT_TRUE(backend_ptr->commit_transaction());
r = dbb.close();
ASSERT_TRUE(r);
}
TEST(lmdb, basic_test)
{
basic_test<db::lmdb_db_backend>();
}
TEST(mdbx, basic_test)
{
basic_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// multithread_test_1
//////////////////////////////////////////////////////////////////////////////
template<typename db_backend_t>
struct multithread_test_1_t : public db::i_db_callback
{
enum { c_keys_count = 128 };
crypto::hash m_keys[c_keys_count];
size_t m_randomly_mixed_indexes_1[c_keys_count];
size_t m_randomly_mixed_indexes_2[c_keys_count];
epee::shared_recursive_mutex m_db_lock;
std::shared_ptr<db_backend_t> m_backend;
db::basic_db_accessor m_dbb;
db::container_handle m_table_id;
size_t m_counter;
multithread_test_1_t()
: m_backend(std::make_shared<db_backend_t>())
, m_dbb(m_backend, m_db_lock)
, m_table_id(0)
, m_counter(0)
{
for (size_t i = 0; i < c_keys_count; ++i)
{
m_keys[i] = i == 0 ? null_hash : crypto::cn_fast_hash(&m_keys[i - 1], sizeof(crypto::hash));
m_randomly_mixed_indexes_1[i] = i;
m_randomly_mixed_indexes_2[i] = i;
}
// prepare m_randomly_mixed_indexes_1 and m_randomly_mixed_indexes_2
for (size_t i = 0; i < c_keys_count * 100; ++i)
{
size_t a = random_t_from_range<size_t>(0, c_keys_count - 1);
size_t b = random_t_from_range<size_t>(0, c_keys_count - 1);
size_t c = random_t_from_range<size_t>(0, c_keys_count - 1);
size_t d = random_t_from_range<size_t>(0, c_keys_count - 1);
size_t tmp = m_randomly_mixed_indexes_1[a];
m_randomly_mixed_indexes_1[a] = m_randomly_mixed_indexes_1[b];
m_randomly_mixed_indexes_1[b] = tmp;
tmp = m_randomly_mixed_indexes_2[c];
m_randomly_mixed_indexes_2[c] = m_randomly_mixed_indexes_2[d];
m_randomly_mixed_indexes_2[d] = tmp;
}
}
void adder_thread(std::atomic<bool>& stop_flag)
{
epee::log_space::log_singletone::set_thread_log_prefix("[ adder ] ");
//epee::misc_utils::sleep_no_w(1000);
size_t i = 0;
for(size_t n = 0; n < 1000; ++n)
{
// get pseudorandom key index
size_t key_index = m_randomly_mixed_indexes_1[i];
i = (i + 1) % c_keys_count;
const crypto::hash& key = m_keys[key_index];
bool r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "begin_transaction");
// get a value by the given key
std::string value;
if (!m_backend->get(m_table_id, (const char*)&key, sizeof key, value))
{
// if such key does not exist -- add it
char buffer[128];
crypto::generate_random_bytes(sizeof buffer, buffer);
r = m_backend->set(m_table_id, (const char*)&key, sizeof key, buffer, sizeof buffer);
CHECK_AND_ASSERT_MES_NO_RET(r, "set");
size_t table_size = m_backend->size(m_table_id);
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "commit_transaction");
LOG_PRINT_L1("added key index " << key_index << ", table size: " << table_size);
}
else
{
// if key exists in the table - do nothing
m_backend->abort_transaction();
}
epee::misc_utils::sleep_no_w(1);
}
LOG_PRINT_L0("adder_thread stopped");
}
void deleter_thread(std::atomic<bool>& stop_flag)
{
epee::log_space::log_singletone::set_thread_log_prefix("[deleter] ");
epee::misc_utils::sleep_no_w(1000);
// get pseudorandom key index
size_t i = 0;
while(!stop_flag)
{
size_t key_index = m_randomly_mixed_indexes_2[i];
i = (i + 1) % c_keys_count;
const crypto::hash& key = m_keys[key_index];
bool r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "begin_transaction");
// get a value by the given key
std::string value;
if (m_backend->get(m_table_id, (const char*)&key, sizeof key, value))
{
// if key exists in the table -- remove it
r = m_backend->erase(m_table_id, (const char*)&key, sizeof key);
CHECK_AND_ASSERT_MES_NO_RET(r, "erase");
size_t table_size = m_backend->size(m_table_id);
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "commit_transaction");
LOG_PRINT_L1("erased key index " << key_index << ", table size: " << table_size);
if (table_size == 2)
break;
}
else
{
// if no such key exists in the table - do nothing
m_backend->abort_transaction();
}
epee::misc_utils::sleep_no_w(1);
}
LOG_PRINT_L0("deleter_thread stopped");
}
void reader_thread()
{
epee::log_space::log_singletone::set_thread_log_prefix("[reader ] ");
epee::misc_utils::sleep_no_w(1000);
// get pseudorandom key index
size_t i = 17;
uint64_t sum = 0; // just for fun
for(;;)
{
size_t key_index = m_randomly_mixed_indexes_2[i];
i = (i + 1) % c_keys_count;
const crypto::hash& key = m_keys[key_index];
bool r = m_backend->begin_transaction(true); // request Read-Only transaction
CHECK_AND_ASSERT_MES_NO_RET(r, "begin_transaction(RO=true)");
// get a value by the given key
std::string value;
if (m_backend->get(m_table_id, (const char*)&key, sizeof key, value))
{
sum += *reinterpret_cast<const uint64_t*>(value.data());
size_t table_size = m_backend->size(m_table_id);
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "commit_transaction");
if (table_size == 2)
break;
}
else
{
// if no such key exists in the table - do nothing
m_backend->abort_transaction();
}
epee::misc_utils::sleep_no_w(1);
}
LOG_PRINT_L0("reader_thread stopped, sum = " << sum);
}
bool check()
{
size_t table_size = m_backend->size(m_table_id);
CHECK_AND_ASSERT_MES(table_size == 2, false, "2 elements are expected to left");
m_counter = 0;
bool r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES(r, false, "begin_transaction");
m_backend->enumerate(m_table_id, this);
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES(r, false, "commit_transaction");
return m_counter == 2;
}
// class i_db_visitor
virtual bool on_enum_item(size_t i, const void* key_data, size_t key_size, const void* value_data, size_t value_size) override
{
CHECK_AND_ASSERT_MES(key_size == sizeof(crypto::hash), false, "invalid key size: " << key_size);
const crypto::hash *p_key = reinterpret_cast<const crypto::hash*>(key_data);
size_t key_index = SIZE_MAX;
for(size_t j = 0; j < c_keys_count && key_index == SIZE_MAX; ++j)
if (m_keys[j] == *p_key)
key_index = j;
CHECK_AND_ASSERT_MES(key_index != SIZE_MAX, false, "visitor gets a non-existing key");
LOG_PRINT_L1("visitor: #" << i << ", key index: " << key_index);
if (i != m_counter)
{
LOG_ERROR("invalid visitor index i: " << i << ", m_counter: " << m_counter);
m_counter = SIZE_MAX;
return false;
}
++m_counter;
return true;
};
bool run()
{
bool r = m_dbb.open("multithread_test_1_t");
CHECK_AND_ASSERT_MES(r, false, "m_dbb.open");
r = m_backend->open_container("table1_", m_table_id);
CHECK_AND_ASSERT_MES(r, false, "open_table");
r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES(r, false, "begin_transaction");
r = m_backend->clear(m_table_id);
CHECK_AND_ASSERT_MES(r, false, "clear_table");
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES(r, false, "commit_transaction");
std::atomic<bool> stop_adder(false), stop_deleter(false);
std::thread adder_t(&multithread_test_1_t::adder_thread, this, std::ref(stop_adder));
std::thread deleter_t(&multithread_test_1_t::deleter_thread, this, std::ref(stop_deleter));
std::vector<std::thread> readers_t;
const size_t reader_threads_count = 2;
for (size_t i = 0; i < reader_threads_count; ++i)
readers_t.emplace_back(std::thread(&multithread_test_1_t::reader_thread, this));
for (auto& t : readers_t)
t.join();
adder_t.join();
//stop_deleter = true;
deleter_t.join();
r = check();
m_dbb.close();
return r;
}
}; // multithread_test_1_t
template<typename db_backend_t>
void multithread_test_1()
{
char prng_state[200] = {};
crypto::random_prng_get_state(prng_state, sizeof prng_state); // store current RPNG state
crypto::random_prng_initialize_with_seed(0); // this mades this test deterministic
bool result = false;
try
{
multithread_test_1_t<db_backend_t> t;
result = t.run();
}
catch (std::exception& e)
{
LOG_ERROR("Caught exception: " << e.what());
}
// restore PRNG state to keep other tests unaffected
crypto::random_prng_get_state(prng_state, sizeof prng_state);
ASSERT_TRUE(result);
}
TEST(lmdb, multithread_test_1)
{
multithread_test_1<db::lmdb_db_backend>();
}
TEST(mdbx, multithread_test_1)
{
multithread_test_1<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// basic_accessor_test
//////////////////////////////////////////////////////////////////////////////
struct simple_serializable_t
{
std::string name;
uint64_t number;
BEGIN_SERIALIZE_OBJECT()
FIELD(name)
FIELD(number)
END_SERIALIZE()
};
inline bool operator==(const simple_serializable_t &lhs, const simple_serializable_t &rhs)
{
return lhs.name == rhs.name &&
lhs.number == rhs.number;
}
struct simple_pod_t
{
char c;
uint64_t u;
float f;
};
inline bool operator==(const simple_pod_t &_v1, const simple_pod_t &_v2)
{
return std::memcmp(&_v1, &_v2, sizeof _v1) == 0;
}
template<typename db_backend_t>
void basic_accessor_test()
{
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(backend_ptr, db_lock);
bool r = false;
r = dbb.open("bridge_basic_test");
ASSERT_TRUE(r);
db::container_handle tid_decapod;
r = backend_ptr->open_container("decapod", tid_decapod);
ASSERT_TRUE(r);
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(tid_decapod));
const char key[] = "nxjdu47flrp20soam19e7nfhxbcy48owks03of92sbf31n1oqkanmdb47";
simple_serializable_t s_object;
s_object.number = 1001100102;
s_object.name = "bender";
r = dbb.set_t_object(tid_decapod, key, s_object);
ASSERT_TRUE(r);
dbb.commit_transaction();
ASSERT_TRUE(dbb.begin_transaction());
simple_serializable_t s_object2;
r = dbb.get_t_object(tid_decapod, key, s_object2);
ASSERT_TRUE(r);
ASSERT_EQ(s_object, s_object2);
// del object by key and make sure it does not exist anymore
r = dbb.erase(tid_decapod, key);
ASSERT_TRUE(r);
r = dbb.get_t_object(tid_decapod, key, s_object2);
ASSERT_FALSE(r);
// second erase shoud also fail
r = dbb.erase(tid_decapod, key);
ASSERT_FALSE(r);
dbb.commit_transaction();
// POD type
const char key_pod[] = "alqocyfu7sbxhaoo5kdnrt77tgwderhjs9a9sdjf324nfjksd9f0s90f2";
ASSERT_TRUE(dbb.begin_transaction());
simple_pod_t p_object1 = {' ', 0xf7f7f7f7d3d3d3d3ull, 2.002319f};
r = dbb.set_pod_object(tid_decapod, key_pod, p_object1);
ASSERT_TRUE(r);
dbb.commit_transaction();
ASSERT_TRUE(dbb.begin_transaction());
simple_pod_t p_object2;
r = dbb.get_pod_object(tid_decapod, key_pod, p_object2);
ASSERT_TRUE(r);
ASSERT_EQ(p_object1, p_object2);
// del object by key and make sure it does not exist anymore
r = dbb.erase(tid_decapod, key_pod);
ASSERT_TRUE(r);
r = dbb.get_pod_object(tid_decapod, key_pod, p_object2);
ASSERT_FALSE(r);
// second erase shoud also fail
r = dbb.erase(tid_decapod, key_pod);
ASSERT_FALSE(r);
dbb.commit_transaction();
r = dbb.close();
ASSERT_TRUE(r);
}
TEST(lmdb, basic_accessor_test)
{
basic_accessor_test<db::lmdb_db_backend>();
}
TEST(mdbx, basic_accessor_test)
{
basic_accessor_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// single_value_test
//////////////////////////////////////////////////////////////////////////////
struct serializable_string
{
serializable_string() {}
explicit serializable_string(const std::string& v) : v(v) {}
std::string v;
BEGIN_SERIALIZE_OBJECT()
FIELD(v)
END_SERIALIZE()
};
template<typename db_backend_t>
void single_value_test()
{
const std::string options_table_name("options");
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::basic_key_value_accessor<uint64_t, uint64_t /* does not matter */, false /* does not matter */ > options_container(dbb);
db::solo_db_value<uint64_t, uint64_t, decltype(options_container), false> option_uint64(0, options_container);
db::solo_db_value<uint64_t, serializable_string, decltype(options_container), true> option_serializable_obj(1, options_container);
db::solo_db_value<uint64_t, crypto::hash, decltype(options_container), false> option_hash(2, options_container);
ASSERT_TRUE(dbb.open("single_value_test"));
// clear table
db::container_handle options_tid;
ASSERT_TRUE(backend_ptr->open_container(options_table_name, options_tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(options_tid));
dbb.commit_transaction();
ASSERT_TRUE(options_container.init(options_table_name));
// check defaults
ASSERT_TRUE(dbb.begin_transaction());
uint64_t v = option_uint64;
ASSERT_EQ(v, 0);
serializable_string ss = option_serializable_obj;
ASSERT_EQ(ss.v, std::string(""));
crypto::hash h = option_hash;
ASSERT_EQ(h, null_hash);
dbb.commit_transaction();
// set single values
ASSERT_TRUE(dbb.begin_transaction());
option_uint64 = 97;
option_serializable_obj = serializable_string("New York advertising men");
option_hash = crypto::cn_fast_hash(options_table_name.c_str(), options_table_name.size());
dbb.commit_transaction();
ASSERT_TRUE(dbb.close());
// reopen DB
ASSERT_TRUE(dbb.open("single_value_test"));
ASSERT_TRUE(options_container.init(options_table_name));
// get single value
ASSERT_TRUE(dbb.begin_transaction());
v = option_uint64;
ASSERT_EQ(v, 97);
ss = option_serializable_obj;
ASSERT_EQ(ss.v, "New York advertising men");
h = option_hash;
ASSERT_EQ(h, crypto::cn_fast_hash(options_table_name.c_str(), options_table_name.size()));
dbb.commit_transaction();
ASSERT_TRUE(dbb.close());
}
TEST(lmdb, single_value_test)
{
single_value_test<db::lmdb_db_backend>();
}
TEST(mdbx, single_value_test)
{
single_value_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// array_basic_test
//////////////////////////////////////////////////////////////////////////////
template<typename db_backend_t>
void array_basic_test()
{
bool r = false;
const std::string array_table_name("test_array");
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::basic_key_to_array_accessor<uint64_t, serializable_string, true> db_array(dbb);
ASSERT_TRUE(dbb.open("array_basic_test"));
// clear table
db::container_handle tid;
ASSERT_TRUE(backend_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(tid));
dbb.commit_transaction();
ASSERT_TRUE(db_array.init(array_table_name));
// check defaults
ASSERT_TRUE(dbb.begin_transaction());
size_t count = db_array.get_item_size(97);
ASSERT_EQ(count, 0);
std::shared_ptr<const serializable_string> ptr;
r = false;
try
{
ptr = db_array.get_subitem(97, 0);
}
catch (...)
{
r = true;
}
ASSERT_TRUE(r);
dbb.commit_transaction();
// write
ASSERT_TRUE(dbb.begin_transaction());
db_array.push_back_item(97, serializable_string("507507507507507507507507"));
db_array.push_back_item(97, serializable_string("787878787878787878787878"));
db_array.push_back_item(97, serializable_string("ringing phone"));
count = db_array.get_item_size(97);
ASSERT_EQ(count, 3);
dbb.commit_transaction();
ASSERT_TRUE(dbb.begin_transaction());
db_array.push_back_item(97, serializable_string("ring"));
db_array.push_back_item(97, serializable_string("ring"));
db_array.push_back_item(97, serializable_string("ring"));
count = db_array.get_item_size(97);
ASSERT_EQ(count, 6);
dbb.abort_transaction();
ASSERT_TRUE(dbb.begin_transaction());
count = db_array.get_item_size(97);
ASSERT_EQ(count, 3);
dbb.commit_transaction();
ASSERT_TRUE(dbb.close());
// reopen DB
ASSERT_TRUE(dbb.open("array_basic_test"));
ASSERT_TRUE(db_array.init(array_table_name));
ASSERT_TRUE(dbb.begin_transaction());
count = db_array.get_item_size(97);
ASSERT_EQ(count, 3);
ptr = db_array.get_subitem(97, 0);
ASSERT_TRUE((bool)ptr);
ASSERT_EQ(ptr->v, "507507507507507507507507");
ptr = db_array.get_subitem(97, 1);
ASSERT_TRUE((bool)ptr);
ASSERT_EQ(ptr->v, "787878787878787878787878");
ptr = db_array.get_subitem(97, 2);
ASSERT_TRUE((bool)ptr);
ASSERT_EQ(ptr->v, "ringing phone");
ASSERT_EQ(db_array.get_item_size(555), 0);
db_array.pop_back_item(555);
ASSERT_EQ(db_array.get_item_size(555), 0);
db_array.pop_back_item(97);
db_array.pop_back_item(97);
dbb.commit_transaction();
ASSERT_TRUE(dbb.begin_transaction());
count = db_array.get_item_size(97);
ASSERT_EQ(count, 1);
ptr = db_array.get_subitem(97, 0);
ASSERT_TRUE((bool)ptr);
ASSERT_EQ(ptr->v, "507507507507507507507507");
dbb.commit_transaction();
ASSERT_TRUE(dbb.close());
}
TEST(lmdb, array_basic_test)
{
array_basic_test<db::lmdb_db_backend>();
}
TEST(mdbx, array_basic_test)
{
array_basic_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// array_accessor_test
//////////////////////////////////////////////////////////////////////////////
template<typename db_backend_t>
void array_accessor_test()
{
bool r = false;
const std::string array_table_name("array");
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::array_accessor<serializable_string, true> db_array(dbb);
ASSERT_TRUE(dbb.open("array_accessor_test"));
// clear table
db::container_handle tid;
ASSERT_TRUE(backend_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(tid));
dbb.commit_transaction();
// check defaults
ASSERT_TRUE(db_array.init(array_table_name));
ASSERT_TRUE(db_array.begin_transaction(true));
size_t count = db_array.size();
ASSERT_EQ(count, 0);
count = db_array.size_no_cache();
ASSERT_EQ(count, 0);
std::shared_ptr<const serializable_string> ptr;
r = false;
try
{
ptr = db_array.back();
}
catch(...)
{
r = true;
}
ASSERT_TRUE(r);
ASSERT_FALSE(db_array.clear()); // clear() should fail on read-only transaction
r = false;
try
{
ptr = db_array[0];
}
catch(...)
{
r = true;
}
ASSERT_TRUE(r);
db_array.commit_transaction();
ASSERT_TRUE(db_array.begin_transaction());
db_array.push_back(serializable_string("A"));
db_array.push_back(serializable_string("B"));
db_array.push_back(serializable_string("C"));
db_array.push_back(serializable_string("D"));
db_array.push_back(serializable_string("E"));
db_array.commit_transaction();
ASSERT_TRUE(db_array.begin_transaction());
ptr = db_array[4];
ASSERT_EQ(ptr->v, "E");
db_array.pop_back();
ASSERT_EQ(db_array.size(), 4);
r = false;
try
{
ptr = db_array[4];
}
catch(...)
{
r = true;
}
ASSERT_TRUE(r);
ptr = db_array[3];
ASSERT_EQ(ptr->v, "D");
db_array.push_back(serializable_string("X"));
ptr = db_array[4];
ASSERT_EQ(ptr->v, "X");
ASSERT_TRUE(db_array.clear());
db_array.commit_transaction();
ASSERT_TRUE(db_array.begin_transaction(true));
count = db_array.size();
ASSERT_EQ(count, 0);
db_array.commit_transaction();
}
TEST(lmdb, array_accessor_test)
{
array_accessor_test<db::lmdb_db_backend>();
}
TEST(mdbx, array_accessor_test)
{
array_accessor_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// key_value_test
//////////////////////////////////////////////////////////////////////////////
template<typename db_backend_t>
void key_value_test()
{
bool r = false;
const std::string array_table_name("key-value");
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::basic_key_value_accessor<uint64_t, serializable_string, true> db_key_value_map(dbb);
ASSERT_TRUE(dbb.open("key-value_test"));
// clear table
db::container_handle tid;
ASSERT_TRUE(backend_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(tid));
dbb.commit_transaction();
// check defaults
ASSERT_TRUE(db_key_value_map.init(array_table_name));
ASSERT_TRUE(db_key_value_map.begin_transaction(true));
size_t count = db_key_value_map.size();
ASSERT_EQ(count, 0);
count = db_key_value_map.size_no_cache();
ASSERT_EQ(count, 0);
std::shared_ptr<const serializable_string> ptr;
r = false;
try
{
ptr = db_key_value_map[99];
}
catch (std::out_of_range&)
{
r = true;
}
ASSERT_TRUE(r);
ptr = db_key_value_map.get(99);
ASSERT_TRUE(!ptr);
ASSERT_FALSE(db_key_value_map.clear()); // clear() should fail on read-only transaction
db_key_value_map.commit_transaction();
ASSERT_TRUE(db_key_value_map.begin_transaction());
db_key_value_map.set(99, serializable_string("A"));
db_key_value_map.set(100, serializable_string("B"));
db_key_value_map.set(101, serializable_string("C"));
db_key_value_map.set(102, serializable_string("D"));
db_key_value_map.commit_transaction();
ASSERT_TRUE(db_key_value_map.begin_transaction());
ASSERT_EQ(db_key_value_map.size(), 4);
ptr = db_key_value_map[102];
ASSERT_EQ(ptr->v, "D");
ASSERT_TRUE(db_key_value_map.erase_validate(102));
ASSERT_EQ(db_key_value_map.size(), 3);
db_key_value_map.erase(102);
ASSERT_EQ(db_key_value_map.size(), 3);
ASSERT_FALSE(db_key_value_map.erase_validate(102));
ASSERT_EQ(db_key_value_map.size(), 3);
r = false;
try
{
ptr = db_key_value_map[0];
}
catch (std::out_of_range&)
{
r = true;
}
ASSERT_TRUE(r);
ptr = db_key_value_map[99];
ASSERT_EQ(ptr->v, "A");
db_key_value_map.set(102, serializable_string("W"));
ASSERT_EQ(db_key_value_map.size(), 4);
ptr = db_key_value_map[102];
ASSERT_EQ(ptr->v, "W");
ASSERT_TRUE(db_key_value_map.clear());
db_key_value_map.commit_transaction();
ASSERT_TRUE(db_key_value_map.begin_transaction(true));
ASSERT_EQ(db_key_value_map.size(), 0);
db_key_value_map.commit_transaction();
}
TEST(lmdb, key_value_test)
{
key_value_test<db::lmdb_db_backend>();
}
TEST(mdbx, key_value_test)
{
key_value_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// 2gb_test
//////////////////////////////////////////////////////////////////////////////
template<typename db_backend_t>
void db_2gb_test()
{
bool r = false;
epee::shared_recursive_mutex rw_lock;
epee::log_space::log_singletone::enable_channels("lmdb");
static const uint64_t buffer_size = 64 * 1024; // 64 KB
static const uint64_t db_total_size = static_cast<uint64_t>(2.1 * 1024 * 1024 * 1024); // 2.1 GB -- a bit more than 2GB to test 2GB boundary
static const std::string db_file_path = boost::algorithm::replace_all_copy(boost::algorithm::replace_all_copy(std::string("2gb_") + typeid(db_backend_t).name() + "_test", ":", "_"), " ", "_");
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
db::basic_db_accessor bdba(backend_ptr, rw_lock);
//
// write data
//
r = bdba.open(db_file_path, CACHE_SIZE);
ASSERT_TRUE(r);
db::container_handle h;
r = backend_ptr->open_container("c1", h);
ASSERT_TRUE(r);
ASSERT_TRUE(backend_ptr->begin_transaction());
ASSERT_TRUE(backend_ptr->clear(h));
std::vector<uint8_t> buffer;
buffer.resize(buffer_size);
crypto::generate_random_bytes(buffer_size, buffer.data());
std::vector<std::vector<uint8_t>> buffer_paranoidal_copies;
buffer_paranoidal_copies.resize(3);
for (size_t i = 0; i < buffer_paranoidal_copies.size(); ++i)
buffer_paranoidal_copies[i].assign(buffer.begin(), buffer.end());
auto check_buffer_paranoidal_copies = [&]() {
for (size_t buffer_index = 0; buffer_index < buffer_paranoidal_copies.size(); ++buffer_index)
{
for(size_t i = 0; i < buffer_size; ++i)
{
if (buffer[i] != buffer_paranoidal_copies[buffer_index][i])
{
std::cout << "!!! buffer differs from paranoidal copy #" << buffer_index << " at byte " << i << ": " << static_cast<uint32_t>(buffer[i]) << " != " << static_cast<uint32_t>(buffer_paranoidal_copies[buffer_index][i]) << std::endl;
break;
}
}
}
};
check_buffer_paranoidal_copies();
uint64_t total_data = 0;
for (uint64_t key = 0; key < db_total_size / buffer_size; ++key)
{
r = backend_ptr->set(h, (char*)&key, sizeof key, reinterpret_cast<const char*>(buffer.data()), buffer_size);
ASSERT_TRUE(r);
total_data += buffer_size;
if (key % 1024 == 0)
{
ASSERT_TRUE(backend_ptr->commit_transaction());
//ASSERT_TRUE(backend_ptr->resize_if_needed());
ASSERT_TRUE(backend_ptr->begin_transaction());
std::cout << total_data / 1024 / 1024 << " MB written to DB" << ENDL;
}
}
ASSERT_TRUE(backend_ptr->commit_transaction());
r = bdba.close();
ASSERT_TRUE(r);
//
// read data and check
//
r = bdba.open(db_file_path);
ASSERT_TRUE(r);
r = backend_ptr->open_container("c1", h);
ASSERT_TRUE(r);
ASSERT_TRUE(backend_ptr->begin_transaction());
std::string out_buffer;
total_data = 0;
for (uint64_t key = 0; key < db_total_size / buffer_size; ++key)
{
r = backend_ptr->get(h, (char*)&key, sizeof key, out_buffer);
ASSERT_TRUE(r);
ASSERT_EQ(buffer_size, out_buffer.size());
if (memcmp(buffer.data(), out_buffer.c_str(), buffer_size) != 0)
{
// read data doesn't match with written one
std::cout << "ERROR: data missmatch at key " << key << ", total_data = " << total_data << std::endl;
// paranoid checks
check_buffer_paranoidal_copies();
size_t wrong_bytes = 0;
size_t wrong_bytes_min_idx = SIZE_MAX;
size_t wrong_bytes_max_idx = 0;
for (size_t i = 0; i < buffer_size; ++i)
{
if (buffer[i] != static_cast<unsigned char>(out_buffer[i]))
{
++wrong_bytes;
if (wrong_bytes_min_idx == SIZE_MAX)
wrong_bytes_min_idx = i;
if (wrong_bytes_max_idx < i)
wrong_bytes_max_idx = i;
if (wrong_bytes < 10)
{
std::cout << "wrong byte at buffer offset " << boost::format("0x%04x") % i << ", file offset " << boost::format("0x%08x") % (total_data + i) << ": " <<
boost::format("%02x") % static_cast<unsigned int>(static_cast<unsigned char>(out_buffer[i])) << " = " << std::bitset<8>(out_buffer[i]) << " instead of " <<
boost::format("%02x") % static_cast<unsigned int>(buffer[i]) << " = " << std::bitset<8>(buffer[i]) << std::endl;
}
}
}
std::cout << "wrong bytes: " << wrong_bytes << " of " << buffer_size << " (" << std::fixed << std::setprecision(2) << 100.0 * wrong_bytes / buffer_size << "%)" << std::endl;
size_t line_len = 32;
size_t wrong_bytes_min_line = wrong_bytes_min_idx / line_len;
size_t wrong_bytes_max_line = wrong_bytes_max_idx / line_len + 1;
for (size_t l = wrong_bytes_min_line; l < wrong_bytes_max_line; ++l)
{
std::cout << boost::format("\n0x%04x ") % ( l * line_len );
for(size_t i = l * line_len; i < (l + 1) * line_len; ++i)
std::cout << boost::format("%02x") % static_cast<unsigned int>(buffer[i]) << (i % 4 == 3 ? " " : "");
std::cout << " ";
for(size_t i = l * line_len; i < (l + 1) * line_len; ++i)
std::cout << boost::format("%02x") % static_cast<unsigned int>(static_cast<unsigned char>(out_buffer[i])) << (i % 4 == 3 ? " " : "");
}
std::cout << std::endl;
ASSERT_TRUE(false);
}
total_data += buffer_size;
if (key % 1024 == 0)
std::cout << total_data / 1024 / 1024 << " MB read from DB" << ENDL;
}
ASSERT_TRUE(backend_ptr->commit_transaction());
r = bdba.close();
ASSERT_TRUE(r);
boost::filesystem::remove_all(db_file_path);
}
TEST(lmdb, 2gb_test)
{
db_2gb_test<db::lmdb_db_backend>();
}
TEST(mdbx, 2gb_test)
{
db_2gb_test<db::mdbx_db_backend>();
}
} // namespace lmdb_test
Reference in a new issue View git blame Copy permalink