Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity

unit_test: all lmdb tests adapted for mdbx too

Browse source
This commit is contained in:
sowle 2020-07-07 13:19:27 +03:00
parent ff7dfff21c
commit 8a3f3579c1
No known key found for this signature in database
GPG key ID: C07A24B2D89D49FC
1 changed files with 159 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

238
tests/unit_tests/lmdb_tests.cpp → tests/unit_tests/db_tests.cpp
View file

@ -20,7 +20,7 @@
using namespace tools;
namespace lmdb_test
namespace db_test
{
crypto::hash null_hash = AUTO_VAL_INIT(null_hash);
@ -38,11 +38,12 @@ namespace lmdb_test
//////////////////////////////////////////////////////////////////////////////
// basic_test
//////////////////////////////////////////////////////////////////////////////
TEST(lmdb, basic_test)
template<typename db_backend_t>
void basic_test()
{
std::shared_ptr<db::lmdb_db_backend> lmdb_ptr = std::make_shared<db::lmdb_db_backend>();
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(lmdb_ptr, db_lock);
db::basic_db_accessor dbb(backend_ptr, db_lock);
bool r = false;
@ -50,20 +51,20 @@ namespace lmdb_test
ASSERT_TRUE(r);
db::container_handle tid_decapod;
r = lmdb_ptr->open_container("decapod", tid_decapod);
r = backend_ptr->open_container("decapod", tid_decapod);
ASSERT_TRUE(r);
ASSERT_TRUE(lmdb_ptr->begin_transaction());
ASSERT_TRUE(backend_ptr->begin_transaction());
ASSERT_TRUE(lmdb_ptr->clear(tid_decapod));
ASSERT_TRUE(backend_ptr->clear(tid_decapod));
uint64_t key = 10;
std::string buf = "nxjdu47flrp20soam19e7nfhxbcy48owks03of92sbf31n1oqkanmdb47";
r = lmdb_ptr->set(tid_decapod, (char*)&key, sizeof key, buf.c_str(), buf.size());
r = backend_ptr->set(tid_decapod, (char*)&key, sizeof key, buf.c_str(), buf.size());
ASSERT_TRUE(r);
ASSERT_TRUE(lmdb_ptr->commit_transaction());
ASSERT_TRUE(backend_ptr->commit_transaction());
r = dbb.close();
ASSERT_TRUE(r);
@ -71,26 +72,38 @@ namespace lmdb_test
r = dbb.open("test_lmdb");
ASSERT_TRUE(r);
r = lmdb_ptr->open_container("decapod", tid_decapod);
r = backend_ptr->open_container("decapod", tid_decapod);
ASSERT_TRUE(r);
ASSERT_TRUE(lmdb_ptr->begin_transaction());
ASSERT_TRUE(backend_ptr->begin_transaction());
std::string out_buffer;
r = lmdb_ptr->get(tid_decapod, (char*)&key, sizeof key, out_buffer);
r = backend_ptr->get(tid_decapod, (char*)&key, sizeof key, out_buffer);
ASSERT_TRUE(r);
ASSERT_EQ(buf, out_buffer);
ASSERT_TRUE(lmdb_ptr->commit_transaction());
ASSERT_TRUE(backend_ptr->commit_transaction());
r = dbb.close();
ASSERT_TRUE(r);
}
TEST(lmdb, basic_test)
{
basic_test<db::lmdb_db_backend>();
}
TEST(mdbx, basic_test)
{
basic_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// multithread_test_1
//////////////////////////////////////////////////////////////////////////////
template<typename db_backend_t>
struct multithread_test_1_t : public db::i_db_callback
{
enum { c_keys_count = 128 };
@ -98,14 +111,14 @@ namespace lmdb_test
size_t m_randomly_mixed_indexes_1[c_keys_count];
size_t m_randomly_mixed_indexes_2[c_keys_count];
epee::shared_recursive_mutex m_db_lock;
std::shared_ptr<db::lmdb_db_backend> m_lmdb_adapter;
std::shared_ptr<db_backend_t> m_backend;
db::basic_db_accessor m_dbb;
db::container_handle m_table_id;
size_t m_counter;
multithread_test_1_t()
: m_lmdb_adapter(std::make_shared<db::lmdb_db_backend>())
, m_dbb(m_lmdb_adapter, m_db_lock)
: m_backend(std::make_shared<db_backend_t>())
, m_dbb(m_backend, m_db_lock)
, m_table_id(0)
, m_counter(0)
{
@ -147,22 +160,22 @@ namespace lmdb_test
i = (i + 1) % c_keys_count;
const crypto::hash& key = m_keys[key_index];
bool r = m_lmdb_adapter->begin_transaction();
bool r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "begin_transaction");
// get a value by the given key
std::string value;
if (!m_lmdb_adapter->get(m_table_id, (const char*)&key, sizeof key, value))
if (!m_backend->get(m_table_id, (const char*)&key, sizeof key, value))
{
// if such key does not exist -- add it
char buffer[128];
crypto::generate_random_bytes(sizeof buffer, buffer);
r = m_lmdb_adapter->set(m_table_id, (const char*)&key, sizeof key, buffer, sizeof buffer);
r = m_backend->set(m_table_id, (const char*)&key, sizeof key, buffer, sizeof buffer);
CHECK_AND_ASSERT_MES_NO_RET(r, "set");
size_t table_size = m_lmdb_adapter->size(m_table_id);
size_t table_size = m_backend->size(m_table_id);
r = m_lmdb_adapter->commit_transaction();
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "commit_transaction");
LOG_PRINT_L1("added key index " << key_index << ", table size: " << table_size);
@ -170,7 +183,7 @@ namespace lmdb_test
else
{
// if key exists in the table - do nothing
m_lmdb_adapter->abort_transaction();
m_backend->abort_transaction();
}
epee::misc_utils::sleep_no_w(1);
}
@ -190,20 +203,20 @@ namespace lmdb_test
size_t key_index = m_randomly_mixed_indexes_2[i];
i = (i + 1) % c_keys_count;
const crypto::hash& key = m_keys[key_index];
bool r = m_lmdb_adapter->begin_transaction();
bool r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "begin_transaction");
// get a value by the given key
std::string value;
if (m_lmdb_adapter->get(m_table_id, (const char*)&key, sizeof key, value))
if (m_backend->get(m_table_id, (const char*)&key, sizeof key, value))
{
// if key exists in the table -- remove it
r = m_lmdb_adapter->erase(m_table_id, (const char*)&key, sizeof key);
r = m_backend->erase(m_table_id, (const char*)&key, sizeof key);
CHECK_AND_ASSERT_MES_NO_RET(r, "erase");
size_t table_size = m_lmdb_adapter->size(m_table_id);
size_t table_size = m_backend->size(m_table_id);
r = m_lmdb_adapter->commit_transaction();
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "commit_transaction");
LOG_PRINT_L1("erased key index " << key_index << ", table size: " << table_size);
@ -213,7 +226,7 @@ namespace lmdb_test
else
{
// if no such key exists in the table - do nothing
m_lmdb_adapter->abort_transaction();
m_backend->abort_transaction();
}
epee::misc_utils::sleep_no_w(1);
}
@ -234,18 +247,18 @@ namespace lmdb_test
size_t key_index = m_randomly_mixed_indexes_2[i];
i = (i + 1) % c_keys_count;
const crypto::hash& key = m_keys[key_index];
bool r = m_lmdb_adapter->begin_transaction(true); // request Read-Only transaction
bool r = m_backend->begin_transaction(true); // request Read-Only transaction
CHECK_AND_ASSERT_MES_NO_RET(r, "begin_transaction(RO=true)");
// get a value by the given key
std::string value;
if (m_lmdb_adapter->get(m_table_id, (const char*)&key, sizeof key, value))
if (m_backend->get(m_table_id, (const char*)&key, sizeof key, value))
{
sum += *reinterpret_cast<const uint64_t*>(value.data());
size_t table_size = m_lmdb_adapter->size(m_table_id);
size_t table_size = m_backend->size(m_table_id);
r = m_lmdb_adapter->commit_transaction();
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES_NO_RET(r, "commit_transaction");
if (table_size == 2)
@ -254,7 +267,7 @@ namespace lmdb_test
else
{
// if no such key exists in the table - do nothing
m_lmdb_adapter->abort_transaction();
m_backend->abort_transaction();
}
epee::misc_utils::sleep_no_w(1);
}
@ -263,16 +276,16 @@ namespace lmdb_test
bool check()
{
size_t table_size = m_lmdb_adapter->size(m_table_id);
size_t table_size = m_backend->size(m_table_id);
CHECK_AND_ASSERT_MES(table_size == 2, false, "2 elements are expected to left");
m_counter = 0;
bool r = m_lmdb_adapter->begin_transaction();
bool r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES(r, false, "begin_transaction");
m_lmdb_adapter->enumerate(m_table_id, this);
m_backend->enumerate(m_table_id, this);
r = m_lmdb_adapter->commit_transaction();
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES(r, false, "commit_transaction");
return m_counter == 2;
@ -308,13 +321,13 @@ namespace lmdb_test
{
bool r = m_dbb.open("multithread_test_1_t");
CHECK_AND_ASSERT_MES(r, false, "m_dbb.open");
r = m_lmdb_adapter->open_container("table1_", m_table_id);
r = m_backend->open_container("table1_", m_table_id);
CHECK_AND_ASSERT_MES(r, false, "open_table");
r = m_lmdb_adapter->begin_transaction();
r = m_backend->begin_transaction();
CHECK_AND_ASSERT_MES(r, false, "begin_transaction");
r = m_lmdb_adapter->clear(m_table_id);
r = m_backend->clear(m_table_id);
CHECK_AND_ASSERT_MES(r, false, "clear_table");
r = m_lmdb_adapter->commit_transaction();
r = m_backend->commit_transaction();
CHECK_AND_ASSERT_MES(r, false, "commit_transaction");
std::atomic<bool> stop_adder(false), stop_deleter(false);
@ -336,9 +349,10 @@ namespace lmdb_test
m_dbb.close();
return r;
}
};
}; // multithread_test_1_t
TEST(lmdb, multithread_test_1)
template<typename db_backend_t>
void multithread_test_1()
{
char prng_state[200] = {};
crypto::random_prng_get_state(prng_state, sizeof prng_state); // store current RPNG state
@ -347,7 +361,7 @@ namespace lmdb_test
bool result = false;
try
{
multithread_test_1_t t;
multithread_test_1_t<db_backend_t> t;
result = t.run();
}
catch (std::exception& e)
@ -361,8 +375,19 @@ namespace lmdb_test
ASSERT_TRUE(result);
}
TEST(lmdb, multithread_test_1)
{
multithread_test_1<db::lmdb_db_backend>();
}
TEST(mdbx, multithread_test_1)
{
multithread_test_1<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// bridge_basic_test
// basic_accessor_test
//////////////////////////////////////////////////////////////////////////////
struct simple_serializable_t
{
@ -393,11 +418,12 @@ namespace lmdb_test
return std::memcmp(&_v1, &_v2, sizeof _v1) == 0;
}
TEST(lmdb, bridge_basic_test)
template<typename db_backend_t>
void basic_accessor_test()
{
std::shared_ptr<db::lmdb_db_backend> lmdb_ptr = std::make_shared<db::lmdb_db_backend>();
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(lmdb_ptr, db_lock);
db::basic_db_accessor dbb(backend_ptr, db_lock);
bool r = false;
@ -405,7 +431,7 @@ namespace lmdb_test
ASSERT_TRUE(r);
db::container_handle tid_decapod;
r = lmdb_ptr->open_container("decapod", tid_decapod);
r = backend_ptr->open_container("decapod", tid_decapod);
ASSERT_TRUE(r);
ASSERT_TRUE(dbb.begin_transaction());
@ -474,6 +500,16 @@ namespace lmdb_test
ASSERT_TRUE(r);
}
TEST(lmdb, basic_accessor_test)
{
basic_accessor_test<db::lmdb_db_backend>();
}
TEST(mdbx, basic_accessor_test)
{
basic_accessor_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// single_value_test
//////////////////////////////////////////////////////////////////////////////
@ -489,13 +525,14 @@ namespace lmdb_test
END_SERIALIZE()
};
TEST(lmdb, single_value_test)
template<typename db_backend_t>
void single_value_test()
{
const std::string options_table_name("options");
std::shared_ptr<db::lmdb_db_backend> lmdb_ptr = std::make_shared<db::lmdb_db_backend>();
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(lmdb_ptr, db_lock);
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::basic_key_value_accessor<uint64_t, uint64_t /* does not matter */, false /* does not matter */ > options_container(dbb);
db::solo_db_value<uint64_t, uint64_t, decltype(options_container), false> option_uint64(0, options_container);
@ -506,7 +543,7 @@ namespace lmdb_test
// clear table
db::container_handle options_tid;
ASSERT_TRUE(lmdb_ptr->open_container(options_table_name, options_tid));
ASSERT_TRUE(backend_ptr->open_container(options_table_name, options_tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(options_tid));
dbb.commit_transaction();
@ -560,19 +597,29 @@ namespace lmdb_test
ASSERT_TRUE(dbb.close());
}
TEST(lmdb, single_value_test)
{
single_value_test<db::lmdb_db_backend>();
}
TEST(mdbx, single_value_test)
{
single_value_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// array_basic_test
//////////////////////////////////////////////////////////////////////////////
TEST(lmdb, array_basic_test)
template<typename db_backend_t>
void array_basic_test()
{
bool r = false;
const std::string array_table_name("test_array");
std::shared_ptr<db::lmdb_db_backend> lmdb_ptr = std::make_shared<db::lmdb_db_backend>();
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(lmdb_ptr, db_lock);
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::basic_key_to_array_accessor<uint64_t, serializable_string, true> db_array(dbb);
@ -580,7 +627,7 @@ namespace lmdb_test
// clear table
db::container_handle tid;
ASSERT_TRUE(lmdb_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(backend_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(tid));
dbb.commit_transaction();
@ -681,18 +728,30 @@ namespace lmdb_test
ASSERT_TRUE(dbb.close());
}
TEST(lmdb, array_basic_test)
{
array_basic_test<db::lmdb_db_backend>();
}
TEST(mdbx, array_basic_test)
{
array_basic_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// array_accessor_test
//////////////////////////////////////////////////////////////////////////////
TEST(lmdb, array_accessor_test)
template<typename db_backend_t>
void array_accessor_test()
{
bool r = false;
const std::string array_table_name("array");
std::shared_ptr<db::lmdb_db_backend> lmdb_ptr = std::make_shared<db::lmdb_db_backend>();
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(lmdb_ptr, db_lock);
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::array_accessor<serializable_string, true> db_array(dbb);
@ -700,7 +759,7 @@ namespace lmdb_test
// clear table
db::container_handle tid;
ASSERT_TRUE(lmdb_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(backend_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(tid));
dbb.commit_transaction();
@ -787,18 +846,29 @@ namespace lmdb_test
db_array.commit_transaction();
}
TEST(lmdb, array_accessor_test)
{
array_accessor_test<db::lmdb_db_backend>();
}
TEST(mdbx, array_accessor_test)
{
array_accessor_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// key_value_test
//////////////////////////////////////////////////////////////////////////////
TEST(lmdb, key_value_test)
template<typename db_backend_t>
void key_value_test()
{
bool r = false;
const std::string array_table_name("key-value");
std::shared_ptr<db::lmdb_db_backend> lmdb_ptr = std::make_shared<db::lmdb_db_backend>();
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
epee::shared_recursive_mutex db_lock;
db::basic_db_accessor dbb(lmdb_ptr, db_lock);
db::basic_db_accessor dbb(backend_ptr, db_lock);
db::basic_key_value_accessor<uint64_t, serializable_string, true> db_key_value_map(dbb);
@ -806,7 +876,7 @@ namespace lmdb_test
// clear table
db::container_handle tid;
ASSERT_TRUE(lmdb_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(backend_ptr->open_container(array_table_name, tid));
ASSERT_TRUE(dbb.begin_transaction());
ASSERT_TRUE(dbb.clear(tid));
dbb.commit_transaction();
@ -889,6 +959,16 @@ namespace lmdb_test
db_key_value_map.commit_transaction();
}
TEST(lmdb, key_value_test)
{
key_value_test<db::lmdb_db_backend>();
}
TEST(mdbx, key_value_test)
{
key_value_test<db::mdbx_db_backend>();
}
//////////////////////////////////////////////////////////////////////////////
// 2gb_test
@ -905,8 +985,8 @@ namespace lmdb_test
static const uint64_t db_total_size = static_cast<uint64_t>(2.1 * 1024 * 1024 * 1024); // 2.1 GB -- a bit more than 2GB to test 2GB boundary
static const std::string db_file_path = boost::algorithm::replace_all_copy(boost::algorithm::replace_all_copy(std::string("2gb_") + typeid(db_backend_t).name() + "_test", ":", "_"), " ", "_");
std::shared_ptr<db_backend_t> lmdb_ptr = std::make_shared<db_backend_t>();
db::basic_db_accessor bdba(lmdb_ptr, rw_lock);
std::shared_ptr<db_backend_t> backend_ptr = std::make_shared<db_backend_t>();
db::basic_db_accessor bdba(backend_ptr, rw_lock);
//
// write data
@ -916,11 +996,11 @@ namespace lmdb_test
ASSERT_TRUE(r);
db::container_handle h;
r = lmdb_ptr->open_container("c1", h);
r = backend_ptr->open_container("c1", h);
ASSERT_TRUE(r);
ASSERT_TRUE(lmdb_ptr->begin_transaction());
ASSERT_TRUE(lmdb_ptr->clear(h));
ASSERT_TRUE(backend_ptr->begin_transaction());
ASSERT_TRUE(backend_ptr->clear(h));
std::vector<uint8_t> buffer;
buffer.resize(buffer_size);
@ -950,20 +1030,20 @@ namespace lmdb_test
uint64_t total_data = 0;
for (uint64_t key = 0; key < db_total_size / buffer_size; ++key)
{
r = lmdb_ptr->set(h, (char*)&key, sizeof key, reinterpret_cast<const char*>(buffer.data()), buffer_size);
r = backend_ptr->set(h, (char*)&key, sizeof key, reinterpret_cast<const char*>(buffer.data()), buffer_size);
ASSERT_TRUE(r);
total_data += buffer_size;
if (key % 1024 == 0)
{
ASSERT_TRUE(lmdb_ptr->commit_transaction());
//ASSERT_TRUE(lmdb_ptr->resize_if_needed());
ASSERT_TRUE(lmdb_ptr->begin_transaction());
ASSERT_TRUE(backend_ptr->commit_transaction());
//ASSERT_TRUE(backend_ptr->resize_if_needed());
ASSERT_TRUE(backend_ptr->begin_transaction());
std::cout << total_data / 1024 / 1024 << " MB written to DB" << ENDL;
}
}
ASSERT_TRUE(lmdb_ptr->commit_transaction());
ASSERT_TRUE(backend_ptr->commit_transaction());
r = bdba.close();
ASSERT_TRUE(r);
@ -975,16 +1055,16 @@ namespace lmdb_test
r = bdba.open(db_file_path);
ASSERT_TRUE(r);
r = lmdb_ptr->open_container("c1", h);
r = backend_ptr->open_container("c1", h);
ASSERT_TRUE(r);
ASSERT_TRUE(lmdb_ptr->begin_transaction());
ASSERT_TRUE(backend_ptr->begin_transaction());
std::string out_buffer;
total_data = 0;
for (uint64_t key = 0; key < db_total_size / buffer_size; ++key)
{
r = lmdb_ptr->get(h, (char*)&key, sizeof key, out_buffer);
r = backend_ptr->get(h, (char*)&key, sizeof key, out_buffer);
ASSERT_TRUE(r);
ASSERT_EQ(buffer_size, out_buffer.size());
@ -1046,7 +1126,7 @@ namespace lmdb_test
std::cout << total_data / 1024 / 1024 << " MB read from DB" << ENDL;
}
ASSERT_TRUE(lmdb_ptr->commit_transaction());
ASSERT_TRUE(backend_ptr->commit_transaction());
r = bdba.close();
ASSERT_TRUE(r);