lthn/node-util
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updated to latest codebase

Browse source
This commit is contained in:
crypro.zoidberg 2021-06-02 05:53:19 +03:00
parent 71b05fbc29
commit f046b1490d
No known key found for this signature in database
GPG key ID: 22DEB97A54C6FDEC
578 changed files with 116275 additions and 3706 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/common/atomics_boost_serialization.h Normal file
View file

@ -0,0 +1,32 @@
// Copyright (c) 2014-2018 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <atomic>
namespace boost
{
namespace serialization
{
template <class Archive, class value_t>
inline void save(Archive &a, const std::atomic<value_t> &x, const boost::serialization::version_type ver)
{
a << x.load();
}
template <class Archive, class value_t>
inline void load(Archive &a, std::atomic<value_t> &x, const boost::serialization::version_type ver)
{
value_t s = AUTO_VAL_INIT(s);
a >> s;
x.store(s);
}
template <class Archive, class value_t>
inline void serialize(Archive &a, std::atomic<value_t> &x, const boost::serialization::version_type ver)
{
split_free(a, x, ver);
}
}
}

18
src/common/boost_serialization_helper.h
View file

@ -21,8 +21,8 @@ namespace tools
bool serialize_obj_to_file(t_object& obj, const std::string& file_path)
{
TRY_ENTRY();
std::ofstream data_file;
data_file.open( file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc);
boost::filesystem::ofstream data_file;
data_file.open( epee::string_encoding::utf8_to_wstring(file_path) , std::ios_base::binary | std::ios_base::out| std::ios::trunc);
if(data_file.fail())
return false;
@ -30,7 +30,7 @@ namespace tools
a << obj;
return !data_file.fail();
CATCH_ENTRY_L0("serialize_obj_to_file", false);
CATCH_ENTRY_L0("serialize_obj_to_file: could not serialize into " << file_path, false);
}
@ -56,7 +56,7 @@ namespace tools
a << obj;
return !stream.fail();
CATCH_ENTRY_L0("serialize_obj_to_file", false);
CATCH_ENTRY_L0("portble_serialize_obj_to_stream", false);
}
template<class t_object>
@ -64,15 +64,15 @@ namespace tools
{
TRY_ENTRY();
std::ifstream data_file;
data_file.open( file_path, std::ios_base::binary | std::ios_base::in);
boost::filesystem::ifstream data_file;
data_file.open( epee::string_encoding::utf8_to_wstring(file_path), std::ios_base::binary | std::ios_base::in);
if(data_file.fail())
return false;
boost::archive::binary_iarchive a(data_file);
a >> obj;
return !data_file.fail();
CATCH_ENTRY_L0("unserialize_obj_from_file", false);
CATCH_ENTRY_L0("unserialize_obj_from_file: could not load " << file_path, false);
}
template<class t_object>
@ -85,7 +85,7 @@ namespace tools
a >> obj;
return !ss.fail();
CATCH_ENTRY_L0("unserialize_obj_from_obj", false);
CATCH_ENTRY_L0("unserialize_obj_from_buff", false);
}
@ -98,6 +98,6 @@ namespace tools
a >> obj;
return !stream.fail();
CATCH_ENTRY_L0("unserialize_obj_from_file", false);
CATCH_ENTRY_L0("portable_unserialize_obj_from_stream", false);
}
}

184
src/common/callstack_helper.cpp Normal file
View file

@ -0,0 +1,184 @@
// Copyright (c) 2019 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#if defined(WIN32)
#define WIN32_LEAN_AND_MEAN 1
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <Psapi.h>
#pragma comment(lib, "psapi.lib")
#pragma comment(lib, "dbghelp.lib")
#pragma pack( push, before_imagehlp, 8 )
#include <imagehlp.h>
#pragma pack( pop, before_imagehlp )
#include "include_base_utils.h"
#include "callstack_helper.h"
namespace
{
struct module_data
{
std::string image_name;
std::string module_name;
void *base_address;
DWORD load_size;
};
class get_mod_info
{
HANDLE process;
static const int buffer_length = 4096;
public:
get_mod_info(HANDLE h) : process(h) {}
module_data operator()(HMODULE module)
{
module_data ret;
char temp[buffer_length];
MODULEINFO mi;
GetModuleInformation(process, module, &mi, sizeof(mi));
ret.base_address = mi.lpBaseOfDll;
ret.load_size = mi.SizeOfImage;
GetModuleFileNameEx(process, module, temp, sizeof(temp));
ret.image_name = temp;
GetModuleBaseName(process, module, temp, sizeof(temp));
ret.module_name = temp;
std::vector<char> img(ret.image_name.begin(), ret.image_name.end());
std::vector<char> mod(ret.module_name.begin(), ret.module_name.end());
SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address, ret.load_size);
return ret;
}
};
std::string get_symbol_undecorated_name(HANDLE process, DWORD64 address, std::stringstream& ss)
{
SYMBOL_INFO* sym;
static const int max_name_len = 1024;
std::vector<char> sym_buffer(sizeof(SYMBOL_INFO) + max_name_len, '\0');
sym = (SYMBOL_INFO *)sym_buffer.data();
sym->SizeOfStruct = sizeof(SYMBOL_INFO);
sym->MaxNameLen = max_name_len;
DWORD64 displacement;
if (!SymFromAddr(process, address, &displacement, sym))
return std::string("SymFromAddr failed1: ") + epee::string_tools::num_to_string_fast(GetLastError());
if (*sym->Name == '\0')
return std::string("SymFromAddr failed2: ") + epee::string_tools::num_to_string_fast(GetLastError());
/*
ss << " SizeOfStruct : " << sym->SizeOfStruct << ENDL;
ss << " TypeIndex : " << sym->TypeIndex << ENDL; // Type Index of symbol
ss << " Index : " << sym->Index << ENDL;
ss << " Size : " << sym->Size << ENDL;
ss << " ModBase : " << sym->ModBase << ENDL; // Base Address of module comtaining this symbol
ss << " Flags : " << sym->Flags << ENDL;
ss << " Value : " << sym->Value << ENDL; // Value of symbol, ValuePresent should be 1
ss << " Address : " << sym->Address << ENDL; // Address of symbol including base address of module
ss << " Register : " << sym->Register << ENDL; // register holding value or pointer to value
ss << " Scope : " << sym->Scope << ENDL; // scope of the symbol
ss << " Tag : " << sym->Tag << ENDL; // pdb classification
ss << " NameLen : " << sym->NameLen << ENDL; // Actual length of name
ss << " MaxNameLen : " << sym->MaxNameLen << ENDL;
ss << " Name[1] : " << &sym->Name << ENDL; // Name of symbol
*/
std::string und_name(max_name_len, '\0');
DWORD chars_written = UnDecorateSymbolName(sym->Name, &und_name.front(), max_name_len, UNDNAME_COMPLETE);
und_name.resize(chars_written);
return und_name;
}
} // namespace
namespace tools
{
std::string get_callstack_win_x64()
{
// @TODO@
// static epee::static_helpers::wrapper<std::recursive_mutex> cs;
static std::recursive_mutex cs;
std::lock_guard<std::recursive_mutex> lock(cs);
HANDLE h_process = GetCurrentProcess();
HANDLE h_thread = GetCurrentThread();
PCSTR user_search_path = NULL; // may be path to a pdb?
if (!SymInitialize(h_process, user_search_path, false))
return "<win callstack error 1>";
DWORD sym_options = SymGetOptions();
sym_options |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME;
SymSetOptions(sym_options);
DWORD cb_needed;
std::vector<HMODULE> module_handles(1);
EnumProcessModules(h_process, &module_handles[0], static_cast<DWORD>(module_handles.size() * sizeof(HMODULE)), &cb_needed);
module_handles.resize(cb_needed / sizeof(HMODULE));
EnumProcessModules(h_process, &module_handles[0], static_cast<DWORD>(module_handles.size() * sizeof(HMODULE)), &cb_needed);
std::vector<module_data> modules;
std::transform(module_handles.begin(), module_handles.end(), std::back_inserter(modules), get_mod_info(h_process));
void *base = modules[0].base_address;
CONTEXT context;
memset(&context, 0, sizeof context);
RtlCaptureContext( &context );
STACKFRAME64 frame;
memset(&frame, 0, sizeof frame);
frame.AddrPC.Offset = context.Rip;
frame.AddrPC.Mode = AddrModeFlat;
frame.AddrStack.Offset = context.Rsp;
frame.AddrStack.Mode = AddrModeFlat;
frame.AddrFrame.Offset = context.Rbp;
frame.AddrFrame.Mode = AddrModeFlat;
IMAGEHLP_LINE64 line = { 0 };
line.SizeOfStruct = sizeof line;
IMAGE_NT_HEADERS *image_nt_header = ImageNtHeader(base);
std::stringstream ss;
ss << ENDL;
// ss << "main module loaded at 0x" << std::hex << std::setw(16) << std::setfill('0') << base << std::dec << " from " << modules[0].image_name << ENDL;
for (size_t n = 0; n < 250; ++n)
{
if (!StackWalk64(image_nt_header->FileHeader.Machine, h_process, h_thread, &frame, &context, NULL, SymFunctionTableAccess64, SymGetModuleBase64, NULL))
break;
if (frame.AddrReturn.Offset == 0)
break;
std::string fnName = get_symbol_undecorated_name(h_process, frame.AddrPC.Offset, ss);
ss << "0x" << std::setw(16) << std::setfill('0') << std::hex << frame.AddrPC.Offset << " " << std::dec << fnName;
DWORD offset_from_line = 0;
if (SymGetLineFromAddr64(h_process, frame.AddrPC.Offset, &offset_from_line, &line))
ss << "+" << offset_from_line << " " << line.FileName << "(" << line.LineNumber << ")";
for (auto el : modules)
{
if ((DWORD64)el.base_address <= frame.AddrPC.Offset && frame.AddrPC.Offset < (DWORD64)el.base_address + (DWORD64)el.load_size)
{
ss << " : " << el.module_name << " @ 0x" << std::setw(0) << std::hex << (DWORD64)el.base_address << ENDL;
break;
}
}
}
SymCleanup(h_process);
return ss.str();
}
} // namespace tools
#endif // #if defined(WIN32)

24
src/common/callstack_helper.h Normal file
View file

@ -0,0 +1,24 @@
// Copyright (c) 2019 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <epee/include/misc_os_dependent.h>
namespace tools
{
#if defined(WIN32)
extern std::string get_callstack_win_x64();
#endif
inline std::string get_callstack()
{
#if defined(__GNUC__)
return epee::misc_utils::print_trace_default();
#elif defined(WIN32)
return get_callstack_win_x64();
#else
return "";
#endif
}
} // namespace tools

23
src/common/command_line.cpp
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Copyright (c) 2012-2013 The Cryptonote developers
// Distributed under the MIT/X11 software license, see the accompanying
@ -12,14 +12,31 @@ namespace command_line
{
const arg_descriptor<bool> arg_help = {"help", "Produce help message"};
const arg_descriptor<bool> arg_version = {"version", "Output version information"};
const arg_descriptor<std::string> arg_data_dir = {"data-dir", "Specify data directory"};
const arg_descriptor<std::string> arg_data_dir = {"data-dir", "Specify data directory", ""};
const arg_descriptor<int> arg_stop_after_height = { "stop-after-height", "If specified, the daemon will stop immediately after a block with the given height is added", 0 };
const arg_descriptor<std::string> arg_config_file = { "config-file", "Specify configuration file", std::string(CURRENCY_NAME_SHORT ".conf") };
const arg_descriptor<bool> arg_os_version = { "os-version", "" };
const arg_descriptor<std::string> arg_log_dir = { "log-dir", "", "", true};
const arg_descriptor<int> arg_log_level = { "log-level", "", LOG_LEVEL_0, true};
const arg_descriptor<std::string> arg_log_file = { "log-file", "", "" };
const arg_descriptor<int> arg_log_level = { "log-level", "", LOG_LEVEL_0, true };
const arg_descriptor<bool> arg_console = { "no-console", "Disable daemon console commands" };
const arg_descriptor<bool> arg_show_details = { "currency-details", "Display currency details" };
const arg_descriptor<bool> arg_show_rpc_autodoc = { "show_rpc_autodoc", "Display rpc auto-generated documentation template" };
const arg_descriptor<bool> arg_disable_upnp = { "disable-upnp", "Disable UPnP (enhances local network privacy)", false, true };
const arg_descriptor<bool> arg_disable_stop_if_time_out_of_sync = { "disable-stop-if-time-out-of-sync", "Do not stop the daemon if serious time synchronization problem is detected", false, true };
const arg_descriptor<bool> arg_disable_stop_on_low_free_space = { "disable-stop-on-low-free-space", "Do not stop the daemon if free space at data dir is critically low", false, true };
const arg_descriptor<bool> arg_enable_offers_service = { "enable-offers-service", "Enables marketplace feature", false, false};
const arg_descriptor<std::string> arg_db_engine = { "db-engine", "Specify database engine for storage. May be \"lmdb\"(default) or \"mdbx\"", ARG_DB_ENGINE_LMDB, false };
const arg_descriptor<bool> arg_no_predownload = { "no-predownload", "Do not pre-download blockchain database", };
const arg_descriptor<bool> arg_force_predownload = { "force-predownload", "Pre-download blockchain database regardless of it's status", };
const arg_descriptor<bool> arg_validate_predownload = { "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database", };
const arg_descriptor<std::string> arg_predownload_link = { "predownload-link", "Override url for blockchain database pre-downloading", "", true };
}

35
src/common/command_line.h
View file

@ -23,6 +23,25 @@ namespace command_line
struct arg_descriptor<T, false>
{
typedef T value_type;
arg_descriptor(const char* _name, const char* _description):
name(_name),
description(_description),
not_use_default(true),
default_value(T())
{}
arg_descriptor(const char* _name, const char* _description, const T& default_val) :
name(_name),
description(_description),
not_use_default(false),
default_value(default_val)
{}
arg_descriptor(const char* _name, const char* _description, const T& default_val, bool not_use_default) :
name(_name),
description(_description),
default_value(default_val),
not_use_default(not_use_default)
{}
const char* name;
const char* description;
@ -122,7 +141,7 @@ namespace command_line
boost::program_options::basic_parsed_options<charT> parse_command_line(int argc, const charT* const argv[],
const boost::program_options::options_description& desc, bool allow_unregistered = false)
{
auto parser = boost::program_options::command_line_parser(argc, argv);
auto parser = boost::program_options::basic_command_line_parser<charT>(argc, argv);
parser.options(desc);
if (allow_unregistered)
{
@ -172,15 +191,29 @@ namespace command_line
return get_arg<bool, false>(vm, arg);
}
#define ARG_DB_ENGINE_LMDB "lmdb"
#define ARG_DB_ENGINE_MDBX "mdbx"
extern const arg_descriptor<bool> arg_help;
extern const arg_descriptor<bool> arg_version;
extern const arg_descriptor<std::string> arg_data_dir;
extern const arg_descriptor<int> arg_stop_after_height;
extern const arg_descriptor<std::string> arg_config_file;
extern const arg_descriptor<bool> arg_os_version;
extern const arg_descriptor<std::string> arg_log_dir;
extern const arg_descriptor<std::string> arg_log_file;
extern const arg_descriptor<int> arg_log_level;
extern const arg_descriptor<bool> arg_console;
extern const arg_descriptor<bool> arg_show_details;
extern const arg_descriptor<bool> arg_show_rpc_autodoc;
extern const arg_descriptor<bool> arg_disable_upnp;
extern const arg_descriptor<bool> arg_disable_stop_if_time_out_of_sync;
extern const arg_descriptor<bool> arg_disable_stop_on_low_free_space;
extern const arg_descriptor<bool> arg_enable_offers_service;
extern const arg_descriptor<std::string> arg_db_engine;
extern const arg_descriptor<bool> arg_no_predownload;
extern const arg_descriptor<bool> arg_force_predownload;
extern const arg_descriptor<bool> arg_validate_predownload;
extern const arg_descriptor<std::string> arg_predownload_link;
}

66
src/common/config_encrypt_helper.h Normal file
View file

@ -0,0 +1,66 @@
// Copyright (c) 2014-2020 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <string>
#include "wallet/view_iface.h"
namespace tools
{
#pragma pack(push, 1)
struct app_data_file_binary_header
{
uint64_t m_signature;
uint64_t m_cb_body;
};
#pragma pack (pop)
inline
std::string load_encrypted_file(const std::string& path, const std::string& key, std::string& body, uint64_t signature)
{
std::string app_data_buff;
bool r = epee::file_io_utils::load_file_to_string(path, app_data_buff);
if (!r)
{
return API_RETURN_CODE_NOT_FOUND;
}
if (app_data_buff.size() < sizeof(app_data_file_binary_header))
{
LOG_ERROR("app_data_buff.size()(" << app_data_buff.size() << ") < sizeof(app_data_file_binary_header) (" << sizeof(app_data_file_binary_header) << ") check failed while loading from " << path);
return API_RETURN_CODE_INVALID_FILE;
}
crypto::chacha_crypt(app_data_buff, key);
const app_data_file_binary_header* phdr = reinterpret_cast<const app_data_file_binary_header*>(app_data_buff.data());
if (phdr->m_signature != signature)
{
return API_RETURN_CODE_WRONG_PASSWORD;
}
body = app_data_buff.substr(sizeof(app_data_file_binary_header)).c_str();
return API_RETURN_CODE_OK;
}
inline
std::string store_encrypted_file(const std::string& path, const std::string& key, const std::string& body, uint64_t signature)
{
std::string buff(sizeof(app_data_file_binary_header), 0);
app_data_file_binary_header* phdr = (app_data_file_binary_header*)buff.data();
phdr->m_signature = signature;
phdr->m_cb_body = 0; // for future use
buff.append(body);
crypto::chacha_crypt(buff, key);
bool r = epee::file_io_utils::save_string_to_file(path, buff);
if (r)
return API_RETURN_CODE_OK;
else
return API_RETURN_CODE_FAIL;
}
}

18
src/common/crypto_stream_operators.cpp Normal file
View file

@ -0,0 +1,18 @@
// Copyright (c) 2018-2019 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "crypto_stream_operators.h"
bool parse_hash256(const std::string str_hash, crypto::hash& hash)
{
std::string buf;
bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf);
if (!res || buf.size() != sizeof(crypto::hash))
{
std::cout << "invalid hash format: <" << str_hash << '>' << std::endl;
return false;
}
buf.copy(reinterpret_cast<char *>(&hash), sizeof(crypto::hash));
return true;
}

32
src/common/crypto_stream_operators.h
View file

@ -8,29 +8,37 @@
#include "include_base_utils.h"
#include "crypto/crypto.h"
#include "crypto/hash.h"
//------
#include "crypto/RIPEMD160_helper.h"
bool parse_hash256(const std::string str_hash, crypto::hash& hash);
template <class T>
std::ostream &print256(std::ostream &o, const T &v) {
std::ostream &print_t(std::ostream &o, const T &v)
{
return o << "<" << epee::string_tools::pod_to_hex(v) << ">";
}
template <class T>
std::ostream &print16(std::ostream &o, const T &v) {
std::ostream &print16(std::ostream &o, const T &v)
{
return o << "<" << epee::string_tools::pod_to_hex(v).substr(0, 5) << "..>";
}
template <class T>
std::string print16(const T &v) {
std::string print16(const T &v)
{
return std::string("<") + epee::string_tools::pod_to_hex(v).substr(0, 5) + "..>";
}
namespace crypto {
inline std::ostream &operator <<(std::ostream &o, const crypto::public_key &v) { return print256(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::secret_key &v) { return print256(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::key_derivation &v) { return print256(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::key_image &v) { return print256(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::signature &v) { return print256(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::hash &v) { return print256(o, v); }
}
namespace crypto
{
inline std::ostream &operator <<(std::ostream &o, const crypto::public_key &v) { return print_t(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::secret_key &v) { return print_t(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::key_derivation &v) { return print_t(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::key_image &v) { return print_t(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::signature &v) { return print_t(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::hash &v) { return print_t(o, v); }
inline std::ostream &operator <<(std::ostream &o, const crypto::hash160 &v) { return print_t(o, v); }
} // namespace crypto

108
src/common/db_abstract_accessor.h
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Copyright (c) 2012-2013 The Boolberry developers
// Distributed under the MIT/X11 software license, see the accompanying
@ -89,13 +89,16 @@ namespace tools
mutable performance_data m_gperformance_data;
mutable std::unordered_map<container_handle, performance_data> m_performance_data_map;
public:
basic_db_accessor(std::shared_ptr<i_db_backend> backend, epee::shared_recursive_mutex& rwlock) :m_backend(backend), m_rwlock(rwlock), m_is_open(false)
{}
basic_db_accessor(std::shared_ptr<i_db_backend> backend, epee::shared_recursive_mutex& rwlock)
: m_backend(backend), m_rwlock(rwlock), m_is_open(false)
{
}
~basic_db_accessor()
{
close();
}
void reset_backend(std::shared_ptr<i_db_backend> backend) { m_backend = backend; }
performance_data& get_performance_data_for_handle(container_handle h) const { return m_performance_data_map[h]; }
performance_data& get_performance_data_global() const { return m_gperformance_data; }
@ -239,8 +242,11 @@ namespace tools
bool close()
{
m_is_open = false;
if (!m_backend)
return true;
return m_backend->close();
}
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE)
{
bool r = m_backend->open(path, cache_sz);
@ -513,7 +519,9 @@ namespace tools
~basic_key_value_accessor()
{
TRY_ENTRY();
bdb.unbind_parent_container(this);
CATCH_ALL_DO_NOTHING();
}
virtual bool on_write_transaction_begin()
@ -562,11 +570,29 @@ namespace tools
m_set_profiler.m_name = container_name + ":set";
m_explicit_get_profiler.m_name = container_name + ":explicit_get";
m_explicit_set_profiler.m_name = container_name + ":explicit_set";
m_commit_profiler.m_name = container_name + ":commit";;
m_commit_profiler.m_name = container_name + ":commit";
#endif
return bdb.get_backend()->open_container(container_name, m_h);
}
bool deinit()
{
#ifdef ENABLE_PROFILING
m_get_profiler.m_name = "";
m_set_profiler.m_name = "";
m_explicit_get_profiler.m_name = "";
m_explicit_set_profiler.m_name = "";
m_commit_profiler.m_name = "";
#endif
bool r = true;
if (m_h)
r = bdb.get_backend()->close_container(m_h);
m_h = AUTO_VAL_INIT(m_h);
size_cache = 0;
size_cache_valid = false;
return r;
}
template<class t_cb>
void enumerate_keys(t_cb cb) const
{
@ -644,28 +670,27 @@ namespace tools
{
return bdb.size(m_h);
}
size_t clear()
bool clear()
{
bdb.clear(m_h);
bool result = bdb.clear(m_h);
m_isolation.isolated_write_access<bool>([&](){
size_cache_valid = false;
return true;
});
return true;
return result;
}
bool erase_validate(const t_key& k)
{
auto res_ptr = this->get(k);
bdb.erase(m_h, k);
bool result = bdb.erase(m_h, k);
m_isolation.isolated_write_access<bool>([&](){
size_cache_valid = false;
return true;
});
return static_cast<bool>(res_ptr);
return result;
}
void erase(const t_key& k)
{
bdb.erase(m_h, k);
@ -725,7 +750,11 @@ namespace tools
}
~cached_key_value_accessor()
{
m_cache.clear(); //will clear cache isolated
NESTED_TRY_ENTRY();
m_cache.clear(); //will clear cache isolated
NESTED_CATCH_ENTRY(__func__);
}
void clear_cache() const
@ -842,13 +871,10 @@ namespace tools
operator t_value() const
{
static_assert(std::is_pod<t_value>::value, "t_value must be a POD type.");
std::shared_ptr<const t_value> value_ptr = m_accessor.template explicit_get<t_key, t_value, access_strategy_selector<is_t_strategy> >(m_key);
if (value_ptr.get())
return *value_ptr.get();
std::shared_ptr<const t_value> vptr = m_accessor.template explicit_get<t_key, t_value, access_strategy_selector<false> >(m_key);
if (vptr.get())
{
return *vptr.get();
}
return AUTO_VAL_INIT(t_value());
}
};
@ -890,28 +916,48 @@ namespace tools
{
typedef basic_key_value_accessor<composite_key<t_key, uint64_t>, t_value, is_t_access_strategy> basic_accessor_type;
// template<class t_key>
solo_db_value<composite_key<t_key, guid_key>, uint64_t, basic_accessor_type>
get_counter_accessor(const t_key& container_id)
get_counter_accessor(const t_key& container_id)
{
static_assert(std::is_pod<t_key>::value, "t_pod_key must be a POD type.");
composite_key<t_key, guid_key> cc = { container_id, const_counter_suffix};
return solo_db_value<composite_key<t_key, guid_key>, uint64_t, basic_accessor_type >(cc, *this);
}
// template<class t_key>
const solo_db_value<composite_key<t_key, guid_key>, uint64_t, basic_accessor_type >
get_counter_accessor(const t_key& container_id) const
get_counter_accessor(const t_key& container_id) const
{
static_assert(std::is_pod<t_key>::value, "t_pod_key must be a POD type.");
composite_key<t_key, guid_key> cc = { container_id, const_counter_suffix };
static_assert(std::is_pod<t_key>::value, "t_pod_key must be a POD type.");
composite_key<t_key, guid_key> cc = { container_id, const_counter_suffix };
return solo_db_value<composite_key<t_key, guid_key>, uint64_t, basic_accessor_type >(cc, const_cast<basic_accessor_type&>(static_cast<const basic_accessor_type&>(*this)));
}
return solo_db_value<composite_key<t_key, guid_key>, uint64_t, basic_accessor_type >(cc, const_cast<basic_accessor_type&>(static_cast<const basic_accessor_type&>(*this)));
template<typename callback_t>
struct subitems_visitor : public i_db_callback
{
subitems_visitor(callback_t cb)
: m_callback(cb)
{}
virtual bool on_enum_item(uint64_t i, const void* key_data, uint64_t key_size, const void* value_data, uint64_t value_size) override
{
if (key_size != sizeof(composite_key<t_key, uint64_t>))
return true; // skip solo values containing items size
composite_key<t_key, uint64_t> key = AUTO_VAL_INIT(key);
key_from_ptr(key, key_data, key_size);
t_value value = AUTO_VAL_INIT(value);
access_strategy_selector<is_t_access_strategy>::from_buff_to_obj(value_data, value_size, value);
return m_callback(i, key.container_id, key.sufix, value);
}
callback_t m_callback;
};
public:
basic_key_to_array_accessor(basic_db_accessor& db) : basic_key_value_accessor<composite_key<t_key, uint64_t>, t_value, is_t_access_strategy>(db)
{}
@ -968,6 +1014,14 @@ namespace tools
counter = 0;
}
template<typename callback_t>
void enumerate_subitems(callback_t callback) const
{
subitems_visitor<callback_t> visitor(callback);
basic_accessor_type::bdb.get_backend()->enumerate(basic_accessor_type::m_h, &visitor);
}
};
/************************************************************************/

4
src/common/db_backend_base.h
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
@ -38,6 +38,7 @@ namespace tools
virtual void abort_transaction()=0;
virtual bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) = 0;
virtual bool open_container(const std::string& name, container_handle& h)=0;
virtual bool close_container(container_handle& h) = 0;
virtual bool erase(container_handle h, const char* k, size_t s) = 0;
virtual uint64_t size(container_handle h) = 0;
virtual bool get(container_handle h, const char* k, size_t s, std::string& res_buff) = 0;
@ -45,6 +46,7 @@ namespace tools
virtual bool clear(container_handle h) = 0;
virtual bool enumerate(container_handle h, i_db_callback* pcb)=0;
virtual bool get_stat_info(stat_info& si) = 0;
virtual const char* name()=0;
virtual ~i_db_backend(){};
};
}

61
src/common/db_backend_lmdb.cpp
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
@ -7,10 +7,13 @@
#include "misc_language.h"
#include "string_coding.h"
#include "profile_tools.h"
#include "util.h"
#define BUF_SIZE 1024
#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(rc == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
#define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(rc == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
#define ASSERT_MES_AND_THROW_LMDB(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
#undef LOG_DEFAULT_CHANNEL
#define LOG_DEFAULT_CHANNEL "lmdb"
@ -26,7 +29,11 @@ namespace tools
}
lmdb_db_backend::~lmdb_db_backend()
{
close();
NESTED_TRY_ENTRY();
close();
NESTED_CATCH_ENTRY(__func__);
}
bool lmdb_db_backend::open(const std::string& path_, uint64_t cache_sz)
@ -42,9 +49,7 @@ namespace tools
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize");
m_path = path_;
#ifdef WIN32
m_path = epee::string_encoding::convert_ansii_to_utf8(m_path);
#endif
CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path);
res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD , 0644);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_open, m_path=" << m_path);
@ -54,7 +59,6 @@ namespace tools
bool lmdb_db_backend::open_container(const std::string& name, container_handle& h)
{
MDB_dbi dbi = AUTO_VAL_INIT(dbi);
begin_transaction();
int res = mdb_dbi_open(get_current_tx(), name.c_str(), MDB_CREATE, &dbi);
@ -64,6 +68,18 @@ namespace tools
return true;
}
bool lmdb_db_backend::close_container(container_handle& h)
{
static const container_handle null_handle = AUTO_VAL_INIT(null_handle);
CHECK_AND_ASSERT_MES(h != null_handle, false, "close_container is called for null container handle");
MDB_dbi dbi = static_cast<MDB_dbi>(h);
begin_transaction();
mdb_dbi_close(m_penv, dbi);
commit_transaction();
h = null_handle;
return true;
}
bool lmdb_db_backend::close()
{
{
@ -104,8 +120,12 @@ namespace tools
transactions_list& rtxlist = m_txs[std::this_thread::get_id()];
MDB_txn* pparent_tx = nullptr;
MDB_txn* p_new_tx = nullptr;
bool parent_read_only = false;
if (rtxlist.size())
{
pparent_tx = rtxlist.back().ptx;
parent_read_only = rtxlist.back().read_only;
}
if (pparent_tx && read_only)
@ -119,9 +139,20 @@ namespace tools
if (read_only)
flags += MDB_RDONLY;
//don't use parent tx in write transactions if parent tx was read-only (restriction in lmdb)
//see "Nested transactions: Max 1 child, write txns only, no writemap"
if (pparent_tx && parent_read_only)
pparent_tx = nullptr;
CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed");
res = mdb_txn_begin(m_penv, pparent_tx, flags, &p_new_tx);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_txn_begin");
if(res != MDB_SUCCESS)
{
//Important: if mdb_txn_begin is failed need to unlock previously locked mutex
CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock);
//throw exception to avoid regular code execution
ASSERT_MES_AND_THROW_LMDB(res, "Unable to mdb_txn_begin");
}
rtxlist.push_back(tx_entry());
rtxlist.back().count = read_only ? 1 : 0;
@ -295,13 +326,13 @@ namespace tools
PROFILE_FUNC("lmdb_db_backend::set");
int res = 0;
MDB_val key = AUTO_VAL_INIT(key);
MDB_val data = AUTO_VAL_INIT(data);
MDB_val data[2] = {}; // mdb_put may access data[1] if some flags are set, this may trigger static code analizers, so here we allocate two elements to avoid it
key.mv_data = (void*)k;
key.mv_size = ks;
data.mv_data = (void*)v;
data.mv_size = vs;
data[0].mv_data = (void*)v;
data[0].mv_size = vs;
res = mdb_put(get_current_tx(), static_cast<MDB_dbi>(h), &key, &data, 0);
res = mdb_put(get_current_tx(), static_cast<MDB_dbi>(h), &key, data, 0);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put");
return true;
}
@ -359,8 +390,12 @@ namespace tools
}
return true;
}
const char* lmdb_db_backend::name()
{
return "lmdb";
}
}
}
#undef LOG_DEFAULT_CHANNEL
#define LOG_DEFAULT_CHANNEL NULL
#define LOG_DEFAULT_CHANNEL NULL

37
src/common/db_backend_lmdb.h
View file

@ -1,11 +1,11 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <thread>
#include <thread>
#include "include_base_utils.h"
@ -17,9 +17,6 @@ namespace tools
{
namespace db
{
class lmdb_db_backend : public i_db_backend
{
@ -44,23 +41,25 @@ namespace tools
~lmdb_db_backend();
//----------------- i_db_backend -----------------------------------------------------
bool close();
bool begin_transaction(bool read_only = false);
bool commit_transaction();
void abort_transaction();
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE);
bool open_container(const std::string& name, container_handle& h);
bool erase(container_handle h, const char* k, size_t s);
bool get(container_handle h, const char* k, size_t s, std::string& res_buff);
bool clear(container_handle h);
uint64_t size(container_handle h);
bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs);
bool enumerate(container_handle h, i_db_callback* pcb);
bool get_stat_info(tools::db::stat_info& si);
bool close() override;
bool begin_transaction(bool read_only = false) override;
bool commit_transaction() override;
void abort_transaction() override;
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) override;
bool open_container(const std::string& name, container_handle& h) override;
bool close_container(container_handle& h) override;
bool erase(container_handle h, const char* k, size_t s) override;
bool get(container_handle h, const char* k, size_t s, std::string& res_buff) override;
bool clear(container_handle h) override;
uint64_t size(container_handle h) override;
bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs) override;
bool enumerate(container_handle h, i_db_callback* pcb) override;
bool get_stat_info(tools::db::stat_info& si) override;
const char* name() override;
//-------------------------------------------------------------------------------------
bool have_tx();
MDB_txn* get_current_tx();
};
}
}
}

410
src/common/db_backend_mdbx.cpp Normal file
View file

@ -0,0 +1,410 @@
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifdef ENABLED_ENGINE_MDBX
#include "db_backend_mdbx.h"
#include "misc_language.h"
#include "string_coding.h"
#include "profile_tools.h"
#include "util.h"
#define BUF_SIZE 1024
#define CHECK_AND_ASSERT_MESS_MDBX_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDBX_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess);
#define CHECK_AND_ASSERT_THROW_MESS_MDBX_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDBX_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess);
#define ASSERT_MES_AND_THROW_MDBX(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess);
#undef LOG_DEFAULT_CHANNEL
#define LOG_DEFAULT_CHANNEL "mdbx"
// 'mdbx' channel is disabled by default
namespace tools
{
namespace db
{
mdbx_db_backend::mdbx_db_backend() : m_penv(AUTO_VAL_INIT(m_penv))
{
}
mdbx_db_backend::~mdbx_db_backend()
{
NESTED_TRY_ENTRY();
close();
NESTED_CATCH_ENTRY(__func__);
}
bool mdbx_db_backend::open(const std::string& path_, uint64_t cache_sz)
{
int res = 0;
res = mdbx_env_create(&m_penv);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_create");
res = mdbx_env_set_maxdbs(m_penv, 15);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_maxdbs");
intptr_t size_lower = 0;
intptr_t size_now = -1; //don't change current database size
intptr_t size_upper = 0x10000000000; //don't set db file size limit
intptr_t growth_step = 0x40000000; //increment step 1GB
intptr_t shrink_threshold = -1;
intptr_t pagesize = 0x00001000; //4kb
res = mdbx_env_set_geometry(m_penv, size_lower, size_now, size_upper, growth_step, shrink_threshold, pagesize);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_mapsize");
m_path = path_;
CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path);
res = mdbx_env_open(m_penv, m_path.c_str(), MDBX_NORDAHEAD , 0644);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_open, m_path=" << m_path);
return true;
}
bool mdbx_db_backend::open_container(const std::string& name, container_handle& h)
{
MDBX_dbi dbi = AUTO_VAL_INIT(dbi);
begin_transaction();
int res = mdbx_dbi_open(get_current_tx(), name.c_str(), MDBX_CREATE, &dbi);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_dbi_open with container name: " << name);
commit_transaction();
h = static_cast<container_handle>(dbi);
return true;
}
bool mdbx_db_backend::close_container(container_handle& h)
{
static const container_handle null_handle = AUTO_VAL_INIT(null_handle);
CHECK_AND_ASSERT_MES(h != null_handle, false, "close_container is called for null container handle");
MDBX_dbi dbi = static_cast<MDBX_dbi>(h);
begin_transaction();
mdbx_dbi_close(m_penv, dbi);
commit_transaction();
h = null_handle;
return true;
}
bool mdbx_db_backend::close()
{
{
std::lock_guard<boost::recursive_mutex> lock(m_cs);
for (auto& tx_thread : m_txs)
{
for (auto txe : tx_thread.second)
{
int res = mdbx_txn_commit(txe.ptx);
if (res != MDBX_SUCCESS)
{
LOG_ERROR("[DB ERROR]: On close tranactions: " << mdbx_strerror(res));
}
}
}
m_txs.clear();
}
if (m_penv)
{
mdbx_env_close(m_penv);
m_penv = nullptr;
}
return true;
}
bool mdbx_db_backend::begin_transaction(bool read_only)
{
if (!read_only)
{
LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3);
CRITICAL_SECTION_LOCK(m_write_exclusive_lock);
}
PROFILE_FUNC("mdbx_db_backend::begin_transaction");
{
std::lock_guard<boost::recursive_mutex> lock(m_cs);
CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed");
transactions_list& rtxlist = m_txs[std::this_thread::get_id()];
MDBX_txn* pparent_tx = nullptr;
MDBX_txn* p_new_tx = nullptr;
bool parent_read_only = false;
if (rtxlist.size())
{
pparent_tx = rtxlist.back().ptx;
parent_read_only = rtxlist.back().read_only;
}
if (pparent_tx && read_only)
{
++rtxlist.back().count;
}
else
{
int res = 0;
unsigned int flags = 0;
if (read_only)
flags += MDBX_RDONLY;
//don't use parent tx in write transactions if parent tx was read-only (restriction in mdbx)
//see "Nested transactions: Max 1 child, write txns only, no writemap"
if (pparent_tx && parent_read_only)
pparent_tx = nullptr;
CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed");
res = mdbx_txn_begin(m_penv, pparent_tx, flags, &p_new_tx);
if(res != MDBX_SUCCESS)
{
//Important: if mdbx_txn_begin is failed need to unlock previously locked mutex
CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock);
//throw exception to avoid regular code execution
ASSERT_MES_AND_THROW_MDBX(res, "Unable to mdbx_txn_begin");
}
rtxlist.push_back(tx_entry());
rtxlist.back().count = read_only ? 1 : 0;
rtxlist.back().ptx = p_new_tx;
rtxlist.back().read_only = read_only;
}
}
LOG_PRINT_L4("[DB] Transaction started");
return true;
}
MDBX_txn* mdbx_db_backend::get_current_tx()
{
std::lock_guard<boost::recursive_mutex> lock(m_cs);
auto& rtxlist = m_txs[std::this_thread::get_id()];
CHECK_AND_ASSERT_MES(rtxlist.size(), nullptr, "Unable to find active tx for thread " << std::this_thread::get_id());
return rtxlist.back().ptx;
}
bool mdbx_db_backend::pop_tx_entry(tx_entry& txe)
{
std::lock_guard<boost::recursive_mutex> lock(m_cs);
auto it = m_txs.find(std::this_thread::get_id());
CHECK_AND_ASSERT_MES(it != m_txs.end(), false, "[DB] Unable to find id cor current thread");
CHECK_AND_ASSERT_MES(it->second.size(), false, "[DB] No active tx for current thread");
txe = it->second.back();
if (it->second.back().read_only && it->second.back().count == 0)
{
LOG_ERROR("Internal db tx state error: read_only and count readers == 0");
}
if ((it->second.back().read_only && it->second.back().count < 2) || (!it->second.back().read_only && it->second.back().count < 1))
{
it->second.pop_back();
if (!it->second.size())
m_txs.erase(it);
}
else
{
--it->second.back().count;
}
return true;
}
bool mdbx_db_backend::commit_transaction()
{
PROFILE_FUNC("mdbx_db_backend::commit_transaction");
{
tx_entry txe = AUTO_VAL_INIT(txe);
bool r = pop_tx_entry(txe);
CHECK_AND_ASSERT_MES(r, false, "Unable to pop_tx_entry");
if (txe.count == 0 || (txe.read_only && txe.count == 1))
{
int res = 0;
res = mdbx_txn_commit(txe.ptx);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_txn_commit (error " << res << ")");
if (!txe.read_only && !txe.count)
{
CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock);
LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED", LOG_LEVEL_3);
}
}
}
LOG_PRINT_L4("[DB] Transaction committed");
return true;
}
void mdbx_db_backend::abort_transaction()
{
{
tx_entry txe = AUTO_VAL_INIT(txe);
bool r = pop_tx_entry(txe);
CHECK_AND_ASSERT_MES(r, void(), "Unable to pop_tx_entry");
if (txe.count == 0 || (txe.read_only && txe.count == 1))
{
mdbx_txn_abort(txe.ptx);
if (!txe.read_only && !txe.count)
{
CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock);
LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED(ABORTED)", LOG_LEVEL_3);
}
}
}
LOG_PRINT_L4("[DB] Transaction aborted");
}
bool mdbx_db_backend::erase(container_handle h, const char* k, size_t ks)
{
int res = 0;
MDBX_val key = AUTO_VAL_INIT(key);
key.iov_base = (void*)k;
key.iov_len = ks;
res = mdbx_del(get_current_tx(), static_cast<MDBX_dbi>(h), &key, nullptr);
if (res == MDBX_NOTFOUND)
return false;
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_del");
return true;
}
bool mdbx_db_backend::have_tx()
{
std::lock_guard<boost::recursive_mutex> lock(m_cs);
auto it = m_txs.find(std::this_thread::get_id());
if (it == m_txs.end())
return false;
return it->second.size() ? true : false;
}
bool mdbx_db_backend::get(container_handle h, const char* k, size_t ks, std::string& res_buff)
{
PROFILE_FUNC("mdbx_db_backend::get");
int res = 0;
MDBX_val key = AUTO_VAL_INIT(key);
MDBX_val data = AUTO_VAL_INIT(data);
key.iov_base = (void*)k;
key.iov_len = ks;
bool need_to_commit = false;
if (!have_tx())
{
need_to_commit = true;
begin_transaction(true);
}
res = mdbx_get(get_current_tx(), static_cast<MDBX_dbi>(h), &key, &data);
if (need_to_commit)
commit_transaction();
if (res == MDBX_NOTFOUND)
return false;
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_get, h: " << h << ", ks: " << ks);
res_buff.assign((const char*)data.iov_base, data.iov_len);
return true;
}
bool mdbx_db_backend::clear(container_handle h)
{
int res = mdbx_drop(get_current_tx(), static_cast<MDBX_dbi>(h), 0);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_drop");
return true;
}
uint64_t mdbx_db_backend::size(container_handle h)
{
PROFILE_FUNC("mdbx_db_backend::size");
MDBX_stat container_stat = AUTO_VAL_INIT(container_stat);
bool need_to_commit = false;
if (!have_tx())
{
need_to_commit = true;
begin_transaction(true);
}
int res = mdbx_dbi_stat(get_current_tx(), static_cast<MDBX_dbi>(h), &container_stat, sizeof(MDBX_stat));
if (need_to_commit)
commit_transaction();
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_stat");
return container_stat.ms_entries;
}
bool mdbx_db_backend::set(container_handle h, const char* k, size_t ks, const char* v, size_t vs)
{
PROFILE_FUNC("mdbx_db_backend::set");
int res = 0;
MDBX_val key = AUTO_VAL_INIT(key);
MDBX_val data[2] = {}; // mdbx_put may access data[1] if some flags are set, this may trigger static code analizers, so here we allocate two elements to avoid it
key.iov_base = (void*)k;
key.iov_len = ks;
data[0].iov_base = (void*)v;
data[0].iov_len = vs;
res = mdbx_put(get_current_tx(), static_cast<MDBX_dbi>(h), &key, data, 0);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_put");
return true;
}
bool mdbx_db_backend::enumerate(container_handle h, i_db_callback* pcb)
{
CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate");
MDBX_val key = AUTO_VAL_INIT(key);
MDBX_val data = AUTO_VAL_INIT(data);
bool need_to_commit = false;
if (!have_tx())
{
need_to_commit = true;
begin_transaction(true);
}
MDBX_cursor* cursor_ptr = nullptr;
int res = mdbx_cursor_open(get_current_tx(), static_cast<MDBX_dbi>(h), &cursor_ptr);
CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_cursor_open");
CHECK_AND_ASSERT_MES(cursor_ptr, false, "cursor_ptr is null after mdbx_cursor_open");
uint64_t count = 0;
do
{
int res = mdbx_cursor_get(cursor_ptr, &key, &data, MDBX_NEXT);
if (res == MDBX_NOTFOUND)
break;
if (!pcb->on_enum_item(count, key.iov_base, key.iov_len, data.iov_base, data.iov_len))
break;
count++;
} while (cursor_ptr);
mdbx_cursor_close(cursor_ptr);
if (need_to_commit)
commit_transaction();
return true;
}
bool mdbx_db_backend::get_stat_info(tools::db::stat_info& si)
{
si = AUTO_VAL_INIT_T(tools::db::stat_info);
MDBX_envinfo ei = AUTO_VAL_INIT(ei);
mdbx_env_info(m_penv, &ei, sizeof(MDBX_envinfo));
si.map_size = ei.mi_mapsize;
std::lock_guard<boost::recursive_mutex> lock(m_cs);
for (auto& e : m_txs)
{
for (auto& pr : e.second)
{
++si.tx_count;
if(!pr.read_only)
++si.write_tx_count;
}
}
return true;
}
const char* mdbx_db_backend::name()
{
return "mdbx";
}
}
}
#undef LOG_DEFAULT_CHANNEL
#define LOG_DEFAULT_CHANNEL NULL
#endif

67
src/common/db_backend_mdbx.h Normal file
View file

@ -0,0 +1,67 @@
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#ifdef ENABLED_ENGINE_MDBX
#include <thread>
#include "include_base_utils.h"
#include "db_backend_base.h"
#include "db/libmdbx/mdbx.h"
namespace tools
{
namespace db
{
class mdbx_db_backend : public i_db_backend
{
struct tx_entry
{
MDBX_txn* ptx;
bool read_only; // needed for thread-top transaction, for figure out if we need to unlock exclusive access
size_t count; //count of read-only nested emulated transactions
};
typedef std::list<tx_entry> transactions_list;
std::string m_path;
MDBX_env *m_penv;
boost::recursive_mutex m_cs;
boost::recursive_mutex m_write_exclusive_lock;
std::map<std::thread::id, transactions_list> m_txs; // size_t -> count of nested read_only transactions
bool pop_tx_entry(tx_entry& txe);
public:
mdbx_db_backend();
~mdbx_db_backend();
//----------------- i_db_backend -----------------------------------------------------
bool close() override;
bool begin_transaction(bool read_only = false) override;
bool commit_transaction() override;
void abort_transaction() override;
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) override;
bool open_container(const std::string& name, container_handle& h) override;
bool close_container(container_handle& h) override;
bool erase(container_handle h, const char* k, size_t s) override;
bool get(container_handle h, const char* k, size_t s, std::string& res_buff) override;
bool clear(container_handle h) override;
uint64_t size(container_handle h) override;
bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs) override;
bool enumerate(container_handle h, i_db_callback* pcb) override;
bool get_stat_info(tools::db::stat_info& si) override;
const char* name() override;
//-------------------------------------------------------------------------------------
bool have_tx();
MDBX_txn* get_current_tx();
};
}
}
#endif

132
src/common/db_backend_selector.cpp Normal file
View file

@ -0,0 +1,132 @@
// Copyright (c) 2014-2020 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "db_backend_selector.h"
#include "currency_core/currency_config.h"
#include "command_line.h"
#include "db_backend_lmdb.h"
#include "db_backend_mdbx.h"
#define LMDB_MAIN_FILE_NAME "data.mdb"
#define MDBX_MAIN_FILE_NAME "mdbx.dat"
namespace tools
{
namespace db
{
db_backend_selector::db_backend_selector()
: m_engine_type(db_none)
{
}
void db_backend_selector::init_options(boost::program_options::options_description& desc)
{
command_line::add_arg(desc, command_line::arg_db_engine);
}
bool db_backend_selector::init(const boost::program_options::variables_map& vm)
{
try
{
m_config_folder = command_line::get_arg(vm, command_line::arg_data_dir);
if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_LMDB)
{
m_engine_type = db_lmdb;
}
else if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_MDBX)
{
#ifdef ENABLED_ENGINE_MDBX
m_engine_type = db_mdbx;
#else
LOG_PRINT_L0(" DB ENGINE: " << ARG_DB_ENGINE_MDBX << " is not suported by this build(see DISABLE_MDBX cmake option), STOPPING");
#endif
}
else
{
LOG_PRINT_RED_L0("UNKNOWN DB ENGINE: " << command_line::get_arg(vm, command_line::arg_db_engine) << ", STOPPING");
}
}
catch (std::exception& e)
{
LOG_ERROR("internal error: db_backend_selector::init failed on command-line parsing, exception: " << e.what());
return false;
}
if (m_engine_type == db_none)
return false;
return true;
}
std::string db_backend_selector::get_db_folder_path() const
{
//CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited");
return m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX;
}
std::string db_backend_selector::get_temp_db_folder_path() const
{
//CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited");
return get_temp_config_folder() + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX;
}
std::string db_backend_selector::get_pool_db_folder_path() const
{
return m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX;
}
std::string db_backend_selector::get_db_main_file_name() const
{
switch (m_engine_type)
{
case db_lmdb:
return LMDB_MAIN_FILE_NAME;
case db_mdbx:
return MDBX_MAIN_FILE_NAME;
default:
return "";
}
}
std::string db_backend_selector::get_engine_name() const
{
switch (m_engine_type)
{
case db_lmdb:
return "lmdb";
case db_mdbx:
return "mdbx";
default:
return "unknown";
}
}
std::shared_ptr<tools::db::i_db_backend> db_backend_selector::create_backend()
{
switch (m_engine_type)
{
case db_lmdb:
return std::shared_ptr<tools::db::i_db_backend>(new tools::db::lmdb_db_backend);
case db_mdbx:
return std::shared_ptr<tools::db::i_db_backend>(new tools::db::mdbx_db_backend);
default:
LOG_ERROR("db_backend_selector was no inited");
return nullptr;
}
}
std::string db_backend_selector::get_temp_config_folder() const
{
return m_config_folder + "_TEMP";
}
} // namespace db
} // namespace tools

43
src/common/db_backend_selector.h Normal file
View file

@ -0,0 +1,43 @@
// Copyright (c) 2014-2020 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <boost/program_options.hpp>
#include "misc_language.h"
#include "db_backend_base.h"
namespace tools
{
namespace db
{
enum db_engine_type { db_none = 0, db_lmdb, db_mdbx };
class db_backend_selector
{
public:
db_backend_selector();
static void init_options(boost::program_options::options_description& desc);
bool init(const boost::program_options::variables_map& vm);
std::string get_db_folder_path() const;
std::string get_db_main_file_name() const;
db_engine_type get_engine_type() const { return m_engine_type; }
std::string get_engine_name() const;
std::string get_config_folder() const { return m_config_folder; }
std::string get_temp_config_folder() const;
std::string get_temp_db_folder_path() const;
std::string get_pool_db_folder_path() const;
std::shared_ptr<tools::db::i_db_backend> create_backend();
private:
db_engine_type m_engine_type;
std::string m_config_folder;
};
} // namespace db
} // namespace tools

7
src/common/encryption_filter.cpp Normal file
View file

@ -0,0 +1,7 @@
// Copyright (c) 2014-2019 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "encryption_filter.h"
#include "crypto/chacha8_stream.h"

244
src/common/encryption_filter.h Normal file
View file

@ -0,0 +1,244 @@
// Copyright (c) 2014-2018 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <iostream>
#include <iosfwd>
#include <type_traits>
#include <boost/iostreams/categories.hpp> // sink_tag
#include <boost/iostreams/operations.hpp> // boost::iostreams::write, boost::iostreams::read
#include "include_base_utils.h"
#include "crypto/chacha8.h"
#include "crypto/chacha8_stream.h"
namespace tools
{
/************************************************************************/
/* */
/************************************************************************/
class encrypt_chacha_processer_base
{
public:
typedef char char_type;
//typedef boost::iostreams::multichar_output_filter_tag category;
//typedef boost::iostreams::flushable_tag category;
static const uint32_t block_size = ECRYPT_BLOCKLENGTH;
encrypt_chacha_processer_base(std::string const &pass, const crypto::chacha8_iv& iv) :m_iv(iv), m_ctx(AUTO_VAL_INIT(m_ctx))
{
crypto::generate_chacha8_key(pass, m_key);
ECRYPT_keysetup(&m_ctx, &m_key.data[0], sizeof(m_key.data) * 8, sizeof(m_iv.data) * 8);
ECRYPT_ivsetup(&m_ctx, &m_iv.data[0]);
}
~encrypt_chacha_processer_base()
{
}
template<typename cb_handler>
std::streamsize process(char_type const * const buf, std::streamsize const n, cb_handler cb) const
{
if (n == 0)
return n;
if (n%ECRYPT_BLOCKLENGTH == 0 && m_buff.empty())
{
std::vector<char_type> buff(n);
ECRYPT_encrypt_blocks(&m_ctx, (u8*)buf, (u8*)&buff[0], (u32)(n / ECRYPT_BLOCKLENGTH));
cb(&buff[0], n);
//m_underlying_stream.write(&buff[0], n);
return n;
}
else
{
m_buff.append(buf, n);
size_t encr_count = m_buff.size() - m_buff.size() % ECRYPT_BLOCKLENGTH;
if (!encr_count)
return n;
std::vector<char_type> buff(encr_count);
ECRYPT_encrypt_blocks(&m_ctx, (u8*)m_buff.data(), (u8*)&buff[0], (u32)(m_buff.size() / ECRYPT_BLOCKLENGTH));
//m_underlying_stream.write(&buff[0], encr_count);
cb(&buff[0], encr_count);
m_buff.erase(0, encr_count);
return encr_count;
}
}
template<typename cb_handler>
bool flush(cb_handler cb)
{
if (m_buff.empty())
return true;
std::vector<char_type> buff(m_buff.size());
ECRYPT_encrypt_bytes(&m_ctx, (u8*)m_buff.data(), (u8*)&buff[0], (u32)m_buff.size());
cb(&buff[0], m_buff.size());
//m_underlying_stream.write(&buff[0], m_buff.size());
return true;
}
private:
const crypto::chacha8_iv& m_iv;
mutable ECRYPT_ctx m_ctx;
//std::ostream &m_underlying_stream;
crypto::chacha8_key m_key;
mutable std::string m_buff;
};
/************************************************************************/
/* */
/************************************************************************/
class encrypt_chacha_out_filter : public encrypt_chacha_processer_base
{
public:
typedef char char_type;
struct category :
public boost::iostreams::multichar_output_filter_tag,
public boost::iostreams::flushable_tag
{ };
encrypt_chacha_out_filter(std::string const &pass, const crypto::chacha8_iv& iv) :encrypt_chacha_processer_base(pass, iv)
{
}
~encrypt_chacha_out_filter()
{
}
template<typename t_sink>
std::streamsize write(t_sink& snk, char_type const * const buf, std::streamsize const n) const
{
return encrypt_chacha_processer_base::process(buf, n, [&](char_type const * const buf_lambda, std::streamsize const n_lambda) {
boost::iostreams::write(snk, &buf_lambda[0], n_lambda);
});
}
template<typename Sink>
bool flush(Sink& snk)
{
encrypt_chacha_processer_base::flush([&](char_type const * const buf_lambda, std::streamsize const n_lambda) {
boost::iostreams::write(snk, &buf_lambda[0], n_lambda);
});
return true;
}
private:
};
/************************************************************************/
/* */
/************************************************************************/
class encrypt_chacha_in_filter : public encrypt_chacha_processer_base
{
public:
typedef char char_type;
struct category : //public boost::iostreams::seekable_device_tag,
public boost::iostreams::multichar_input_filter_tag,
public boost::iostreams::flushable_tag
//public boost::iostreams::seekable_filter_tag
{ };
encrypt_chacha_in_filter(std::string const &pass, const crypto::chacha8_iv& iv) :encrypt_chacha_processer_base(pass, iv), m_was_eof(false)
{
}
~encrypt_chacha_in_filter()
{
}
template<typename Source>
std::streamsize read(Source& src, char* s, std::streamsize n)
{
if (m_buff.size() >= static_cast<size_t>(n))
{
return withdraw_to_read_buff(s, n);
}
if (m_was_eof && m_buff.empty())
{
return -1;
}
std::streamsize size_to_read_for_decrypt = (n - m_buff.size());
size_to_read_for_decrypt += size_to_read_for_decrypt % encrypt_chacha_processer_base::block_size;
size_t offset_in_buff = m_buff.size();
m_buff.resize(m_buff.size() + size_to_read_for_decrypt);
std::streamsize result = boost::iostreams::read(src, (char*)&m_buff.data()[offset_in_buff], size_to_read_for_decrypt);
if (result == size_to_read_for_decrypt)
{
//regular read proocess, readed data enought to get decrypteds
encrypt_chacha_processer_base::process(&m_buff.data()[offset_in_buff], size_to_read_for_decrypt, [&](char_type const* const buf_lambda, std::streamsize const n_lambda)
{
CHECK_AND_ASSERT_THROW_MES(n_lambda == size_to_read_for_decrypt, "Error in decrypt: check n_lambda == size_to_read_for_decrypt failed");
std::memcpy((char*)&m_buff.data()[offset_in_buff], buf_lambda, n_lambda);
});
return withdraw_to_read_buff(s, n);
}
else
{
//been read some size_but it's basically might be eof
if (!m_was_eof)
{
size_t offset_before_flush = offset_in_buff;
if (result != -1)
{
//eof
encrypt_chacha_processer_base::process(&m_buff.data()[offset_in_buff], result, [&](char_type const* const buf_lambda, std::streamsize const n_lambda) {
std::memcpy((char*)&m_buff.data()[offset_in_buff], buf_lambda, n_lambda);
offset_before_flush = offset_in_buff + n_lambda;
});
}
encrypt_chacha_processer_base::flush([&](char_type const* const buf_lambda, std::streamsize const n_lambda) {
if (n_lambda + offset_before_flush > m_buff.size())
{
m_buff.resize(n_lambda + offset_before_flush);
}
std::memcpy((char*)&m_buff.data()[offset_before_flush], buf_lambda, n_lambda);
m_buff.resize(offset_before_flush + n_lambda);
});
//just to make sure that it's over
std::string buff_stub(10, ' ');
std::streamsize r = boost::iostreams::read(src, (char*)&buff_stub.data()[0], 10);
CHECK_AND_ASSERT_THROW_MES(r == -1, "expected EOF");
m_was_eof = true;
return withdraw_to_read_buff(s, n);
}
else
{
return -1;
}
}
}
template<typename Sink>
bool flush(Sink& snk)
{
return true;
}
private:
std::streamsize withdraw_to_read_buff(char* s, std::streamsize n)
{
size_t copy_size = m_buff.size() > static_cast<size_t>(n) ? static_cast<size_t>(n) : m_buff.size();
std::memcpy(s, m_buff.data(), copy_size);
m_buff.erase(0, copy_size);
return copy_size;
}
std::string m_buff;
bool m_was_eof;
};
}

42
src/common/error_codes.h Normal file
View file

@ -0,0 +1,42 @@
// Copyright (c) 2014-2018 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#define API_RETURN_CODE_OK BASIC_RESPONSE_STATUS_OK
#define API_RETURN_CODE_FAIL BASIC_RESPONSE_STATUS_FAILED
#define API_RETURN_CODE_NOT_FOUND BASIC_RESPONSE_STATUS_NOT_FOUND
#define API_RETURN_CODE_ACCESS_DENIED "ACCESS_DENIED"
#define API_RETURN_CODE_INTERNAL_ERROR "INTERNAL_ERROR"
#define API_RETURN_CODE_NOT_ENOUGH_MONEY "NOT_ENOUGH_MONEY"
#define API_RETURN_CODE_NOT_ENOUGH_OUTPUTS_FOR_MIXING "NOT_ENOUGH_OUTPUTS_FOR_MIXING"
#define API_RETURN_CODE_INTERNAL_ERROR_QUE_FULL "INTERNAL_ERROR_QUE_FULL"
#define API_RETURN_CODE_BAD_ARG "BAD_ARG"
#define API_RETURN_CODE_BAD_ARG_EMPTY_DESTINATIONS "BAD_ARG_EMPTY_DESTINATIONS"
#define API_RETURN_CODE_BAD_ARG_WRONG_FEE "BAD_ARG_WRONG_FEE"
#define API_RETURN_CODE_BAD_ARG_INVALID_ADDRESS "BAD_ARG_INVALID_ADDRESS"
#define API_RETURN_CODE_BAD_ARG_WRONG_AMOUNT "BAD_ARG_WRONG_AMOUNT"
#define API_RETURN_CODE_BAD_ARG_WRONG_PAYMENT_ID "BAD_ARG_WRONG_PAYMENT_ID"
#define API_RETURN_CODE_WRONG_PASSWORD "WRONG_PASSWORD"
#define API_RETURN_CODE_WALLET_WRONG_ID "WALLET_WRONG_ID"
#define API_RETURN_CODE_WALLET_WATCH_ONLY_NOT_SUPPORTED "WALLET_WATCH_ONLY_NOT_SUPPORTED"
#define API_RETURN_CODE_WALLET_AUDITABLE_NOT_SUPPORTED "WALLET_AUDITABLE_NOT_SUPPORTED"
#define API_RETURN_CODE_FILE_NOT_FOUND "FILE_NOT_FOUND"
#define API_RETURN_CODE_ALREADY_EXISTS "ALREADY_EXISTS"
#define API_RETURN_CODE_CANCELED "CANCELED"
#define API_RETURN_CODE_FILE_RESTORED "FILE_RESTORED"
#define API_RETURN_CODE_TRUE "TRUE"
#define API_RETURN_CODE_FALSE "FALSE"
#define API_RETURN_CODE_CORE_BUSY "CORE_BUSY"
#define API_RETURN_CODE_OVERFLOW "OVERFLOW"
#define API_RETURN_CODE_BUSY "BUSY"
#define API_RETURN_CODE_INVALID_FILE "INVALID_FILE"
#define API_RETURN_CODE_WRONG_SEED "WRONG_SEED"
#define API_RETURN_CODE_GENESIS_MISMATCH "GENESIS_MISMATCH"
#define API_RETURN_CODE_DISCONNECTED "DISCONNECTED"
#define API_RETURN_CODE_UNINITIALIZED "UNINITIALIZED"
#define API_RETURN_CODE_TX_IS_TOO_BIG "TX_IS_TOO_BIG"
#define API_RETURN_CODE_TX_REJECTED "TX_REJECTED"
#define API_RETURN_CODE_HTLC_ORIGIN_HASH_MISSMATCHED "HTLC_ORIGIN_HASH_MISSMATCHED"

18
src/common/int-util.h
View file

@ -10,12 +10,22 @@
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <sys/param.h>
#if defined(_MSC_VER)
#include <intrin.h>
#include <stdlib.h>
#if !defined(__ORDER_LITTLE_ENDIAN__)
#define __ORDER_LITTLE_ENDIAN__ 1011012001
#endif
#if !defined(__BYTE_ORDER__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#endif
#if !defined(__ORDER_BIG_ENDIAN__)
#define __ORDER_BIG_ENDIAN__ 0
#endif
static inline uint32_t rol32(uint32_t x, int r) {
static_assert(sizeof(uint32_t) == sizeof(unsigned int), "this code assumes 32-bit integers");
@ -194,11 +204,11 @@ static inline void memcpy_swap64(void *dst, const void *src, size_t n) {
}
}
#if !defined(BYTE_ORDER) || !defined(LITTLE_ENDIAN) || !defined(BIG_ENDIAN)
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__)
static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not enabled");
#endif
#if BYTE_ORDER == LITTLE_ENDIAN
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define SWAP32LE IDENT32
#define SWAP32BE SWAP32
#define swap32le ident32
@ -217,7 +227,7 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e
#define memcpy_swap64be memcpy_swap64
#endif
#if BYTE_ORDER == BIG_ENDIAN
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define SWAP32BE IDENT32
#define SWAP32LE SWAP32
#define swap32be ident32

20
src/common/make_hashable.h
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2020 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Copyright (c) 2012-2013 The Cryptonote developers
// Copyright (c) 2012-2013 The Boolberry developers
@ -37,3 +37,21 @@ namespace std { \
} \
}; \
}
namespace std
{
// this allows using std::pair<> as a key in unordered std containers
template <class T1, class T2>
struct hash<pair<T1, T2>>
{
size_t operator()(const pair<T1, T2>& p) const
{
auto hash1 = hash<T1>{}(p.first);
auto hash2 = hash<T2>{}(p.second);
return hash1 ^ hash2;
}
};
} // namespace std

68
src/common/miniupnp_helper.h
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Copyright (c) 2012-2013 The Boolberry developers
// Distributed under the MIT/X11 software license, see the accompanying
@ -11,9 +11,9 @@
#include <boost/thread.hpp>
#include "include_base_utils.h"
extern "C" {
#include "miniupnpc/miniupnpc.h"
#include "miniupnpc/upnpcommands.h"
#include "miniupnpc/upnperrors.h"
#include "miniupnp/miniupnpc/miniupnpc.h"
#include "miniupnp/miniupnpc/upnpcommands.h"
#include "miniupnp/miniupnpc/upnperrors.h"
}
#include "misc_language.h"
@ -30,26 +30,37 @@ namespace tools
char m_lanaddr[64];
int m_IGD;
boost::thread m_mapper_thread;
boost::thread m_initializer_thread;
uint32_t m_external_port;
uint32_t m_internal_port;
uint32_t m_period_ms;
public:
miniupnp_helper():m_devlist(nullptr),
m_urls(AUTO_VAL_INIT(m_urls)),
m_data(AUTO_VAL_INIT(m_data)),
m_IGD(0)
m_IGD(0),
m_external_port(0),
m_internal_port(0),
m_period_ms(0)
{
m_lanaddr[0] = 0;
}
~miniupnp_helper()
{
NESTED_TRY_ENTRY();
deinit();
NESTED_CATCH_ENTRY(__func__);
}
bool start_regular_mapping(uint32_t internal_port, uint32_t external_port, uint32_t period_ms)
{
m_external_port = external_port;
m_internal_port = internal_port;
m_period_ms = period_ms;
if(!init())
return false;
m_mapper_thread = boost::thread([=](){run_port_mapping_loop(internal_port, external_port, period_ms);});
return true;
}
@ -72,27 +83,42 @@ namespace tools
bool init()
{
deinit();
m_initializer_thread = boost::thread([=]()
{
deinit();
int error = 0;
m_devlist = upnpDiscover(2000, nullptr, nullptr, 0, 0, &error);
if(error)
{
LOG_PRINT_L0("Failed to call upnpDiscover");
return false;
}
m_IGD = UPNP_GetValidIGD(m_devlist, &m_urls, &m_data, m_lanaddr, sizeof(m_lanaddr));
if(m_IGD != 1)
{
LOG_PRINT_L2("IGD not found");
return false;
}
int error = 0;
m_devlist = upnpDiscover(2000, nullptr, nullptr, 0, 0, 2, &error);
if (error)
{
LOG_PRINT_L0("Failed to call upnpDiscover");
return false;
}
m_IGD = UPNP_GetValidIGD(m_devlist, &m_urls, &m_data, m_lanaddr, sizeof(m_lanaddr));
if (m_IGD != 1)
{
LOG_PRINT_L2("IGD not found");
return false;
}
m_mapper_thread = boost::thread([=]() {run_port_mapping_loop(m_internal_port, m_external_port, m_period_ms); });
return true;
});
return true;
}
bool deinit()
{
if(m_initializer_thread.get_id() != boost::this_thread::get_id())
{
if (m_initializer_thread.joinable())
{
m_initializer_thread.interrupt();
m_initializer_thread.join();
}
}
stop_mapping();
if(m_devlist)

13
src/common/mnemonic-encoding.cpp
View file

@ -48,8 +48,6 @@ namespace tools
{
using namespace std;
const int NUMWORDS = 1626;
const map<string,uint32_t> wordsMap = {
{"like", 0},
{"just", 1},
@ -3358,7 +3356,7 @@ namespace tools
throw runtime_error("Invalid binary data size for mnemonic encoding");
// 4 bytes -> 3 words. 8 digits base 16 -> 3 digits base 1626
string res;
for (unsigned int i=0; i < binary.size() / 4; i++, res += ' ')
for (unsigned int i=0; i < binary.size() / 4; i++)
{
const uint32_t* val =
reinterpret_cast<const uint32_t*>(&binary[i * 4]);
@ -3369,8 +3367,9 @@ namespace tools
res += wordsArray[w1] + " ";
res += wordsArray[w2] + " ";
res += wordsArray[w3];
res += wordsArray[w3] + " ";
}
res.erase(--res.end()); // remove trailing space
return res;
}
std::string word_by_num(uint32_t n)
@ -3380,6 +3379,12 @@ namespace tools
return wordsArray[n];
}
bool valid_word(const std::string& w)
{
auto it = wordsMap.find(w);
return it != wordsMap.end();
}
uint64_t num_by_word(const std::string& w)
{
auto it = wordsMap.find(w);

3
src/common/mnemonic-encoding.h
View file

@ -40,9 +40,12 @@ namespace tools
{
namespace mnemonic_encoding
{
constexpr int NUMWORDS = 1626;
std::vector<unsigned char> text2binary(const std::string& text);
std::string binary2text(const std::vector<unsigned char>& binary);
std::string word_by_num(uint32_t n);
uint64_t num_by_word(const std::string& w);
bool valid_word(const std::string& w);
}
}

265
src/common/ntp.cpp Normal file
View file

@ -0,0 +1,265 @@
// Copyright (c) 2019 Zano Project
// Note: class udp_blocking_client is a slightly modified version of an example
// taken from https://www.boost.org/doc/libs/1_53_0/doc/html/boost_asio/example/timeouts/blocking_udp_client.cpp
//
// Copyright (c) 2003-2012 Christopher M. Kohlhoff (chris at kohlhoff dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include <cstdlib>
#include <iostream>
#include <boost/asio/deadline_timer.hpp>
#include <boost/asio/io_service.hpp>
#include <boost/asio/ip/udp.hpp>
#include <boost/bind.hpp>
#include <boost/date_time/posix_time/posix_time_types.hpp>
#include <boost/array.hpp>
#include <epee/include/misc_log_ex.h>
#include <chrono>
#include "ntp.h"
using boost::asio::deadline_timer;
using boost::asio::ip::udp;
//----------------------------------------------------------------------
//
// This class manages socket timeouts by applying the concept of a deadline.
// Each asynchronous operation is given a deadline by which it must complete.
// Deadlines are enforced by an "actor" that persists for the lifetime of the
// client object:
//
// +----------------+
// | |
// | check_deadline |<---+
// | | |
// +----------------+ | async_wait()
// | |
// +---------+
//
// If the actor determines that the deadline has expired, any outstanding
// socket operations are cancelled. The socket operations themselves are
// implemented as transient actors:
//
// +---------------+
// | |
// | receive |
// | |
// +---------------+
// |
// async_- | +----------------+
// receive() | | |
// +--->| handle_receive |
// | |
// +----------------+
//
// The client object runs the io_service to block thread execution until the
// actor completes.
//
namespace
{
class udp_blocking_client
{
public:
udp_blocking_client(const udp::endpoint& listen_endpoint, udp::socket& socket, boost::asio::io_service& io_service)
: socket_(socket),
io_service_(io_service),
deadline_(io_service)
{
// No deadline is required until the first socket operation is started. We
// set the deadline to positive infinity so that the actor takes no action
// until a specific deadline is set.
deadline_.expires_at(boost::posix_time::pos_infin);
// Start the persistent actor that checks for deadline expiry.
check_deadline();
}
std::size_t receive(const boost::asio::mutable_buffer& buffer,
boost::posix_time::time_duration timeout, boost::system::error_code& ec)
{
// Set a deadline for the asynchronous operation.
deadline_.expires_from_now(timeout);
// Set up the variables that receive the result of the asynchronous
// operation. The error code is set to would_block to signal that the
// operation is incomplete. Asio guarantees that its asynchronous
// operations will never fail with would_block, so any other value in
// ec indicates completion.
ec = boost::asio::error::would_block;
std::size_t length = 0;
// Start the asynchronous operation itself. The handle_receive function
// used as a callback will update the ec and length variables.
socket_.async_receive(boost::asio::buffer(buffer),
boost::bind(&udp_blocking_client::handle_receive, _1, _2, &ec, &length));
// Block until the asynchronous operation has completed.
do io_service_.run_one(); while (ec == boost::asio::error::would_block);
return length;
}
private:
void check_deadline()
{
// Check whether the deadline has passed. We compare the deadline against
// the current time since a new asynchronous operation may have moved the
// deadline before this actor had a chance to run.
if (deadline_.expires_at() <= deadline_timer::traits_type::now())
{
// The deadline has passed. The outstanding asynchronous operation needs
// to be cancelled so that the blocked receive() function will return.
//
// Please note that cancel() has portability issues on some versions of
// Microsoft Windows, and it may be necessary to use close() instead.
// Consult the documentation for cancel() for further information.
socket_.cancel();
// There is no longer an active deadline. The expiry is set to positive
// infinity so that the actor takes no action until a new deadline is set.
deadline_.expires_at(boost::posix_time::pos_infin);
}
// Put the actor back to sleep.
deadline_.async_wait(boost::bind(&udp_blocking_client::check_deadline, this));
}
static void handle_receive(
const boost::system::error_code& ec, std::size_t length,
boost::system::error_code* out_ec, std::size_t* out_length)
{
*out_ec = ec;
*out_length = length;
}
private:
boost::asio::io_service& io_service_;
udp::socket& socket_;
deadline_timer deadline_;
};
#pragma pack(push, 1)
struct ntp_packet
{
uint8_t li_vn_mode; // Eight bits. li, vn, and mode.
// li. Two bits. Leap indicator.
// vn. Three bits. Version number of the protocol.
// mode. Three bits. Client will pick mode 3 for client.
uint8_t stratum; // Eight bits. Stratum level of the local clock.
uint8_t poll; // Eight bits. Maximum interval between successive messages.
uint8_t precision; // Eight bits. Precision of the local clock.
uint32_t rootDelay; // 32 bits. Total round trip delay time.
uint32_t rootDispersion; // 32 bits. Max error aloud from primary clock source.
uint32_t refId; // 32 bits. Reference clock identifier.
uint32_t refTm_s; // 32 bits. Reference time-stamp seconds.
uint32_t refTm_f; // 32 bits. Reference time-stamp fraction of a second.
uint64_t orig_tm; // 64 bits. Originate time-stamp (set by client)
uint32_t rxTm_s; // 32 bits. Received time-stamp seconds.
uint32_t rxTm_f; // 32 bits. Received time-stamp fraction of a second.
uint32_t txTm_s; // 32 bits and the most important field the client cares about. Transmit time-stamp seconds.
uint32_t txTm_f; // 32 bits. Transmit time-stamp fraction of a second.
}; // Total: 384 bits or 48 bytes.
#pragma pack(pop)
static_assert(sizeof(ntp_packet) == 48, "ntp_packet has invalid size");
} // namespace
namespace tools
{
int64_t get_ntp_time(const std::string& host_name, size_t timeout_sec)
{
try
{
boost::asio::io_service io_service;
boost::asio::ip::udp::resolver resolver(io_service);
boost::asio::ip::udp::resolver::query query(boost::asio::ip::udp::v4(), host_name, "ntp");
boost::asio::ip::udp::endpoint receiver_endpoint = *resolver.resolve(query);
boost::asio::ip::udp::socket socket(io_service);
socket.open(boost::asio::ip::udp::v4());
ntp_packet packet_sent = AUTO_VAL_INIT(packet_sent);
packet_sent.li_vn_mode = 0x1b;
auto packet_sent_time = std::chrono::high_resolution_clock::now();
socket.send_to(boost::asio::buffer(&packet_sent, sizeof packet_sent), receiver_endpoint);
ntp_packet packet_received = AUTO_VAL_INIT(packet_received);
boost::asio::ip::udp::endpoint sender_endpoint;
udp_blocking_client ubc(sender_endpoint, socket, io_service);
boost::system::error_code ec;
ubc.receive(boost::asio::buffer(&packet_received, sizeof packet_received), boost::posix_time::seconds(static_cast<long>(timeout_sec)), ec);
if (ec)
{
LOG_PRINT_L3("NTP: get_ntp_time(" << host_name << "): boost error: " << ec.message());
return 0;
}
auto packet_received_time = std::chrono::high_resolution_clock::now();
int64_t roundtrip_mcs = std::chrono::duration_cast<std::chrono::microseconds>(packet_received_time - packet_sent_time).count();
uint64_t roundtrip_s = roundtrip_mcs > 2000000 ? roundtrip_mcs / 2000000 : 0;
time_t ntp_time = ntohl(packet_received.txTm_s);
if (ntp_time <= 2208988800U)
{
LOG_PRINT_L3("NTP: get_ntp_time(" << host_name << "): wrong txTm_s: " << packet_received.txTm_s);
return 0;
}
// LOG_PRINT_L2("NTP: get_ntp_time(" << host_name << "): RAW time received: " << ntp_time << ", refTm_s: " << ntohl(packet_received.refTm_s) << ", stratum: " << packet_received.stratum << ", round: " << roundtrip_mcs);
ntp_time -= 2208988800U; // Unix time starts from 01/01/1970 == 2208988800U
ntp_time += roundtrip_s;
return ntp_time;
}
catch (const std::exception& e)
{
LOG_PRINT_L2("NTP: get_ntp_time(" << host_name << "): exception: " << e.what());
return 0;
}
catch (...)
{
LOG_PRINT_L2("NTP: get_ntp_time(" << host_name << "): unknown exception");
return 0;
}
}
#define TIME_SYNC_NTP_SERVERS "time1.google.com", "time2.google.com", "time3.google.com", "time4.google.com" /* , "pool.ntp.org" -- we need to request a vender zone before using this pool */
#define TIME_SYNC_NTP_TIMEOUT_SEC 5
#define TIME_SYNC_NTP_ATTEMPTS_COUNT 3 // max number of attempts when getting time from NTP server
int64_t get_ntp_time()
{
static const std::vector<std::string> ntp_servers { TIME_SYNC_NTP_SERVERS };
for (size_t att = 0; att < TIME_SYNC_NTP_ATTEMPTS_COUNT; ++att)
{
const std::string& ntp_server = ntp_servers[att % ntp_servers.size()];
LOG_PRINT_L3("NTP: trying to get time from " << ntp_server);
int64_t time = get_ntp_time(ntp_server, TIME_SYNC_NTP_TIMEOUT_SEC);
if (time > 0)
{
LOG_PRINT_L3("NTP: " << ntp_server << " responded with " << time << " (" << epee::misc_utils::get_time_str_v2(time) << ")");
return time;
}
}
LOG_PRINT_RED("NTP: unable to get time from NTP with " << TIME_SYNC_NTP_ATTEMPTS_COUNT << " trials", LOG_LEVEL_2);
return 0; // smth went wrong
}
} // namespace tools

18
src/common/ntp.h Normal file
View file

@ -0,0 +1,18 @@
// Copyright (c) 2019 Zano Project
#pragma once
#include <cstdint>
#include <string>
namespace tools
{
// requests current time via NTP from 'host_hame' using 'timeout_sec'
// may return zero -- means error
int64_t get_ntp_time(const std::string& host_name, size_t timeout_sec = 5);
// request time via predefined NTP servers
// may return zero -- mean error
int64_t get_ntp_time();
} // namespace tools

127
src/common/pod_array_file_container.h Normal file
View file

@ -0,0 +1,127 @@
// Copyright (c) 2012-2013 The Cryptonote developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <string>
#include <boost/filesystem/fstream.hpp>
#include <boost/filesystem/operations.hpp>
namespace tools
{
template<typename pod_t>
class pod_array_file_container
{
public:
pod_array_file_container()
{}
~pod_array_file_container()
{
close();
}
bool open(const std::wstring& filename, bool create_if_not_exist, bool* p_corrupted = nullptr, std::string* p_reason = nullptr)
{
if (!create_if_not_exist && !boost::filesystem::exists(filename))
{
if (p_reason)
*p_reason = "file doest not exist";
return false;
}
m_stream.open(filename, std::ios::binary | std::ios::app | std::ios::in);
if (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit)
{
if (p_reason)
*p_reason = "file could not be opened";
return false;
}
if (p_corrupted)
*p_corrupted = false;
size_t file_size = size_bytes();
if (file_size % sizeof(pod_t) != 0)
{
// currupted
if (p_corrupted)
*p_corrupted = true;
size_t corrected_size = file_size - file_size % sizeof(pod_t);
// truncate to nearest item boundary
close();
boost::filesystem::resize_file(filename, corrected_size);
m_stream.open(filename, std::ios::binary | std::ios::app | std::ios::in);
if ((m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit) ||
size_bytes() != corrected_size)
{
if (p_reason)
*p_reason = "truncation failed";
return false;
}
if (p_reason)
*p_reason = std::string("file was corrupted, truncated: ") + epee::string_tools::num_to_string_fast(file_size) + " -> " + epee::string_tools::num_to_string_fast(corrected_size);
}
return true;
}
void close()
{
m_stream.close();
}
bool push_back(const pod_t& item)
{
if (!m_stream.is_open() || (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit))
return false;
m_stream.seekp(0, std::ios_base::end);
m_stream.write(reinterpret_cast<const char*>(&item), sizeof item);
if (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit)
return false;
m_stream.flush();
return true;
}
bool get_item(size_t index, pod_t& result) const
{
if (!m_stream.is_open() || (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit))
return false;
size_t offset = index * sizeof result;
m_stream.seekg(offset);
if (m_stream.rdstate() != std::ios::goodbit)
return false;
m_stream.read(reinterpret_cast<char*>(&result), sizeof result);
return m_stream.gcount() == sizeof result;
}
size_t size_bytes() const
{
if (!m_stream.is_open() || (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit))
return 0;
m_stream.seekg(0, std::ios_base::end);
return m_stream.tellg();
}
size_t size() const
{
return size_bytes() / sizeof(pod_t);
}
private:
mutable boost::filesystem::fstream m_stream;
};
} // namespace tools

226
src/common/pre_download.h Normal file
View file

@ -0,0 +1,226 @@
// Copyright (c) 2020 Zano Project
// Copyright (c) 2012-2018 The Boolberry developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <boost/program_options.hpp>
#include "net/http_client.h"
#include "db_backend_selector.h"
#include "crypto/crypto.h"
#include "currency_core/currency_core.h"
namespace tools
{
struct pre_download_entry
{
const char* url;
const char* hash;
uint64_t packed_size;
uint64_t unpacked_size;
};
#ifndef TESTNET
static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_mdbx_95_1000000.pak", "6b0bbba85bc420eaae5ec68373e528f70bffaa17fb111c796e951d06ad71e4fe", 1104150892, 2147450880 };
static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_lmdb_95_1000000.pak", "b4d45c727dbf1b92671f9fd1a9624e79019e890bd3d33cb71e011ab4bcb0d21e", 1450748151, 2114449408 };
#else
static constexpr pre_download_entry c_pre_download_mdbx = { "", "", 0, 0 };
static constexpr pre_download_entry c_pre_download_lmdb = { "", "", 0, 0 };
#endif
static constexpr uint64_t pre_download_min_size_difference = 512 * 1024 * 1024; // minimum difference in size between local DB and the downloadable one to start downloading
template<class callback_t>
bool process_predownload(const boost::program_options::variables_map& vm, callback_t cb_should_stop)
{
tools::db::db_backend_selector dbbs;
bool r = dbbs.init(vm);
CHECK_AND_ASSERT_MES(r, false, "db_backend_selector failed to initialize");
std::string config_folder = dbbs.get_config_folder();
std::string working_folder = dbbs.get_db_folder_path();
std::string db_main_file_path = working_folder + "/" + dbbs.get_db_main_file_name();
pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? c_pre_download_lmdb : c_pre_download_mdbx;
// override pre-download link if necessary
std::string url = pre_download.url;
if (command_line::has_arg(vm, command_line::arg_predownload_link))
url = command_line::get_arg(vm, command_line::arg_predownload_link);
boost::system::error_code ec;
uint64_t sz = boost::filesystem::file_size(db_main_file_path, ec);
if (pre_download.unpacked_size == 0 || !(ec || (pre_download.unpacked_size > sz && pre_download.unpacked_size - sz > pre_download_min_size_difference) || command_line::has_arg(vm, command_line::arg_force_predownload)) )
{
LOG_PRINT_MAGENTA("Pre-downloading not needed (db file size = " << sz << ")", LOG_LEVEL_0);
return true;
}
// okay, let's download
std::string downloading_file_path = db_main_file_path + ".download";
LOG_PRINT_MAGENTA("Trying to download blockchain database from " << url << " ...", LOG_LEVEL_0);
epee::net_utils::http::interruptible_http_client cl;
crypto::stream_cn_hash hash_stream;
auto last_update = std::chrono::system_clock::now();
auto cb = [&hash_stream, &last_update, &cb_should_stop](const std::string& buff, uint64_t total_bytes, uint64_t received_bytes)
{
if (cb_should_stop(total_bytes, received_bytes))
{
LOG_PRINT_MAGENTA(ENDL << "Interrupting download", LOG_LEVEL_0);
return false;
}
hash_stream.update(buff.data(), buff.size());
auto dif = std::chrono::system_clock::now() - last_update;
if (dif >= std::chrono::milliseconds(300))
{
boost::io::ios_flags_saver ifs(std::cout);
std::cout << "Received " << received_bytes / 1048576 << " of " << total_bytes / 1048576 << " MiB ( " << std::fixed << std::setprecision(1) << 100.0 * received_bytes / total_bytes << " %)\r";
last_update = std::chrono::system_clock::now();
}
return true;
};
tools::create_directories_if_necessary(working_folder);
r = cl.download_and_unzip(cb, downloading_file_path, url, 5000 /* timout */, "GET", std::string(), 30 /* fails count */);
if (!r)
{
LOG_PRINT_RED("Download failed", LOG_LEVEL_0);
return false;
}
crypto::hash data_hash = hash_stream.calculate_hash();
if (epee::string_tools::pod_to_hex(data_hash) != pre_download.hash)
{
LOG_ERROR("hash missmatch in downloaded file, got: " << epee::string_tools::pod_to_hex(data_hash) << ", expected: " << pre_download.hash);
return false;
}
LOG_PRINT_GREEN("Download succeeded, hash " << pre_download.hash << " is correct" , LOG_LEVEL_0);
if (!command_line::has_arg(vm, command_line::arg_validate_predownload))
{
boost::filesystem::remove(db_main_file_path, ec);
if (ec)
{
LOG_ERROR("Failed to remove " << db_main_file_path);
return false;
}
LOG_PRINT_L1("Removed " << db_main_file_path);
boost::filesystem::rename(downloading_file_path, db_main_file_path, ec);
if (ec)
{
LOG_ERROR("Failed to rename " << downloading_file_path << " -> " << db_main_file_path);
return false;
}
LOG_PRINT_L1("Renamed " << downloading_file_path << " -> " << db_main_file_path);
LOG_PRINT_GREEN("Blockchain successfully replaced with the new pre-downloaded data file", LOG_LEVEL_0);
return true;
}
//
// paranoid mode
// move downloaded blockchain into a temporary folder
//
LOG_PRINT_MAGENTA(ENDL << "Downloaded blockchain database is about to be validated and added to the local database block-by-block" << ENDL, LOG_LEVEL_0);
std::string path_to_temp_datafolder = dbbs.get_temp_config_folder();
std::string path_to_temp_blockchain = dbbs.get_temp_db_folder_path();
std::string path_to_temp_blockchain_file = path_to_temp_blockchain + "/" + dbbs.get_db_main_file_name();
tools::create_directories_if_necessary(path_to_temp_blockchain);
boost::filesystem::rename(downloading_file_path, path_to_temp_blockchain_file, ec);
if (ec)
{
LOG_ERROR("Rename failed: " << downloading_file_path << " -> " << path_to_temp_blockchain_file);
return false;
}
// remove old blockchain database from disk
boost::filesystem::remove_all(working_folder, ec);
if (ec)
{
LOG_ERROR("Failed to remove all from " << working_folder);
return false;
}
std::string pool_db_path = dbbs.get_pool_db_folder_path();
boost::filesystem::remove_all(pool_db_path, ec);
if (ec)
{
LOG_ERROR("Failed to remove all from " << pool_db_path);
return false;
}
// source core
currency::core source_core(nullptr);
boost::program_options::variables_map source_core_vm;
source_core_vm.insert(std::make_pair("data-dir", boost::program_options::variable_value(path_to_temp_datafolder, false)));
source_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false)));
//source_core_vm.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false)));
r = source_core.init(source_core_vm);
CHECK_AND_ASSERT_MES(r, false, "Failed to init source core");
// target core
currency::core target_core(nullptr);
boost::program_options::variables_map target_core_vm(vm);
target_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false)));
//vm_with_fast_sync.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false)));
r = target_core.init(target_core_vm);
CHECK_AND_ASSERT_MES(r, false, "Failed to init target core");
CHECK_AND_ASSERT_MES(target_core.get_top_block_height() == 0, false, "Target blockchain initialized not empty");
uint64_t total_blocks = source_core.get_current_blockchain_size();
LOG_PRINT_GREEN("Manually processing blocks from 1 to " << total_blocks << "...", LOG_LEVEL_0);
for (uint64_t i = 1; i != total_blocks; i++)
{
std::list<currency::block> blocks;
std::list<currency::transaction> txs;
bool r = source_core.get_blocks(i, 1, blocks, txs);
CHECK_AND_ASSERT_MES(r && blocks.size()==1, false, "Failed to get block " << i << " from core");
currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash);
for (auto& tx : txs)
{
r = target_core.handle_incoming_tx(tx, tvc, true /* kept_by_block */);
CHECK_AND_ASSERT_MES(r && tvc.m_added_to_pool == true, false, "Failed to add a tx from block " << i << " from core");
}
currency::block_verification_context bvc = AUTO_VAL_INIT(bvc);
r = target_core.handle_incoming_block(*blocks.begin(), bvc);
CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain == true, false, "Failed to add block " << i << " to core");
if (!(i % 100))
std::cout << "Block " << i << "(" << (i * 100) / total_blocks << "%) \r";
if (cb_should_stop(total_blocks, i))
{
LOG_PRINT_MAGENTA(ENDL << "Interrupting updating db...", LOG_LEVEL_0);
return false;
}
}
LOG_PRINT_GREEN("Processing finished, " << total_blocks << " successfully added.", LOG_LEVEL_0);
target_core.deinit();
source_core.deinit();
boost::filesystem::remove_all(path_to_temp_datafolder, ec);
if (ec)
{
LOG_ERROR("Failed to remove all from " << path_to_temp_datafolder);
}
return true;
}
}

187
src/common/util.cpp
View file

@ -10,6 +10,7 @@ using namespace epee;
#include "util.h"
#include "currency_core/currency_config.h"
#include "version.h"
#ifdef WIN32
#include <windows.h>
@ -21,6 +22,9 @@ using namespace epee;
#include <sys/utsname.h>
#endif
#include <boost/asio.hpp>
#include "string_coding.h"
namespace tools
{
@ -361,7 +365,69 @@ namespace tools
return pszOS;
}
}
#else
void signal_handler::GenerateCrashDump(EXCEPTION_POINTERS *pep /* = NULL*/)
{
SYSTEMTIME sysTime = { 0 };
GetSystemTime(&sysTime);
// get the computer name
char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 };
DWORD compNameLen = ARRAYSIZE(compName);
GetComputerNameA(compName, &compNameLen);
// build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP
char path[MAX_PATH*10] = { 0 };
std::string folder = epee::log_space::log_singletone::get_default_log_folder();
sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_" PROJECT_VERSION_LONG "_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp",
folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay,
sysTime.wHour, sysTime.wMinute, sysTime.wSecond);
HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE,
0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE))
{
// Create the minidump
MINIDUMP_EXCEPTION_INFORMATION mdei;
mdei.ThreadId = GetCurrentThreadId();
mdei.ExceptionPointers = pep;
mdei.ClientPointers = FALSE;
MINIDUMP_CALLBACK_INFORMATION mci;
mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback;
mci.CallbackParam = 0;
MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory |
MiniDumpWithDataSegs |
MiniDumpWithHandleData |
MiniDumpWithFullMemoryInfo |
MiniDumpWithThreadInfo |
MiniDumpWithUnloadedModules);
BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(),
hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci);
if (!rv)
{
LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path);
}
else
{
LOG_PRINT_L0("Minidump file created on path: " << path);
}
// Close the file
CloseHandle(hFile);
}
else
{
LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path);
}
}
#else // ifdef WIN32
std::string get_nix_version_display_string()
{
utsname un;
@ -386,18 +452,22 @@ std::string get_nix_version_display_string()
#ifdef WIN32
std::string get_special_folder_path(int nfolder, bool iscreate)
std::wstring get_special_folder_path_w(int nfolder, bool iscreate)
{
namespace fs = boost::filesystem;
char psz_path[MAX_PATH] = "";
wchar_t psz_path[MAX_PATH] = L"";
if(SHGetSpecialFolderPathA(NULL, psz_path, nfolder, iscreate))
if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate))
{
return psz_path;
}
LOG_ERROR("SHGetSpecialFolderPathA() failed, could not obtain requested path.");
return "";
LOG_ERROR("SHGetSpecialFolderPathW(" << nfolder << ", " << iscreate << ") failed, could not obtain requested path.");
return L"";
}
std::string get_special_folder_path_utf8(int nfolder, bool iscreate)
{
return epee::string_encoding::wstring_to_utf8(get_special_folder_path_w(nfolder, iscreate));
}
#endif
@ -412,9 +482,9 @@ std::string get_nix_version_display_string()
#ifdef WIN32
// Windows
#ifdef _M_X64
config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT;
config_folder = get_special_folder_path_utf8(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT;
#else
config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT + "-x86";
config_folder = get_special_folder_path_utf8(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT + "-x86";
#endif
#else
std::string pathRet;
@ -454,7 +524,7 @@ std::string get_nix_version_display_string()
std::string wallets_dir;
#ifdef WIN32
// Windows
wallets_dir = get_special_folder_path(CSIDL_PERSONAL, true) + "/" + CURRENCY_NAME_BASE;
wallets_dir = get_special_folder_path_utf8(CSIDL_PERSONAL, true) + "/" + CURRENCY_NAME_BASE;
#else
std::string pathRet;
char* pszHome = getenv("HOME");
@ -489,7 +559,7 @@ std::string get_nix_version_display_string()
{
namespace fs = boost::filesystem;
boost::system::error_code ec;
fs::path fs_path(path);
fs::path fs_path = epee::string_encoding::utf8_to_wstring(path);
if (fs::is_directory(fs_path, ec))
{
return true;
@ -588,4 +658,97 @@ std::string get_nix_version_display_string()
return static_cast<uint64_t>(in.tellg());
}
}
bool check_remote_client_version(const std::string& client_ver)
{
std::string v = client_ver.substr(0, client_ver.find('[')); // remove commit id
v = v.substr(0, v.rfind('.')); // remove build number
int v_major = 0, v_minor = 0, v_revision = 0;
size_t dot_pos = v.find('.');
if (dot_pos == std::string::npos || !epee::string_tools::string_to_num_fast(v.substr(0, dot_pos), v_major))
return false;
v = v.substr(dot_pos + 1);
dot_pos = v.find('.');
if (!epee::string_tools::string_to_num_fast(v.substr(0, dot_pos), v_minor))
return false;
if (dot_pos != std::string::npos)
{
// revision
v = v.substr(dot_pos + 1);
if (!epee::string_tools::string_to_num_fast(v, v_revision))
return false;
}
// got v_major, v_minor, v_revision
// allow 1.1.x and greater
if (v_major < 1)
return false;
if (v_major == 1 && v_minor < 1)
return false;
return true;
}
//this code was taken from https://stackoverflow.com/a/8594696/5566653
//credits goes to @nijansen: https://stackoverflow.com/users/1056003/nijansen
bool copy_dir( boost::filesystem::path const & source, boost::filesystem::path const & destination)
{
namespace fs = boost::filesystem;
try
{
// Check whether the function call is valid
if (!fs::exists(source) ||!fs::is_directory(source))
{
LOG_ERROR("Source directory " << source.string() << " does not exist or is not a directory.");
return false;
}
if (!fs::exists(destination))
{
if (!fs::create_directory(destination))
{
LOG_ERROR("Unable to create destination directory" << destination.string());
return false;
}
}
// Create the destination directory
}
catch (fs::filesystem_error const & e)
{
LOG_ERROR("Exception: " << e.what());
return false;
}
// Iterate through the source directory
for (fs::directory_iterator file(source); file != fs::directory_iterator(); ++file)
{
try
{
fs::path current(file->path());
if (fs::is_directory(current))
{
// Found directory: Recursion
if (!copy_dir(current, destination / current.filename()))
{
return false;
}
}
else
{
// Found file: Copy
fs::copy_file( current, destination / current.filename());
}
}
catch (fs::filesystem_error const & e)
{
LOG_ERROR("Exception: " << e.what());
}
}
return true;
}
} // namespace tools

77
src/common/util.h
View file

@ -17,11 +17,18 @@
#include "crypto/hash.h"
#include "misc_language.h"
#include "p2p/p2p_protocol_defs.h"
#include "ntp.h"
#if defined(WIN32)
#include <dbghelp.h>
#endif
#ifdef NDEBUG
#define BUILD_TYPE "Release"
#else
#define BUILD_TYPE "Debug"
#endif
namespace tools
{
std::string get_host_computer_name();
@ -29,6 +36,9 @@ namespace tools
std::string get_default_user_dir();
std::string get_current_username();
std::string get_os_version_string();
bool copy_dir(boost::filesystem::path const & source, boost::filesystem::path const & destination);
bool check_remote_client_version(const std::string& client_ver);
bool create_directories_if_necessary(const std::string& path);
std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name);
@ -231,67 +241,7 @@ namespace tools
}
static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL)
{
SYSTEMTIME sysTime = { 0 };
GetSystemTime(&sysTime);
// get the computer name
char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 };
DWORD compNameLen = ARRAYSIZE(compName);
GetComputerNameA(compName, &compNameLen);
// build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP
char path[MAX_PATH*10] = { 0 };
std::string folder = epee::log_space::log_singletone::get_default_log_folder();
sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp",
folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay,
sysTime.wHour, sysTime.wMinute, sysTime.wSecond);
HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE,
0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE))
{
// Create the minidump
MINIDUMP_EXCEPTION_INFORMATION mdei;
mdei.ThreadId = GetCurrentThreadId();
mdei.ExceptionPointers = pep;
mdei.ClientPointers = FALSE;
MINIDUMP_CALLBACK_INFORMATION mci;
mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback;
mci.CallbackParam = 0;
MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory |
MiniDumpWithDataSegs |
MiniDumpWithHandleData |
MiniDumpWithFullMemoryInfo |
MiniDumpWithThreadInfo |
MiniDumpWithUnloadedModules);
BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(),
hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci);
if (!rv)
{
LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path);
}
else
{
LOG_PRINT_L0("Minidump file created on path: " << path);
}
// Close the file
CloseHandle(hFile);
}
else
{
LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path);
}
}
static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL);
static LONG WINAPI win_unhandled_exception_handler(_In_ struct _EXCEPTION_POINTERS *ep)
{
@ -316,14 +266,14 @@ namespace tools
static void handle_signal()
{
static std::mutex m_mutex;
static epee::static_helpers::wrapper<std::mutex> m_mutex;
std::unique_lock<std::mutex> lock(m_mutex);
m_handler();
}
static void handle_fatal_signal(int sig_number, void* address)
{
static std::mutex m_mutex_fatal;
static epee::static_helpers::wrapper<std::mutex> m_mutex_fatal;
std::unique_lock<std::mutex> lock(m_mutex_fatal);
m_fatal_handler(sig_number, address);
uninstall_fatal();
@ -333,4 +283,5 @@ namespace tools
static std::function<void(void)> m_handler;
static std::function<void(int, void*)> m_fatal_handler;
};
}

25
src/contrib/db/CMakeLists.txt
View file

@ -1,7 +1,20 @@
add_subdirectory(liblmdb)
if(MSVC)
target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172)
else()
# Warnings as used by LMDB itself (LMDB_0.9.23)
target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized)
if(CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android")
message("excluded db support for IOS build")
return()
endif()
message("DB ENGINE: lmdb")
add_subdirectory(liblmdb)
if(MSVC)
target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172)
else()
# Warnings as used by LMDB itself (LMDB_0.9.23)
target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized)
endif()
if(NOT DISABLE_MDBX)
message("DB ENGINE: mdbx")
add_subdirectory(libmdbx)
endif()

2
src/contrib/db/liblmdb/CMakeLists.txt
View file

@ -4,7 +4,7 @@ set (lmdb_sources mdb.c midl.c)
include_directories("${CMAKE_CURRENT_SOURCE_DIR}")
if(NOT MSVC)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-missing-braces -Wno-aggregate-return")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-missing-braces -Wno-aggregate-return -Wno-discarded-qualifiers -Wno-unused-but-set-variable -Wno-implicit-fallthrough -Wno-maybe-uninitialized ")
endif()
if(FREEBSD)
add_definitions(-DMDB_DSYNC=O_SYNC)

4
src/contrib/db/liblmdb/mdb.c
View file

@ -132,8 +132,10 @@ extern int cacheflush(char *addr, int nbytes, int cache);
#ifdef _WIN32
typedef int64_t off64_t;
#else
#elif __APPLE__
typedef off_t off64_t;
#else
#endif

20
src/contrib/db/libmdbx/.circleci/config.yml Normal file
View file

@ -0,0 +1,20 @@
version: 2
jobs:
build:
docker:
- image: circleci/buildpack-deps:artful
environment:
- TESTDB: /tmp/test.db
- TESTLOG: /tmp/test.log
steps:
- checkout
- run: make all
- run: ulimit -c unlimited && make check
- run:
command: |
mkdir -p /tmp/artifacts
mv -t /tmp/artifacts $TESTLOG $TESTDB core.*
when: on_fail
- store_artifacts:
path: /tmp/artifacts
destination: test-artifacts

3
src/contrib/db/libmdbx/.clang-format Normal file
View file

@ -0,0 +1,3 @@
BasedOnStyle: LLVM
Standard: Cpp11
ReflowComments: true

35
src/contrib/db/libmdbx/.gitignore vendored Normal file
View file

@ -0,0 +1,35 @@
*.[ao]
*.bak
*.exe
*.gcda
*.gcno
*.gcov
*.lo
*.orig
*.rej
*.so
*[~#]
.idea
.le.ini
.vs/
cmake-build-*
@*
core
mdbx_example
libmdbx.creator.user
mdbx_chk
mdbx_copy
mdbx_dump
mdbx_load
mdbx_stat
mdbx_test
test.log
test/tmp.db
test/tmp.db-lck
tmp.db
tmp.db-lck
valgrind.*
src/elements/version.c
src/elements/config.h
dist/
*.tar*

61
src/contrib/db/libmdbx/.travis.yml Normal file
View file

@ -0,0 +1,61 @@
language: c cpp
matrix:
include:
- os: linux
dist: precise
env: CC=cc CXX=c++
- os: linux
dist: trusty
compiler: clang
env: CC=clang CXX=clang++
- os: linux
dist: xenial
compiler: gcc
env: CC=gcc CXX=g++
- os: linux
dist: bionic
compiler: clang
env: CC=clang CXX=clang++
- os: osx
osx_image: xcode11
env: CC=cc CXX=c++
- os: osx
osx_image: xcode9.4
env: CC=cc CXX=c++
script: >
if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then
git fetch --unshallow --tags --prune &&
git submodule foreach --recursive git fetch --unshallow --tags --prune &&
(if which clang-format-6.0 > /dev/null && make reformat && [[ -n $(git diff) ]];
then
echo "You must run 'make reformat' before submitting a pull request";
echo "";
git diff;
exit -1;
fi) &&
make --keep-going all && MALLOC_CHECK_=7 MALLOC_PERTURB_=42 make --keep-going check
else
[ ! -s cov-int/scm_log.txt ] || cat cov-int/scm_log.txt;
fi && sleep 3
env:
global:
- secure: "M+W+heGGyRQJoBq2W0uqWVrpL4KBXmL0MFL7FSs7f9vmAaDyEgziUXeZRj3GOKzW4kTef3LpIeiu9SmvqSMoQivGGiomZShqPVl045o/OUgRCAT7Al1RLzEZ0efSHpIPf0PZ6byEf6GR2ML76OfuL6JxTVdnz8iVyO2sgLE1HbX1VeB+wgd/jfMeOBhCCXskfK6MLyZihfMYsiYZYSaV98ZDhDLSlzuuRIgzb0bMi8aL6AErs0WLW0NelRBeHkKPYfAUc85pdQHscgrJw6Rh/zT6+8BQ/q5f4IgWhiu4xoRg3Ngl7SNoedRQh93ADM3UG2iGl6HDFpVORaXcFWKAtuYY+kHQ0HB84BRYpQmeBuXNpltsfxQ3d1Q3u0RlE45zRvmr2+X1mFnkcNUAWISLPbsOUlriDQM8irGwRpho77/uYnRC00bJsHW//s6+uPf9zrAw1nI4f0y3PAWukGF/xs6HAI3FZPsuSSnx18Tj3Opgbc9Spop+V3hkhdiJoPGpNKTkFX4ZRXfkPgoRVJmtp4PpbpH0Ps/mCriKjMEfGGi0HcVCi0pEGLXiecdqJ5KPg5+22zNycEujQBJcNTKd9shN+R3glrbmhAxTEzGdGwxXXJ2ybwJ2PWJLMYZ7g98nLyX+uQPaA3BlsbYJHNeS5283/9pJsd9DzfHKsN2nFSc="
before_install:
- echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-
- ${CC} --version
- ${CXX} --version
addons:
coverity_scan:
project:
name: "ReOpen/libmdbx"
version: 0.1
description: "Build submitted via Travis CI"
notification_email: leo@yuriev.ru
build_command_prepend: "git fetch --unshallow --tags --prune && make dist"
build_command: "make MDBX_OPTIONS=-DMDBX_DEBUG=2 -C dist all"
branch_pattern: coverity_scan

32
src/contrib/db/libmdbx/AUTHORS Normal file
View file

@ -0,0 +1,32 @@
Contributors
============
Alexey Naumov <alexey.naumov@gmail.com>
Chris Mikkelson <cmikk@qwest.net>
Claude Brisson <claude.brisson@gmail.com>
David Barbour <dmbarbour@gmail.com>
David Wilson <dw@botanicus.net>
dreamsxin <dreamsxin@126.com>
Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no>
Heiko Becker <heirecka@exherbo.org>
Howard Chu <hyc@openldap.org>, <hyc@symas.com>
Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
James Rouzier <rouzier@gmail.com>
Jean-Christophe DUBOIS <jcd@tribudubois.net>
John Hewson <john@jahewson.com>
Klaus Malorny <klaus.malorny@knipp.de>
Kurt Zeilenga <kurt.zeilenga@isode.com>
Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com>
Lorenz Bauer <lmb@cloudflare.com>
Luke Yeager <lyeager@nvidia.com>
Martin Hedenfalk <martin@bzero.se>
Ondrej Kuznik <ondrej.kuznik@acision.com>
Orivej Desh <orivej@gmx.fr>
Oskari Timperi <oskari.timperi@iki.fi>
Pavel Medvedev <pmedvedev@gmail.com>
Philipp Storz <philipp.storz@bareos.com>
Quanah Gibson-Mount <quanah@openldap.org>
Salvador Ortiz <sog@msg.com.mx>
Sebastien Launay <sebastien@slaunay.fr>
Vladimir Romanov <vromanov@gmail.com>
Zano Foundation <crypto.sowle@gmail.com>

192
src/contrib/db/libmdbx/CMakeLists.dist-minimal Normal file
View file

@ -0,0 +1,192 @@
##
## This is the minimal template for CMakeList.txt which could be used
## to build libmdbx from the "amalgamated form" of libmdbx's source code.
##
## The amalgamated form is intended to embedding libmdbx in other projects
## in cases when using as git-submodule is not acceptable or inconveniently.
##
## The amalgamated form could be generated from full git repository
## on Linux just by `make dist`.
##
##
## Copyright 2019 Leonid Yuriev <leo@yuriev.ru>
## and other libmdbx authors: please see AUTHORS file.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
##
##
## libmdbx = { Revised and extended descendant of Symas LMDB. }
## Please see README.md at https://github.com/leo-yuriev/libmdbx
##
## Libmdbx is superior to LMDB in terms of features and reliability,
## not inferior in performance. libmdbx works on Linux, FreeBSD, MacOS X
## and other systems compliant with POSIX.1-2008, but also support Windows
## as a complementary platform.
##
## The next version is under active non-public development and will be
## released as MithrilDB and libmithrildb for libraries & packages.
## Admittedly mythical Mithril is resembling silver but being stronger and
## lighter than steel. Therefore MithrilDB is rightly relevant name.
##
## MithrilDB will be radically different from libmdbx by the new database
## format and API based on C++17, as well as the Apache 2.0 License.
## The goal of this revolution is to provide a clearer and robust API,
## add more features and new valuable properties of database.
##
## The Future will (be) Positive. Всё будет хорошо.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
if(NOT CMAKE_VERSION VERSION_LESS 3.9)
cmake_policy(SET CMP0069 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.12)
cmake_policy(SET CMP0075 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.13)
cmake_policy(SET CMP0077 NEW)
endif()
if(DEFINED PROJECT_NAME)
set(SUBPROJECT ON)
set(NOT_SUBPROJECT OFF)
else()
set(SUBPROJECT OFF)
set(NOT_SUBPROJECT ON)
project(libmdbx C CXX)
endif()
find_package(Threads REQUIRED)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11)
if(NOT HAS_C11 LESS 0)
set(MDBX_C_STANDARD 11)
else()
set(MDBX_C_STANDARD 99)
endif()
message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx")
# not supported by this (minimal) script
add_definitions(-DMDBX_AVOID_CRT=0)
# provide build timestamp
string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC)
add_definitions(-DMDBX_BUILD_TIMESTAMP="${MDBX_BUILD_TIMESTAMP}")
# provide compiler info
execute_process(COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1"
OUTPUT_VARIABLE MDBX_BUILD_COMPILER
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
RESULT_VARIABLE rc)
if(rc OR NOT MDBX_BUILD_COMPILER)
string(STRIP "${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}" MDBX_BUILD_COMPILER)
endif()
add_definitions(-DMDBX_BUILD_COMPILER="${MDBX_BUILD_COMPILER}")
# provide cpu/arch-system pair
if(CMAKE_C_COMPILER_TARGET)
set(MDBX_BUILD_TARGET "${CMAKE_C_COMPILER_TARGET}")
elseif(CMAKE_C_PLATFORM_ID AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_C_PLATFORM_ID}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_LIBRARY_ARCHITECTURE)
string(STRIP "${CMAKE_LIBRARY_ARCHITECTURE}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_GENERATOR_PLATFORM}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_SYSTEM_ARCH)
string(STRIP "${CMAKE_SYSTEM_ARCH}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
else()
string(STRIP "${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
endif()
add_definitions(-DMDBX_BUILD_TARGET="${MDBX_BUILD_TARGET}")
# provide build target-config
if(CMAKE_CONFIGURATION_TYPES)
add_definitions(-DMDBX_BUILD_CONFIG="$<CONFIG>")
else()
add_definitions(-DMDBX_BUILD_CONFIG="${CMAKE_BUILD_TYPE}")
endif()
# provide build cflags
set(MDBX_BUILD_FLAGS "")
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS})
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES})
if(CMAKE_CONFIGURATION_TYPES)
add_definitions(-DMDBX_BUILD_FLAGS_CONFIG="$<$<CONFIG:Debug>:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$<CONFIG:Release>:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$<CONFIG:RelWithDebInfo>:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$<CONFIG:MinSizeRel>:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>")
else()
string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE)
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}})
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES_${CMAKE_BUILD_TYPE_UPPERCASE}})
endif()
list(REMOVE_DUPLICATES MDBX_BUILD_FLAGS)
string(REPLACE ";" " " MDBX_BUILD_FLAGS "${MDBX_BUILD_FLAGS}")
add_definitions(-DMDBX_BUILD_FLAGS="${MDBX_BUILD_FLAGS}")
# shared library
if(NOT DEFINED MDBX_BUILD_SHARED_LIBRARY)
if(DEFINED BUILD_SHARED_LIBS)
option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS})
else()
option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ON)
endif()
endif()
if(MDBX_BUILD_SHARED_LIBRARY)
add_library(mdbx SHARED mdbx.c mdbx.h)
set_target_properties(mdbx PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
PUBLIC_HEADER mdbx.h)
target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS INTERFACE LIBMDBX_IMPORTS)
if(DEFINED INTERPROCEDURAL_OPTIMIZATION)
set_target_properties(mdbx PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
endif()
target_link_libraries(mdbx PRIVATE ${CMAKE_THREAD_LIBS_INIT})
if(WIN32)
target_link_libraries(mdbx PRIVATE ntdll.lib)
endif()
endif()
# static library used for tools, to avoid rpath/dll-path troubles
add_library(mdbx-static STATIC EXCLUDE_FROM_ALL mdbx.c mdbx.h)
set_target_properties(mdbx-static PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
PUBLIC_HEADER mdbx.h)
target_link_libraries(mdbx-static INTERFACE ${CMAKE_THREAD_LIBS_INIT})
if(DEFINED INTERPROCEDURAL_OPTIMIZATION)
set_target_properties(mdbx-static PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
endif()
if(WIN32)
target_link_libraries(mdbx-static INTERFACE ntdll.lib)
endif()
# mdbx-tools
foreach(TOOL mdbx_chk mdbx_copy mdbx_stat mdbx_dump mdbx_load)
add_executable(${TOOL} ${TOOL}.c)
set_target_properties(${TOOL} PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON)
if(DEFINED INTERPROCEDURAL_OPTIMIZATION)
set_target_properties(${TOOL} PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
endif()
target_link_libraries(${TOOL} mdbx-static)
endforeach()
cmake_policy(POP)

342
src/contrib/db/libmdbx/CMakeLists.txt Normal file
View file

@ -0,0 +1,342 @@
##
## Copyright 2019 Leonid Yuriev <leo@yuriev.ru>
## and other libmdbx authors: please see AUTHORS file.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
##
##
## libmdbx = { Revised and extended descendant of Symas LMDB. }
## Please see README.md at https://github.com/leo-yuriev/libmdbx
##
## Libmdbx is superior to LMDB in terms of features and reliability,
## not inferior in performance. libmdbx works on Linux, FreeBSD, MacOS X
## and other systems compliant with POSIX.1-2008, but also support Windows
## as a complementary platform.
##
## The next version is under active non-public development and will be
## released as MithrilDB and libmithrildb for libraries & packages.
## Admittedly mythical Mithril is resembling silver but being stronger and
## lighter than steel. Therefore MithrilDB is rightly relevant name.
##
## MithrilDB will be radically different from libmdbx by the new database
## format and API based on C++17, as well as the Apache 2.0 License.
## The goal of this revolution is to provide a clearer and robust API,
## add more features and new valuable properties of database.
##
## The Future will (be) Positive. Всё будет хорошо.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
if(NOT CMAKE_VERSION VERSION_LESS 3.13)
cmake_policy(SET CMP0077 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.12)
cmake_policy(SET CMP0075 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.9)
cmake_policy(SET CMP0069 NEW)
include(CheckIPOSupported)
check_ipo_supported(RESULT CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE)
else()
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE FALSE)
endif()
if(DEFINED PROJECT_NAME)
set(SUBPROJECT ON)
set(NOT_SUBPROJECT OFF)
if(NOT DEFINED BUILD_TESTING)
set(BUILD_TESTING OFF)
endif()
else()
set(SUBPROJECT OFF)
set(NOT_SUBPROJECT ON)
project(libmdbx C CXX)
if(NOT DEFINED BUILD_TESTING)
set(BUILD_TESTING ON)
endif()
endif()
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
macro(add_mdbx_option NAME DESCRIPTION DEFAULT)
list(APPEND MDBX_BUILD_OPTIONS ${NAME})
if(NOT ${DEFAULT} STREQUAL "AUTO")
option(${NAME} "${DESCRIPTION}" ${DEFAULT})
endif()
endmacro()
# only for compatibility testing
# set(CMAKE_CXX_STANDARD 14)
if(NOT "$ENV{TEAMCITY_PROCESS_FLOW_ID}" STREQUAL "")
set(CI TEAMCITY)
message(STATUS "TeamCity CI")
elseif(NOT "$ENV{TRAVIS}" STREQUAL "")
set(CI TRAVIS)
message(STATUS "Travis CI")
elseif(NOT "$ENV{CIRCLECI}" STREQUAL "")
set(CI CIRCLE)
message(STATUS "Circle CI")
elseif(NOT "$ENV{APPVEYOR}" STREQUAL "")
set(CI APPVEYOR)
message(STATUS "AppVeyor CI")
elseif(NOT "$ENV{CI}" STREQUAL "")
set(CI "$ENV{CI}")
message(STATUS "Other CI (${CI})")
else()
message(STATUS "Assume No any CI environment")
unset(CI)
endif()
# output all mdbx-related targets in single directory
if(NOT DEFINED MDBX_OUTPUT_DIR)
set(MDBX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
endif()
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
set(CMAKE_PDB_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
include(CheckLibraryExists)
include(CheckIncludeFiles)
include(CheckCCompilerFlag)
include(CheckSymbolExists)
include(CheckCSourceRuns)
include(CheckCXXSourceRuns)
include(CheckCSourceCompiles)
include(CheckCXXSourceCompiles)
include(TestBigEndian)
include(CheckFunctionExists)
include(FindPackageMessage)
include(CheckStructHasMember)
include(CMakeDependentOption)
include(GNUInstallDirs)
if(CMAKE_C_COMPILER_ID STREQUAL "MSVC" AND MSVC_VERSION LESS 1900)
message(SEND_ERROR "MSVC compiler ${MSVC_VERSION} is too old for building MDBX."
" At least 'Microsoft Visual Studio 2015' is required.")
endif()
# Set default build type to Release. This is to ease a User's life.
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE STRING
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE)
include(cmake/utils.cmake)
include(cmake/compiler.cmake)
include(cmake/profile.cmake)
find_program(ECHO echo)
find_program(CAT cat)
find_program(GIT git)
find_program(LD ld)
# CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H)
# CHECK_INCLUDE_FILES(sys/uio.h HAVE_SYS_UIO_H)
# CHECK_INCLUDE_FILES(sys/stat.h HAVE_SYS_STAT_H)
CHECK_FUNCTION_EXISTS(pow NOT_NEED_LIBM)
if(NOT_NEED_LIBM)
set(LIB_MATH "")
else()
set(CMAKE_REQUIRED_LIBRARIES m)
CHECK_FUNCTION_EXISTS(pow HAVE_LIBM)
if(HAVE_LIBM)
set(LIB_MATH m)
else()
message(FATAL_ERROR "No libm found for math support")
endif()
endif()
find_package(Threads REQUIRED)
if(SUBPROJECT)
if(NOT DEFINED BUILD_SHARED_LIBS)
option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" OFF)
endif()
if(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE)
option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON)
endif()
else()
option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" ON)
option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON)
if (CC_HAS_ARCH_NATIVE)
option(BUILD_FOR_NATIVE_CPU "Generate code for the compiling machine CPU" OFF)
endif()
if(CMAKE_CONFIGURATION_TYPES OR NOT CMAKE_BUILD_TYPE_UPPERCASE STREQUAL "DEBUG")
set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT ON)
else()
set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT OFF)
endif()
if(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE
OR GCC_LTO_AVAILABLE OR MSVC_LTO_AVAILABLE OR CLANG_LTO_AVAILABLE)
option(INTERPROCEDURAL_OPTIMIZATION "Enable interprocedural/LTO optimization" ${INTERPROCEDURAL_OPTIMIZATION_DEFAULT})
endif()
if(INTERPROCEDURAL_OPTIMIZATION)
if(GCC_LTO_AVAILABLE)
set(LTO_ENABLED TRUE)
set(CMAKE_AR ${CMAKE_GCC_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE)
set(CMAKE_NM ${CMAKE_GCC_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE)
set(CMAKE_RANLIB ${CMAKE_GCC_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE)
message(STATUS "MDBX indulge Link-Time Optimization by GCC")
elseif(CLANG_LTO_AVAILABLE)
set(LTO_ENABLED TRUE)
set(CMAKE_AR ${CMAKE_CLANG_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE)
set(CMAKE_NM ${CMAKE_CLANG_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE)
set(CMAKE_RANLIB ${CMAKE_CLANG_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE)
message(STATUS "MDBX indulge Link-Time Optimization by CLANG")
elseif(MSVC_LTO_AVAILABLE)
set(LTO_ENABLED TRUE)
message(STATUS "MDBX indulge Link-Time Optimization by MSVC")
elseif(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE)
message(STATUS "MDBX indulge Interprocedural Optimization by CMake")
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
set(LTO_ENABLED TRUE)
else()
message(WARNING "Unable to engage interprocedural/LTO optimization.")
endif()
else()
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION FALSE)
set(LTO_ENABLED FALSE)
endif()
find_program(VALGRIND valgrind)
if(VALGRIND)
# LY: cmake is ugly and nasty.
# - therefore memcheck-options should be defined before including ctest;
# - otherwise ctest may ignore it.
set(MEMORYCHECK_SUPPRESSIONS_FILE
"${PROJECT_SOURCE_DIR}/test/valgrind_suppress.txt"
CACHE FILEPATH "Suppressions file for Valgrind" FORCE)
set(MEMORYCHECK_COMMAND_OPTIONS
"--trace-children=yes --leak-check=full --track-origins=yes --error-exitcode=42 --error-markers=@ --errors-for-leak-kinds=definite --fair-sched=yes --suppressions=${MEMORYCHECK_SUPPRESSIONS_FILE}"
CACHE STRING "Valgrind options" FORCE)
set(VALGRIND_COMMAND_OPTIONS "${MEMORYCHECK_COMMAND_OPTIONS}" CACHE STRING "Valgrind options" FORCE)
endif()
#
# Enable 'make tags' target.
find_program(CTAGS ctags)
if(CTAGS)
add_custom_target(tags COMMAND ${CTAGS} -R -f tags
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
add_custom_target(ctags DEPENDS tags)
endif(CTAGS)
#
# Enable 'make reformat' target.
find_program(CLANG_FORMAT
NAMES clang-format-6.0 clang-format-5.0 clang-format-4.0
clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format)
if(CLANG_FORMAT AND UNIX)
add_custom_target(reformat
VERBATIM
COMMAND
git ls-files |
grep -E \\.\(c|cxx|cc|cpp|h|hxx|hpp\)\(\\.in\)?\$ |
xargs ${CLANG_FORMAT} -i --style=file
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
endif()
if(NOT "${PROJECT_BINARY_DIR}" STREQUAL "${PROJECT_SOURCE_DIR}")
add_custom_target(distclean)
add_custom_command(TARGET distclean
COMMAND ${CMAKE_COMMAND} -E remove_directory "${PROJECT_BINARY_DIR}"
COMMENT "Removing the build directory and its content")
elseif(IS_DIRECTORY .git AND GIT)
add_custom_target(distclean)
add_custom_command(TARGET distclean
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${GIT} submodule foreach --recursive git clean -f -X -d
COMMAND ${GIT} clean -f -X -d
COMMENT "Removing all build files from the source directory")
endif()
setup_compile_flags()
endif(SUBPROJECT)
list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11)
if(NOT HAS_C11 LESS 0)
set(MDBX_C_STANDARD 11)
else()
set(MDBX_C_STANDARD 99)
endif()
message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx")
##############################################################################
##############################################################################
#
# #### ##### ##### # #### # # ####
# # # # # # # # # ## # #
# # # # # # # # # # # # ####
# # # ##### # # # # # # # #
# # # # # # # # # ## # #
# #### # # # #### # # ####
#
set(MDBX_BUILD_OPTIONS ENABLE_ASAN MDBX_USE_VALGRIND ENABLE_GPROF ENABLE_GCOV)
add_mdbx_option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS})
add_mdbx_option(MDBX_ALLOY_BUILD "Build MDBX library as single object file" ON)
add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON)
add_mdbx_option(MDBX_TXN_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO)
mark_as_advanced(MDBX_TXN_CHECKPID)
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
add_mdbx_option(MDBX_DISABLE_GNU_SOURCE "Don't use nonstandard GNU/Linux extension functions" OFF)
mark_as_advanced(MDBX_DISABLE_GNU_SOURCE)
endif()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
add_mdbx_option(MDBX_OSX_SPEED_INSTEADOF_DURABILITY "Disable use fcntl(F_FULLFSYNC) in favor of speed" OFF)
mark_as_advanced(MDBX_OSX_SPEED_INSTEADOF_DURABILITY)
endif()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
add_mdbx_option(MDBX_AVOID_CRT "Avoid dependence from MSVC CRT" ${NOT_SUBPROJECT})
if(NOT MDBX_BUILD_SHARED_LIBRARY)
add_mdbx_option(MDBX_CONFIG_MANUAL_TLS_CALLBACK
"Provide mdbx_dll_handler() for manual initialization" OFF)
mark_as_advanced(MDBX_CONFIG_MANUAL_TLS_CALLBACK)
endif()
else()
add_mdbx_option(MDBX_USE_ROBUST "Use POSIX.1-2008 robust mutexes" AUTO)
mark_as_advanced(MDBX_USE_ROBUST)
add_mdbx_option(MDBX_USE_OFDLOCKS "Use Open file description locks (aka OFD locks, non-POSIX)" AUTO)
mark_as_advanced(MDBX_USE_OFDLOCKS)
endif()
option(MDBX_ENABLE_TESTS "Build MDBX tests." ${BUILD_TESTING})
################################################################################
################################################################################
add_subdirectory(src)
if(MDBX_ENABLE_TESTS)
add_subdirectory(test)
endif()
set(PACKAGE "libmdbx")
set(CPACK_PACKAGE_VERSION_MAJOR ${MDBX_VERSION_MAJOR})
set(CPACK_PACKAGE_VERSION_MINOR ${MDBX_VERSION_MINOR})
set(CPACK_PACKAGE_VERSION_PATCH ${MDBX_VERSION_RELEASE})
set(CPACK_PACKAGE_VERSION_COMMIT ${MDBX_VERSION_REVISION})
set(PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${CPACK_PACKAGE_VERSION_COMMIT}")
message(STATUS "libmdbx package version is ${PACKAGE_VERSION}")
cmake_policy(POP)

22
src/contrib/db/libmdbx/COPYRIGHT Normal file
View file

@ -0,0 +1,22 @@
Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
Copyright 2011-2015 Howard Chu, Symas Corp.
Copyright 2015,2016 Peter-Service R&D LLC.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted only as authorized by the OpenLDAP
Public License.
A copy of this license is available in the file LICENSE in the
top-level directory of the distribution or, alternatively, at
<http://www.OpenLDAP.org/license.html>.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Individual files and/or contributed packages may be copyright by
other parties and/or subject to additional restrictions.
This work also contains materials derived from public sources.
Additional information about OpenLDAP can be obtained at
<http://www.openldap.org/>.

362
src/contrib/db/libmdbx/GNUmakefile Normal file
View file

@ -0,0 +1,362 @@
# This makefile is for GNU Make, and nowadays provided
# just for compatibility and preservation of traditions.
# Please use CMake in case of any difficulties or problems.
#
# Preprocessor macros (for MDBX_OPTIONS) of interest...
# Note that the defaults should already be correct for most platforms;
# you should not need to change any of these. Read their descriptions
# in README and source code if you do. There may be other macros of interest.
SHELL := /bin/bash
# install sandbox
SANDBOX ?=
# install prefixes (inside sandbox)
prefix ?= /usr/local
mandir ?= $(prefix)/man
# lib/bin suffix for multiarch/biarch, e.g. '.x86_64'
suffix ?=
CC ?= gcc
LD ?= ld
MDBX_OPTIONS ?= -DNDEBUG=1
CFLAGS ?= -Os -g3 -Wall -Werror -Wextra -Wpedantic -ffunction-sections -fPIC -fvisibility=hidden -std=gnu11 -pthread -Wno-tautological-compare
# LY: '--no-as-needed,-lrt' for ability to built with modern glibc, but then run with the old
LDFLAGS ?= $(shell $(LD) --help 2>/dev/null | grep -q -- --gc-sections && echo '-Wl,--gc-sections,-z,relro,-O1')$(shell $(LD) --help 2>/dev/null | grep -q -- -dead_strip && echo '-Wl,-dead_strip')
EXE_LDFLAGS ?= -pthread
################################################################################
UNAME := $(shell uname -s 2>/dev/null || echo Unknown)
define uname2sosuffix
case "$(UNAME)" in
Darwin*|Mach*) echo dylib;;
CYGWIN*|MINGW*|MSYS*|Windows*) echo dll;;
*) echo so;;
esac
endef
SO_SUFFIX := $(shell $(uname2sosuffix))
HEADERS := mdbx.h
LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX)
TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk
MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1
.PHONY: mdbx all install clean test dist check
all: $(LIBRARIES) $(TOOLS)
mdbx: libmdbx.a libmdbx.$(SO_SUFFIX)
tools: $(TOOLS)
strip: all
strip libmdbx.$(SO_SUFFIX) $(TOOLS)
clean:
rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *~ tmp.db/* \
*.gcov *.log *.err src/*.o test/*.o mdbx_example dist \
config.h src/elements/config.h src/elements/version.c *.tar*
libmdbx.a: mdbx-static.o
$(AR) rs $@ $?
libmdbx.$(SO_SUFFIX): mdbx-dylib.o
$(CC) $(CFLAGS) $^ -pthread -shared $(LDFLAGS) -o $@
#> dist-cutoff-begin
ifeq ($(wildcard mdbx.c),mdbx.c)
#< dist-cutoff-end
################################################################################
# Amalgamated source code, i.e. distributed after `make dists`
MAN_SRCDIR := man1/
config.h: mdbx.c $(lastword $(MAKEFILE_LIST))
(echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \
&& echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS)"' \
&& echo '#define MDBX_BUILD_COMPILER "$(shell set -o pipefail; $(CC) --version | head -1 || echo 'Please use GCC or CLANG compatible compiler')"' \
&& echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || echo 'Please use GCC or CLANG compatible compiler')"' \
) > $@
mdbx-dylib.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c mdbx.c -o $@
mdbx-static.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c mdbx.c -o $@
mdbx_%: mdbx_%.c libmdbx.a
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) -o $@
#> dist-cutoff-begin
else
################################################################################
# Plain (non-amalgamated) sources with test
define uname2osal
case "$(UNAME)" in
CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;;
*) echo unix;;
esac
endef
define uname2titer
case "$(UNAME)" in
Darwin*|Mach*) echo 2;;
*) echo 12;;
esac
endef
DIST_EXTRA := LICENSE README.md CMakeLists.txt GNUmakefile $(addprefix man1/, $(MANPAGES))
DIST_SRC := mdbx.h mdbx.c $(addsuffix .c, $(TOOLS))
TEST_DB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.db
TEST_LOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.log
TEST_OSAL := $(shell $(uname2osal))
TEST_ITER := $(shell $(uname2titer))
TEST_SRC := test/osal-$(TEST_OSAL).cc $(filter-out $(wildcard test/osal-*.cc), $(wildcard test/*.cc))
TEST_INC := $(wildcard test/*.h)
TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC))
CXX ?= g++
CXXSTD ?= $(shell $(CXX) -std=c++27 -c test/test.cc -o /dev/null 2>/dev/null && echo -std=c++17 || echo -std=c++11)
CXXFLAGS := $(CXXSTD) $(filter-out -std=gnu11,$(CFLAGS))
MAN_SRCDIR := src/man1/
ALLOY_DEPS := $(wildcard src/elements/*)
MDBX_VERSION_GIT = ${shell set -o pipefail; git describe --tags | sed -n 's|^v*\([0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\)\(.*\)|\1|p' || echo 'Please fetch tags and/or install latest git version'}
MDBX_GIT_TIMESTAMP = $(shell git show --no-patch --format=%cI HEAD || echo 'Please install latest get version')
MDBX_GIT_DESCRIBE = $(shell git describe --tags --long --dirty=-dirty || echo 'Please fetch tags and/or install latest git version')
MDBX_VERSION_SUFFIX = $(shell set -o pipefail; echo -n '$(MDBX_GIT_DESCRIBE)' | tr -c -s '[a-zA-Z0-9]' _)
MDBX_BUILD_SOURCERY = $(shell set -o pipefail; $(MAKE) -s src/elements/version.c && (openssl dgst -r -sha256 src/elements/version.c || sha256sum src/elements/version.c || shasum -a 256 src/elements/version.c) 2>/dev/null | cut -d ' ' -f 1 || echo 'Please install openssl or sha256sum or shasum')_$(MDBX_VERSION_SUFFIX)
test check: all mdbx_example mdbx_test
rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \
(./mdbx_test --progress --console=no --repeat=$(TEST_ITER) --pathname=$(TEST_DB) --dont-cleanup-after basic && \
./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=1 --pathname=$(TEST_DB) --dont-cleanup-after basic) \
| tee -a $(TEST_LOG) | tail -n 42) \
&& ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
mdbx_example: mdbx.h example/example-mdbx.c libmdbx.$(SO_SUFFIX)
$(CC) $(CFLAGS) -I. example/example-mdbx.c ./libmdbx.$(SO_SUFFIX) -o $@
check-singleprocess: all mdbx_test
rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \
(./mdbx_test --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --hill && \
./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \
./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --nested) \
| tee -a $(TEST_LOG) | tail -n 42) \
&& ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
check-fault: all mdbx_test
rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --inject-writefault=42 --dump-config --dont-cleanup-after basic | tee -a $(TEST_LOG) | tail -n 42) \
; ./mdbx_chk -vvnw $(TEST_DB) && ([ ! -e $(TEST_DB)-copy ] || ./mdbx_chk -vvn $(TEST_DB)-copy)
VALGRIND=valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt
memcheck check-valgrind: all mdbx_test
@echo "$(MDBX_OPTIONS)" | grep -q MDBX_USE_VALGRIND || echo "WARNING: Please build libmdbx with -DMDBX_USE_VALGRIND to avoid false-positives from Valgrind !!!" >&2
rm -f valgrind-*.log $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \
($(VALGRIND) ./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=4 --pathname=$(TEST_DB) --dont-cleanup-after basic && \
$(VALGRIND) ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \
$(VALGRIND) ./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-after basic) \
| tee -a $(TEST_LOG) | tail -n 42) \
&& $(VALGRIND) ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
define test-rule
$(patsubst %.cc,%.o,$(1)): $(1) $(TEST_INC) mdbx.h $(lastword $(MAKEFILE_LIST))
$(CXX) $(CXXFLAGS) $(MDBX_OPTIONS) -c $(1) -o $$@
endef
$(foreach file,$(TEST_SRC),$(eval $(call test-rule,$(file))))
mdbx_%: src/tools/mdbx_%.c libmdbx.a
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) -o $@
mdbx_test: $(TEST_OBJ) libmdbx.$(SO_SUFFIX)
$(CXX) $(CXXFLAGS) $(TEST_OBJ) -Wl,-rpath . -L . -l mdbx $(EXE_LDFLAGS) -o $@
git_DIR := $(shell if [ -d .git ]; then echo .git; elif [ -s .git -a -f .git ]; then grep '^gitdir: ' .git | cut -d ':' -f 2; else echo "Please use libmdbx as a git-submodule or the amalgamated source code" >&2 && echo git_directory; fi)
src/elements/version.c: src/elements/version.c.in $(lastword $(MAKEFILE_LIST)) $(git_DIR)/HEAD $(git_DIR)/index $(git_DIR)/refs/tags
sed \
-e "s|@MDBX_GIT_TIMESTAMP@|$(MDBX_GIT_TIMESTAMP)|" \
-e "s|@MDBX_GIT_TREE@|$(shell git show --no-patch --format=%T HEAD || echo 'Please install latest get version')|" \
-e "s|@MDBX_GIT_COMMIT@|$(shell git show --no-patch --format=%H HEAD || echo 'Please install latest get version')|" \
-e "s|@MDBX_GIT_DESCRIBE@|$(MDBX_GIT_DESCRIBE)|" \
-e "s|\$${MDBX_VERSION_MAJOR}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 1)|" \
-e "s|\$${MDBX_VERSION_MINOR}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 2)|" \
-e "s|\$${MDBX_VERSION_RELEASE}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 3)|" \
-e "s|\$${MDBX_VERSION_REVISION}|$(shell git rev-list --count --no-merges HEAD || echo 'Please fetch tags and/or install latest git version')|" \
src/elements/version.c.in > $@
src/elements/config.h: src/elements/version.c $(lastword $(MAKEFILE_LIST))
(echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \
&& echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS)"' \
&& echo '#define MDBX_BUILD_COMPILER "$(shell set -o pipefail; $(CC) --version | head -1 || echo 'Please use GCC or CLANG compatible compiler')"' \
&& echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || echo 'Please use GCC or CLANG compatible compiler')"' \
&& echo '#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)' \
) > $@
mdbx-dylib.o: src/elements/config.h src/elements/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c src/alloy.c -o $@
mdbx-static.o: src/elements/config.h src/elements/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c src/alloy.c -o $@
.PHONY: dist
dist: libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz $(lastword $(MAKEFILE_LIST))
libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz: $(addprefix dist/, $(DIST_SRC) $(DIST_EXTRA)) $(addprefix dist/man1/,$(MANPAGES))
tar -c -a -f $@ --owner=0 --group=0 -C dist $(DIST_SRC) $(DIST_EXTRA) \
&& rm dist/@tmp-shared_internals.inc
dist/mdbx.h: mdbx.h src/elements/version.c $(lastword $(MAKEFILE_LIST))
mkdir -p dist && cp $< $@
dist/GNUmakefile: GNUmakefile
mkdir -p dist && sed -e '/^#> dist-cutoff-begin/,/^#< dist-cutoff-end/d' $< > $@
dist/@tmp-shared_internals.inc: src/elements/version.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
mkdir -p dist && sed \
-e 's|#pragma once|#define MDBX_ALLOY 1\n#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)|' \
-e 's|#include "../../mdbx.h"|@INCLUDE "mdbx.h"|' \
-e '/#include "defs.h"/r src/elements/defs.h' \
-e '/#include "osal.h"/r src/elements/osal.h' \
src/elements/internals.h > $@
dist/mdbx.c: dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST))
mkdir -p dist && (cat dist/@tmp-shared_internals.inc \
&& cat src/elements/core.c src/elements/osal.c src/elements/version.c \
&& echo '#if defined(_WIN32) || defined(_WIN64)' \
&& cat src/elements/lck-windows.c && echo '#else /* LCK-implementation */' \
&& cat src/elements/lck-posix.c && echo '#endif /* LCK-implementation */' \
) | grep -v -e '#include "' -e '#pragma once' | sed 's|@INCLUDE|#include|' > $@
define dist-tool-rule
dist/$(1).c: src/tools/$(1).c src/tools/wingetopt.h src/tools/wingetopt.c \
dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST))
mkdir -p dist && sed \
-e '/#include "..\/elements\/internals.h"/r dist/@tmp-shared_internals.inc' \
-e '/#include "wingetopt.h"/r src/tools/wingetopt.c' \
src/tools/$(1).c \
| grep -v -e '#include "' -e '#pragma once' -e '#define MDBX_ALLOY' \
| sed 's|@INCLUDE|#include|' > $$@
endef
$(foreach file,$(TOOLS),$(eval $(call dist-tool-rule,$(file))))
dist/man1/mdbx_%.1: src/man1/mdbx_%.1
mkdir -p dist/man1/ && cp $< $@
dist/LICENSE: LICENSE
mkdir -p dist/man1/ && cp $< $@
dist/README.md: README.md
mkdir -p dist/man1/ && cp $< $@
dist/CMakeLists.txt: CMakeLists.dist-minimal
mkdir -p dist/man1/ && cp $< $@
endif
################################################################################
# Cross-compilation simple test
CROSS_LIST = mips-linux-gnu-gcc \
powerpc64-linux-gnu-gcc powerpc-linux-gnu-gcc \
arm-linux-gnueabihf-gcc aarch64-linux-gnu-gcc \
sh4-linux-gnu-gcc mips64-linux-gnuabi64-gcc
# hppa-linux-gnu-gcc - don't supported by current qemu release
# s390x-linux-gnu-gcc - qemu troubles (hang/abort)
# sparc64-linux-gnu-gcc - qemu troubles (fcntl for F_SETLK/F_GETLK)
# alpha-linux-gnu-gcc - qemu (or gcc) troubles (coredump)
CROSS_LIST_NOQEMU = hppa-linux-gnu-gcc s390x-linux-gnu-gcc \
sparc64-linux-gnu-gcc alpha-linux-gnu-gcc
cross-gcc:
@echo "CORRESPONDING CROSS-COMPILERs ARE REQUIRED."
@echo "FOR INSTANCE: apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
@for CC in $(CROSS_LIST_NOQEMU) $(CROSS_LIST); do \
echo "===================== $$CC"; \
$(MAKE) clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) all || exit $$?; \
done
# Unfortunately qemu don't provide robust support for futexes.
# Therefore it is impossible to run full multi-process tests.
cross-qemu:
@echo "CORRESPONDING CROSS-COMPILERs AND QEMUs ARE REQUIRED."
@echo "FOR INSTANCE: "
@echo " 1) apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
@echo " 2) apt install binfmt-support qemu-user-static qemu-user qemu-system-arm qemu-system-mips qemu-system-misc qemu-system-ppc qemu-system-sparc"
@for CC in $(CROSS_LIST); do \
echo "===================== $$CC + qemu"; \
$(MAKE) clean && \
CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static MDBX_OPTIONS="-DMDBX_SAFE4QEMU $(MDBX_OPTIONS)" \
$(MAKE) check-singleprocess || exit $$?; \
done
#< dist-cutoff-end
install: $(LIBRARIES) $(TOOLS) $(HEADERS)
mkdir -p $(SANDBOX)$(prefix)/bin$(suffix) \
&& cp -t $(SANDBOX)$(prefix)/bin$(suffix) $(TOOLS) && \
mkdir -p $(SANDBOX)$(prefix)/lib$(suffix) \
&& cp -t $(SANDBOX)$(prefix)/lib$(suffix) $(LIBRARIES) && \
mkdir -p $(SANDBOX)$(prefix)/include \
&& cp -t $(SANDBOX)$(prefix)/include $(HEADERS) && \
mkdir -p $(SANDBOX)$(mandir)/man1 \
&& cp -t $(SANDBOX)$(mandir)/man1 $(addprefix $(MAN_SRCDIR), $(MANPAGES))
################################################################################
# Benchmarking by ioarena
IOARENA ?= $(shell \
(test -x ../ioarena/@BUILD/src/ioarena && echo ../ioarena/@BUILD/src/ioarena) || \
(test -x ../../@BUILD/src/ioarena && echo ../../@BUILD/src/ioarena) || \
(test -x ../../src/ioarena && echo ../../src/ioarena) || which ioarena)
NN ?= 25000000
ifneq ($(wildcard $(IOARENA)),)
.PHONY: bench clean-bench re-bench
clean-bench:
rm -rf bench-*.txt _ioarena/*
re-bench: clean-bench bench
define bench-rule
bench-$(1)_$(2).txt: $(3) $(IOARENA) $(lastword $(MAKEFILE_LIST))
LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \
$(IOARENA) -D $(1) -B crud -m nosync -n $(2) \
| tee $$@ | grep throughput && \
LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \
$(IOARENA) -D $(1) -B get,iterate -m sync -r 4 -n $(2) \
| tee -a $$@ | grep throughput \
|| mv -f $$@ $$@.error
endef
$(eval $(call bench-rule,mdbx,$(NN),libmdbx.$(SO_SUFFIX)))
$(eval $(call bench-rule,sophia,$(NN)))
$(eval $(call bench-rule,leveldb,$(NN)))
$(eval $(call bench-rule,rocksdb,$(NN)))
$(eval $(call bench-rule,wiredtiger,$(NN)))
$(eval $(call bench-rule,forestdb,$(NN)))
$(eval $(call bench-rule,lmdb,$(NN)))
$(eval $(call bench-rule,nessdb,$(NN)))
$(eval $(call bench-rule,sqlite3,$(NN)))
$(eval $(call bench-rule,ejdb,$(NN)))
$(eval $(call bench-rule,vedisdb,$(NN)))
$(eval $(call bench-rule,dummy,$(NN)))
$(eval $(call bench-rule,debug,10))
bench: bench-mdbx_$(NN).txt
.PHONY: bench-debug
bench-debug: bench-debug_10.txt
bench-quartet: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt bench-rocksdb_$(NN).txt bench-wiredtiger_$(NN).txt
endif

47
src/contrib/db/libmdbx/LICENSE Normal file
View file

@ -0,0 +1,47 @@
The OpenLDAP Public License
Version 2.8, 17 August 2003
Redistribution and use of this software and associated documentation
("Software"), with or without modification, are permitted provided
that the following conditions are met:
1. Redistributions in source form must retain copyright statements
and notices,
2. Redistributions in binary form must reproduce applicable copyright
statements and notices, this list of conditions, and the following
disclaimer in the documentation and/or other materials provided
with the distribution, and
3. Redistributions must contain a verbatim copy of this document.
The OpenLDAP Foundation may revise this license from time to time.
Each revision is distinguished by a version number. You may use
this Software under terms of this license revision or under the
terms of any subsequent revision of the license.
THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
The names of the authors and copyright holders must not be used in
advertising or otherwise to promote the sale, use or other dealing
in this Software without specific, written prior permission. Title
to copyright in this Software shall at all times remain with copyright
holders.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
California, USA. All Rights Reserved. Permission to copy and
distribute verbatim copies of this document is granted.

591
src/contrib/db/libmdbx/README.md Normal file
View file

@ -0,0 +1,591 @@
### The [repository now only mirrored on the Github](https://abf.io/erthink/libmdbx) due to illegal discriminatory restrictions for Russian Crimea and for sovereign crimeans.
<!-- Required extensions: pymdownx.betterem, pymdownx.tilde, pymdownx.emoji, pymdownx.tasklist, pymdownx.superfences -->
-----
libmdbx
======================================
_libmdbx_ is an extremely fast, compact, powerful, embedded
transactional [key-value
store](https://en.wikipedia.org/wiki/Key-value_database)
database, with permissive [OpenLDAP Public License](LICENSE).
_libmdbx_ has a specific set of properties and capabilities,
focused on creating unique lightweight solutions with
extraordinary performance.
The next version is under active non-public development and will be
released as **_MithrilDB_** and `libmithrildb` for libraries & packages.
Admittedly mythical [Mithril](https://en.wikipedia.org/wiki/Mithril) is
resembling silver but being stronger and lighter than steel. Therefore
_MithrilDB_ is rightly relevant name.
> _MithrilDB_ will be radically different from _libmdbx_ by the new
> database format and API based on C++17, as well as the [Apache 2.0
> License](https://www.apache.org/licenses/LICENSE-2.0). The goal of this
> revolution is to provide a clearer and robust API, add more features and
> new valuable properties of database.
*The Future will (be) [Positive](https://www.ptsecurity.com). Всё будет хорошо.*
[![Build Status](https://travis-ci.org/leo-yuriev/libmdbx.svg?branch=master)](https://travis-ci.org/leo-yuriev/libmdbx)
[![Build status](https://ci.appveyor.com/api/projects/status/ue94mlopn50dqiqg/branch/master?svg=true)](https://ci.appveyor.com/project/leo-yuriev/libmdbx/branch/master)
[![Coverity Scan Status](https://scan.coverity.com/projects/12915/badge.svg)](https://scan.coverity.com/projects/reopen-libmdbx)
## Table of Contents
- [Overview](#overview)
- [Comparison with other databases](#comparison-with-other-databases)
- [History & Acknowledgments](#history)
- [Description](#description)
- [Key features](#key-features)
- [Improvements over LMDB](#improvements-over-lmdb)
- [Gotchas](#gotchas)
- [Usage](#usage)
- [Building](#building)
- [Bindings](#bindings)
- [Performance comparison](#performance-comparison)
- [Integral performance](#integral-performance)
- [Read scalability](#read-scalability)
- [Sync-write mode](#sync-write-mode)
- [Lazy-write mode](#lazy-write-mode)
- [Async-write mode](#async-write-mode)
- [Cost comparison](#cost-comparison)
-----
## Overview
_libmdbx_ is revised and extended descendant of amazing [Lightning
Memory-Mapped
Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
_libmdbx_ inherits all features and characteristics from
[LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database),
but resolves some issues and adds several features.
- _libmdbx_ guarantee data integrity after crash unless this was explicitly
neglected in favour of write performance.
- _libmdbx_ allows multiple processes to read and update several key-value
tables concurrently, while being
[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, with minimal
overhead and Olog(N) operation cost.
- _libmdbx_ enforce
[serializability](https://en.wikipedia.org/wiki/Serializability) for
writers by single
[mutex](https://en.wikipedia.org/wiki/Mutual_exclusion) and affords
[wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm#Wait-freedom)
for parallel readers without atomic/interlocked operations, while
writing and reading transactions do not block each other.
- _libmdbx_ uses [B+Trees](https://en.wikipedia.org/wiki/B%2B_tree) and
[Memory-Mapping](https://en.wikipedia.org/wiki/Memory-mapped_file),
doesn't use [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging)
which might be a caveat for some workloads.
- _libmdbx_ implements a simplified variant of the [Berkeley
DB](https://en.wikipedia.org/wiki/Berkeley_DB) and/or
[dbm](https://en.wikipedia.org/wiki/DBM_(computing)) API.
- _libmdbx_ supports Linux, Windows, MacOS, FreeBSD and other systems
compliant with POSIX.1-2008.
### Comparison with other databases
For now please refer to [chapter of "BoltDB comparison with other
databases"](https://github.com/coreos/bbolt#comparison-with-other-databases)
which is also (mostly) applicable to _libmdbx_.
### History
At first the development was carried out within the
[ReOpenLDAP](https://github.com/leo-yuriev/ReOpenLDAP) project. About a
year later _libmdbx_ was separated into standalone project, which was
[presented at Highload++ 2015
conference](http://www.highload.ru/2015/abstracts/1831.html).
Since 2017 _libmdbx_ is used in [Fast Positive Tables](https://github.com/leo-yuriev/libfpta),
and development is funded by [Positive Technologies](https://www.ptsecurity.com).
### Acknowledgments
Howard Chu <hyc@openldap.org> is the author of LMDB, from which
originated the MDBX in 2015.
Martin Hedenfalk <martin@bzero.se> is the author of `btree.c` code, which
was used for begin development of LMDB.
-----
Description
===========
## Key features
1. Key-value pairs are stored in ordered map(s), keys are always sorted,
range lookups are supported.
2. Data is [memory-mapped](https://en.wikipedia.org/wiki/Memory-mapped_file)
into each worker DB process, and could be accessed zero-copy from transactions.
3. Transactions are
[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, through to
[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
and [CoW](https://en.wikipedia.org/wiki/Copy-on-write). Writes are
strongly serialized and aren't blocked by reads, transactions can't
conflict with each other. Reads are guaranteed to get only commited data
([relaxing serializability](https://en.wikipedia.org/wiki/Serializability#Relaxing_serializability)).
4. Read transactions are
[non-blocking](https://en.wikipedia.org/wiki/Non-blocking_algorithm),
don't use [atomic operations](https://en.wikipedia.org/wiki/Linearizability#High-level_atomic_operations).
Readers don't block each other and aren't blocked by writers. Read
performance scales linearly with CPU core count.
> Nonetheless, "connect to DB" (starting the first read transaction in a thread) and
> "disconnect from DB" (closing DB or thread termination) requires a lock
> acquisition to register/unregister at the "readers table".
5. Keys with multiple values are stored efficiently without key
duplication, sorted by value, including integers (valuable for
secondary indexes).
6. Efficient operation on short fixed length keys,
including 32/64-bit integer types.
7. [WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write
Amplification Factor) и RAF (Read Amplification Factor) are Olog(N).
8. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) and
transaction journal. In case of a crash no recovery needed. No need for
regular maintenance. Backups can be made on the fly on working DB
without freezing writers.
9. No additional memory management, all done by basic OS services.
## Improvements over LMDB
_libmdbx_ is superior to _legendary [LMDB](https://symas.com/lmdb/)_ in
terms of features and reliability, not inferior in performance. In
comparison to LMDB, _libmdbx_ make things "just work" perfectly and
out-of-the-box, not silently and catastrophically break down. The list
below is pruned down to the improvements most notable and obvious from
the user's point of view.
1. Automatic on-the-fly database size control by preset parameters, both
reduction and increment.
> _libmdbx_ manage the database size according to parameters specified
> by `mdbx_env_set_geometry()` function,
> ones include the growth step and the truncation threshold.
2. Automatic continuous zero-overhead database compactification.
> _libmdbx_ logically move as possible a freed pages
> at end of allocation area into unallocated space,
> and then release such space if a lot of.
3. LIFO policy for recycling a Garbage Collection items. On systems with a disk
write-back cache, this can significantly increase write performance, up to
several times in a best case scenario.
> LIFO means that for reuse pages will be taken which became unused the lastest.
> Therefore the loop of database pages circulation becomes as short as possible.
> In other words, the number of pages, that are overwritten in memory
> and on disk during a series of write transactions, will be as small as possible.
> Thus creates ideal conditions for the efficient operation of the disk write-back cache.
4. Fast estimation of range query result volume, i.e. how many items can
be found between a `KEY1` and a `KEY2`. This is prerequisite for build
and/or optimize query execution plans.
> _libmdbx_ performs a rough estimate based only on b-tree pages that
> are common for the both stacks of cursors that were set to corresponing
> keys.
5. `mdbx_chk` tool for database integrity check.
6. Guarantee of database integrity even in asynchronous unordered write-to-disk mode.
> _libmdbx_ propose additional trade-off by implementing append-like manner for updates
> in `NOSYNC` and `MAPASYNC` modes, that avoid database corruption after a system crash
> contrary to LMDB. Nevertheless, the `MDBX_UTTERLY_NOSYNC` mode available to match LMDB behaviour,
> and for a special use-cases.
7. Automated steady flush to disk upon volume of changes and/or by
timeout via cheap polling.
8. Sequence generation and three cheap persistent 64-bit markers with ACID.
9. Support for keys and values of zero length, including multi-values
(aka sorted duplicates).
10. The handler of lack-of-space condition with a callback,
that allow you to control and resolve such situations.
11. Support for opening a database in the exclusive mode, including on a network share.
12. Extended transaction info, including dirty and leftover space info
for a write transaction, reading lag and hold over space for read
transactions.
13. Extended whole-database info (aka environment) and reader enumeration.
14. Extended update or delete, _at once_ with getting previous value
and addressing the particular item from multi-value with the same key.
15. Support for explicitly updating the existing record, not insertion a new one.
16. All cursors are uniformly, can be reused and should be closed explicitly,
regardless ones were opened within write or read transaction.
17. Correct update of current record with `MDBX_CURRENT` flag when size
of key or data was changed, including sorted duplicated.
18. Opening database handles is spared from race conditions and
pre-opening is not needed.
19. Ability to determine whether the particular data is on a dirty page
or not, that allows to avoid copy-out before updates.
20. Ability to determine whether the cursor is pointed to a key-value
pair, to the first, to the last, or not set to anything.
21. Returning `MDBX_EMULTIVAL` error in case of ambiguous update or delete.
22. On **MacOS** the `fcntl(F_FULLFSYNC)` syscall is used _by
default_ to synchronize data with the disk, as this is [the only way to
guarantee data
durability](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html)
in case of power failure. Unfortunately, in scenarios with high write
intensity, the use of `F_FULLFSYNC` significant degrades performance
compared to LMDB, where the `fsync()` syscall is used. Therefore,
_libmdbx_ allows you to override this behavior by defining the
`MDBX_OSX_SPEED_INSTEADOF_DURABILITY=1` option while build the library.
23. On **Windows** the `LockFileEx()` syscall is used for locking, since
it allows place the database on network drives, and provides protection
against incompetent user actions (aka
[poka-yoke](https://en.wikipedia.org/wiki/Poka-yoke)). Therefore
_libmdbx_ may be a little lag in performance tests from LMDB where a
named mutexes are used.
## Gotchas
1. There cannot be more than one writer at a time.
> On the other hand, this allows serialize an updates and eliminate any
> possibility of conflicts, deadlocks or logical errors.
2. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) means
relatively big [WAF](https://en.wikipedia.org/wiki/Write_amplification)
(Write Amplification Factor). Because of this syncing data to disk might
be quite resource intensive and be main performance bottleneck during
intensive write workload.
> As compromise _libmdbx_ allows several modes of lazy and/or periodic
> syncing, including `MAPASYNC` mode, which modificate data in memory and
> asynchronously syncs data to disk, moment to sync is picked by OS.
>
> Although this should be used with care, synchronous transactions in a DB
> with transaction journal will require 2 IOPS minimum (probably 3-4 in
> practice) because of filesystem overhead, overhead depends on
> filesystem, not on record count or record size. In _libmdbx_ IOPS count
> will grow logarithmically depending on record count in DB (height of B+
> tree) and will require at least 2 IOPS per transaction too.
3. [CoW](https://en.wikipedia.org/wiki/Copy-on-write) for
[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
is done on memory page level with
[B+trees](https://ru.wikipedia.org/wiki/B-%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE).
Therefore altering data requires to copy about Olog(N) memory pages,
which uses [memory bandwidth](https://en.wikipedia.org/wiki/Memory_bandwidth) and is main
performance bottleneck in `MDBX_MAPASYNC` mode.
> This is unavoidable, but isn't that bad. Syncing data to disk requires
> much more similar operations which will be done by OS, therefore this is
> noticeable only if data sync to persistent storage is fully disabled.
> _libmdbx_ allows to safely save data to persistent storage with minimal
> performance overhead. If there is no need to save data to persistent
> storage then it's much more preferable to use `std::map`.
4. Massive altering of data during a parallel long read operation will
increase the process work set, may exhaust entire free database space and
result in subsequent write performance degradation.
> _libmdbx_ mostly solve this issue by lack-of-space callback and `MDBX_LIFORECLAIM` mode.
> See [`mdbx.h`](mdbx.h) with API description for details.
> The "next" version of libmdbx (MithrilDB) will completely solve this.
5. There are no built-in checksums or digests to verify database integrity.
> The "next" version of _libmdbx_ (MithrilDB) will solve this issue employing [Merkle Tree](https://en.wikipedia.org/wiki/Merkle_tree).
--------------------------------------------------------------------------------
Usage
=====
## Source code embedding
_libmdbx_ provides two official ways for integration in source code form:
1. Using the amalgamated source code.
> The amalgamated source code includes all files requires to build and
> use _libmdbx_, but not for testing _libmdbx_ itself.
2. Adding the complete original source code as a `git submodule`.
> This allows you to build as _libmdbx_ and testing tool.
> On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool.
**_Please, avoid using any other techniques._** Otherwise, at least
don't ask for support and don't name such chimeras `libmdbx`.
The amalgamated source code could be created from original clone of git
repository on Linux by executing `make dist`. As a result, the desired
set of files will be formed in the `dist` subdirectory.
## Building
Both amalgamated and original source code provides build through the use
[CMake](https://cmake.org/) or [GNU
Make](https://www.gnu.org/software/make/) with
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)). All build ways
are completely traditional and have minimal prerequirements like
`build-essential`, i.e. the non-obsolete C/C++ compiler and a
[SDK](https://en.wikipedia.org/wiki/Software_development_kit) for the
target platform. Obviously you need building tools itself, i.e. `git`,
`cmake` or GNU `make` with `bash`.
So just use CMake or GNU Make in your habitual manner and feel free to
fill an issue or make pull request in the case something will be
unexpected or broken down.
#### DSO/DLL unloading and destructors of Thread-Local-Storage objects
When building _libmdbx_ as a shared library or use static _libmdbx_ as a
part of another dynamic library, it is advisable to make sure that your
system ensures the correctness of the call destructors of
Thread-Local-Storage objects when unloading dynamic libraries.
If this is not the case, then unloading a dynamic-link library with
_libmdbx_ code inside, can result in either a resource leak or a crash
due to calling destructors from an already unloaded DSO/DLL object. The
problem can only manifest in a multithreaded application, which makes
the unloading of shared dynamic libraries with _libmdbx_ code inside,
after using _libmdbx_. It is known that TLS-destructors are properly
maintained in the following cases:
- On all modern versions of Windows (Windows 7 and later).
- On systems with the
[`__cxa_thread_atexit_impl()`](https://sourceware.org/glibc/wiki/Destructor%20support%20for%20thread_local%20variables)
function in the standard C library, including systems with GNU libc
version 2.18 and later.
- On systems with libpthread/ntpl from GNU libc with bug fixes
[#21031](https://sourceware.org/bugzilla/show_bug.cgi?id=21031) and
[#21032](https://sourceware.org/bugzilla/show_bug.cgi?id=21032), or
where there are no similar bugs in the pthreads implementation.
### Linux and other platforms with GNU Make
To build the library it is enough to execute `make all` in the directory
of source code, and `make check` for execute the basic tests.
If the `make` installed on the system is not GNU Make, there will be a
lot of errors from make when trying to build. In this case, perhaps you
should use `gmake` instead of `make`, or even `gnu-make`, etc.
### FreeBSD and related platforms
As a rule, in such systems, the default is to use Berkeley Make. And GNU
Make is called by the gmake command or may be missing. In addition,
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) may be absent.
You need to install the required components: GNU Make, bash, C and C++
compilers compatible with GCC or CLANG. After that, to build the
library, it is enough execute `gmake all` (or `make all`) in the
directory with source code, and `gmake check` (or `make check`) to run
the basic tests.
### Windows
For build _libmdbx_ on Windows the _original_ CMake and [Microsoft Visual
Studio](https://en.wikipedia.org/wiki/Microsoft_Visual_Studio) are
recommended.
Building by MinGW, MSYS or Cygwin is potentially possible. However,
these scripts are not tested and will probably require you to modify the
CMakeLists.txt or Makefile respectively.
It should be noted that in _libmdbx_ was efforts to resolve
runtime dependencies from CRT and other libraries Visual Studio.
For this is enough define the `MDBX_AVOID_CRT` during build.
An example of running a basic test script can be found in the
[CI-script](appveyor.yml) for [AppVeyor](https://www.appveyor.com/). To
run the [long stochastic test scenario](test/long_stochastic.sh),
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) is required, and
the such testing is recommended with place the test data on the
[RAM-disk](https://en.wikipedia.org/wiki/RAM_drive).
### MacOS
Current [native build tools](https://en.wikipedia.org/wiki/Xcode) for
MacOS include GNU Make, CLANG and an outdated version of bash.
Therefore, to build the library, it is enough to run `make all` in the
directory with source code, and run `make check` to execute the base
tests. If something goes wrong, it is recommended to install
[Homebrew](https://brew.sh/) and try again.
To run the [long stochastic test scenario](test/long_stochastic.sh), you
will need to install the current (not outdated) version of
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)). To do this, we
recommend that you install [Homebrew](https://brew.sh/) and then execute
`brew install bash`.
## Bindings
| Runtime | GitHub | Author |
| -------- | ------ | ------ |
| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) |
| .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) |
--------------------------------------------------------------------------------
Performance comparison
======================
All benchmarks were done by [IOArena](https://github.com/pmwkaa/ioarena)
and multiple [scripts](https://github.com/pmwkaa/ioarena/tree/HL%2B%2B2015)
runs on Lenovo Carbon-2 laptop, i7-4600U 2.1 GHz, 8 Gb RAM,
SSD SAMSUNG MZNTD512HAGL-000L1 (DXT23L0Q) 512 Gb.
## Integral performance
Here showed sum of performance metrics in 3 benchmarks:
- Read/Search on 4 CPU cores machine;
- Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD)
operations in sync-write mode (fdatasync is called after each
transaction);
- Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD)
operations in lazy-write mode (moment to sync data to persistent storage
is decided by OS).
*Reasons why asynchronous mode isn't benchmarked here:*
1. It doesn't make sense as it has to be done with DB engines, oriented
for keeping data in memory e.g. [Tarantool](https://tarantool.io/),
[Redis](https://redis.io/)), etc.
2. Performance gap is too high to compare in any meaningful way.
![Comparison #1: Integral Performance](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-1.png)
--------------------------------------------------------------------------------
## Read Scalability
Summary performance with concurrent read/search queries in 1-2-4-8
threads on 4 CPU cores machine.
![Comparison #2: Read Scalability](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-2.png)
--------------------------------------------------------------------------------
## Sync-write mode
- Linear scale on left and dark rectangles mean arithmetic mean
transactions per second;
- Logarithmic scale on right is in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**10,000 transactions in sync-write mode**. In case of a crash all data
is consistent and state is right after last successful transaction.
[fdatasync](https://linux.die.net/man/2/fdatasync) syscall is used after
each write transaction in this mode.
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 10,000 small key-value records.
![Comparison #3: Sync-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-3.png)
--------------------------------------------------------------------------------
## Lazy-write mode
- Linear scale on left and dark rectangles mean arithmetic mean of
thousands transactions per second;
- Logarithmic scale on right in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**100,000 transactions in lazy-write mode**. In case of a crash all data
is consistent and state is right after one of last transactions, but
transactions after it will be lost. Other DB engines use
[WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) or transaction
journal for that, which in turn depends on order of operations in
journaled filesystem. _libmdbx_ doesn't use WAL and hands I/O operations
to filesystem and OS kernel (mmap).
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 100,000 small key-value
records.
![Comparison #4: Lazy-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-4.png)
--------------------------------------------------------------------------------
## Async-write mode
- Linear scale on left and dark rectangles mean arithmetic mean of
thousands transactions per second;
- Logarithmic scale on right in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**1,000,000 transactions in async-write mode**. In case of a crash all
data will be consistent and state will be right after one of last
transactions, but lost transaction count is much higher than in
lazy-write mode. All DB engines in this mode do as little writes as
possible on persistent storage. _libmdbx_ uses
[msync(MS_ASYNC)](https://linux.die.net/man/2/msync) in this mode.
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 10,000 small key-value records.
![Comparison #5: Async-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-5.png)
--------------------------------------------------------------------------------
## Cost comparison
Summary of used resources during lazy-write mode benchmarks:
- Read and write IOPS;
- Sum of user CPU time and sys CPU time;
- Used space on persistent storage after the test and closed DB, but not
waiting for the end of all internal housekeeping operations (LSM
compactification, etc).
_ForestDB_ is excluded because benchmark showed it's resource
consumption for each resource (CPU, IOPS) much higher than other engines
which prevents to meaningfully compare it with them.
All benchmark data is gathered by
[getrusage()](http://man7.org/linux/man-pages/man2/getrusage.2.html)
syscall and by scanning data directory.
![Comparison #6: Cost comparison](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-6.png)
--------------------------------------------------------------------------------
```
$ objdump -f -h -j .text libmdbx.so
libmdbx.so: file format elf64-x86-64
architecture: i386:x86-64, flags 0x00000150:
HAS_SYMS, DYNAMIC, D_PAGED
start address 0x0000000000003710
Sections:
Idx Name Size VMA LMA File off Algn
11 .text 00015eff 0000000000003710 0000000000003710 00003710 2**4
CONTENTS, ALLOC, LOAD, READONLY, CODE
```

99
src/contrib/db/libmdbx/appveyor.yml Normal file
View file

@ -0,0 +1,99 @@
version: 0.3.2.{build}
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: OFF
MDBX_AVOID_CRT: OFF
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: ON
MDBX_AVOID_CRT: ON
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: OFF
MDBX_AVOID_CRT: ON
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: ON
MDBX_AVOID_CRT: OFF
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
CMAKE_GENERATOR: Visual Studio 15 2017
TOOLSET: 141
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
CMAKE_GENERATOR: Visual Studio 14 2015
TOOLSET: 140
branches:
except:
- coverity_scan
configuration:
- Debug
- Release
platform:
- Win32
- x64
before_build:
- git clean -x -f -d
- git submodule sync
- git fetch --tags --prune
- git submodule update --init --recursive
- git submodule foreach --recursive git fetch --tags --prune
- cmake --version
build_script:
- ps: |
Write-Output "*******************************************************************************"
Write-Output "Configuration: $env:CONFIGURATION"
Write-Output "Platform: $env:PLATFORM"
Write-Output "Toolchain: $env:CMAKE_GENERATOR v$env:TOOLSET"
Write-Output "Options: MDBX_AVOID_CRT=$env:MDBX_AVOID_CRT MDBX_BUILD_SHARED_LIBRARY=$env:MDBX_BUILD_SHARED_LIBRARY"
Write-Output "*******************************************************************************"
md _build -Force | Out-Null
cd _build
$generator = $env:CMAKE_GENERATOR
if ($env:TOOLSET -lt 142) {
if ($env:PLATFORM -eq "x64") {
$generator = "$generator Win64"
}
& cmake -G "$generator" -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -D MDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY ..
} else {
& cmake -G "$generator" -A $env:PLATFORM -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -DMDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY ..
}
if ($LastExitCode -ne 0) {
throw "Exec: $ErrorMessage"
}
Write-Output "*******************************************************************************"
& cmake --build . --config $env:CONFIGURATION
if ($LastExitCode -ne 0) {
throw "Exec: $ErrorMessage"
}
Write-Output "*******************************************************************************"
test_script:
- ps: |
if (($env:PLATFORM -ne "ARM") -and ($env:PLATFORM -ne "ARM64")) {
& ./$env:CONFIGURATION/mdbx_test.exe --progress --console=no --pathname=test.db --dont-cleanup-after basic > test.log
Get-Content test.log | Select-Object -last 42
if ($LastExitCode -ne 0) {
throw "Exec: $ErrorMessage"
} else {
& ./$env:CONFIGURATION/mdbx_chk.exe -nvv test.db | Tee-Object -file chk.log | Select-Object -last 42
}
}
on_failure:
- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.log
- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.db
- ps: Push-AppveyorArtifact \projects\libmdbx\_build\chk.log

666
src/contrib/db/libmdbx/cmake/compiler.cmake Normal file
View file

@ -0,0 +1,666 @@
## Copyright (c) 2012-2019 Leonid Yuriev <leo@yuriev.ru>.
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
if (CMAKE_VERSION MATCHES ".*MSVC.*")
message(FATAL_ERROR "CMake from MSVC kit is unfit! "
"Please use the original CMake from https://cmake.org/download/")
endif()
if (NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED))
message(FATAL_ERROR "This module required C or C++ to be enabled")
endif()
include(CMakeDependentOption)
if(CMAKE_CXX_COMPILER_LOADED)
include(CheckCXXSourceRuns)
include(CheckCXXSourceCompiles)
include(CheckCXXCompilerFlag)
endif()
if(CMAKE_C_COMPILER_LOADED)
include(CheckCSourceRuns)
include(CheckCSourceCompiles)
include(CheckCCompilerFlag)
endif()
# Check if the same compile family is used for both C and CXX
if(CMAKE_C_COMPILER_LOADED AND CMAKE_CXX_COMPILER_LOADED AND
NOT (CMAKE_C_COMPILER_ID STREQUAL CMAKE_CXX_COMPILER_ID))
message(WARNING "CMAKE_C_COMPILER_ID (${CMAKE_C_COMPILER_ID}) is different "
"from CMAKE_CXX_COMPILER_ID (${CMAKE_CXX_COMPILER_ID}). "
"The final binary may be unusable.")
endif()
if(CMAKE_CXX_COMPILER_LOADED)
set(CMAKE_PRIMARY_LANG "CXX")
else()
set(CMAKE_PRIMARY_LANG "C")
endif()
macro(check_compiler_flag flag variable)
if(CMAKE_CXX_COMPILER_LOADED)
check_cxx_compiler_flag(${flag} ${variable})
else()
check_c_compiler_flag(${flag} ${variable})
endif()
endmacro(check_compiler_flag)
# We support building with Clang and gcc. First check
# what we're using for build.
if(CMAKE_C_COMPILER_LOADED AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(CMAKE_COMPILER_IS_CLANG ON)
set(CMAKE_COMPILER_IS_GNUCC OFF)
endif()
if(CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXx_COMPILER_ID STREQUAL "Clang")
set(CMAKE_COMPILER_IS_CLANG ON)
set(CMAKE_COMPILER_IS_GNUCXX OFF)
endif()
# Hard coding the compiler version is ugly from cmake POV, but
# at least gives user a friendly error message. The most critical
# demand for C++ compiler is support of C++11 lambdas, added
# only in version 4.5 https://gcc.gnu.org/projects/cxx0x.html
if(CMAKE_COMPILER_IS_GNUCC)
if(CMAKE_C_COMPILER_VERSION VERSION_LESS 4.5)
message(FATAL_ERROR "
Your GCC version is ${CMAKE_C_COMPILER_VERSION}, please update")
endif()
endif()
if(CMAKE_COMPILER_IS_GNUCXX)
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.5)
message(FATAL_ERROR "
Your G++ version is ${CMAKE_CXX_COMPILER_VERSION}, please update")
endif()
endif()
if(CMAKE_C_COMPILER_LOADED)
# Check for Elbrus lcc
execute_process(COMMAND ${CMAKE_C_COMPILER} --version
OUTPUT_VARIABLE tmp_lcc_probe_version
RESULT_VARIABLE tmp_lcc_probe_result ERROR_QUIET)
if(tmp_lcc_probe_result EQUAL 0)
string(FIND "${tmp_lcc_probe_version}" "lcc:" tmp_lcc_marker)
string(FIND "${tmp_lcc_probe_version}" ":e2k-" tmp_e2k_marker)
if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker)
execute_process(COMMAND ${CMAKE_C_COMPILER} -print-version
OUTPUT_VARIABLE CMAKE_C_COMPILER_VERSION
RESULT_VARIABLE tmp_lcc_probe_result)
set(CMAKE_COMPILER_IS_ELBRUSC ON)
set(CMAKE_C_COMPILER_ID "Elbrus")
else()
set(CMAKE_COMPILER_IS_ELBRUSC OFF)
endif()
unset(tmp_lcc_marker)
unset(tmp_e2k_marker)
endif()
unset(tmp_lcc_probe_version)
unset(tmp_lcc_probe_result)
endif()
if(CMAKE_CXX_COMPILER_LOADED)
# Check for Elbrus l++
execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version
OUTPUT_VARIABLE tmp_lxx_probe_version
RESULT_VARIABLE tmp_lxx_probe_result ERROR_QUIET)
if(tmp_lxx_probe_result EQUAL 0)
string(FIND "${tmp_lxx_probe_version}" "lcc:" tmp_lcc_marker)
string(FIND "${tmp_lxx_probe_version}" ":e2k-" tmp_e2k_marker)
if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker)
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -print-version
OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERSION
RESULT_VARIABLE tmp_lxx_probe_result)
set(CMAKE_COMPILER_IS_ELBRUSCXX ON)
set(CMAKE_CXX_COMPILER_ID "Elbrus")
else()
set(CMAKE_COMPILER_IS_ELBRUSCXX OFF)
endif()
unset(tmp_lcc_marker)
unset(tmp_e2k_marker)
endif()
unset(tmp_lxx_probe_version)
unset(tmp_lxx_probe_result)
endif()
if(CMAKE_CL_64)
set(MSVC64 1)
endif()
if(WIN32 AND CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG})
execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -dumpmachine
OUTPUT_VARIABLE __GCC_TARGET_MACHINE
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(__GCC_TARGET_MACHINE MATCHES "amd64|x86_64|AMD64")
set(MINGW64 1)
endif()
unset(__GCC_TARGET_MACHINE)
endif()
if(CMAKE_COMPILER_IS_ELBRUSC OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*")
set(E2K TRUE)
set(CMAKE_SYSTEM_ARCH "Elbrus")
elseif((MSVC64 OR MINGW64) AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(X86_64 TRUE)
set(CMAKE_SYSTEM_ARCH "x86_64")
elseif(MINGW OR (MSVC AND NOT CMAKE_CROSSCOMPILING))
set(X86_32 TRUE)
set(CMAKE_SYSTEM_ARCH "x86")
elseif(CMAKE_COMPILER_IS_ELBRUSC OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*")
set(E2K TRUE)
set(CMAKE_SYSTEM_ARCH "Elbrus")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(X86_64 TRUE)
set(CMAKE_SYSTEM_ARCH "x86_64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i686.*|i386.*|x86.*")
set(X86_32 TRUE)
set(CMAKE_SYSTEM_ARCH "x86")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|ARM64.*)" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(AARCH64 TRUE)
set(CMAKE_SYSTEM_ARCH "ARM64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)")
set(ARM32 TRUE)
set(CMAKE_SYSTEM_ARCH "ARM")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64le.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(PPC64LE TRUE)
set(CMAKE_SYSTEM_ARCH "PPC64LE")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(PPC64 TRUE)
set(CMAKE_SYSTEM_ARCH "PPC64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc).*")
set(PPC32 TRUE)
set(CMAKE_SYSTEM_ARCH "PPC")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(mips|MIPS)64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(MIPS64 TRUE)
set(CMAKE_SYSTEM_ARCH "MIPS64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(mips|MIPS).*")
set(MIPS32 TRUE)
set(CMAKE_SYSTEM_ARCH "MIPS")
endif()
if(MSVC)
check_compiler_flag("/WX" CC_HAS_WERROR)
else()
#
# GCC started to warn for unused result starting from 4.2, and
# this is when it introduced -Wno-unused-result
# GCC can also be built on top of llvm runtime (on mac).
check_compiler_flag("-Wno-unknown-pragmas" CC_HAS_WNO_UNKNOWN_PRAGMAS)
check_compiler_flag("-Wextra" CC_HAS_WEXTRA)
check_compiler_flag("-Werror" CC_HAS_WERROR)
check_compiler_flag("-fexceptions" CC_HAS_FEXCEPTIONS)
check_cxx_compiler_flag("-fcxx-exceptions" CC_HAS_FCXX_EXCEPTIONS)
check_compiler_flag("-funwind-tables" CC_HAS_FUNWIND_TABLES)
check_compiler_flag("-fno-omit-frame-pointer" CC_HAS_FNO_OMIT_FRAME_POINTER)
check_compiler_flag("-fno-common" CC_HAS_FNO_COMMON)
check_compiler_flag("-ggdb" CC_HAS_GGDB)
check_compiler_flag("-fvisibility=hidden" CC_HAS_VISIBILITY)
check_compiler_flag("-march=native" CC_HAS_ARCH_NATIVE)
check_compiler_flag("-Og" CC_HAS_DEBUG_FRENDLY_OPTIMIZATION)
check_compiler_flag("-Wall" CC_HAS_WALL)
check_compiler_flag("-Ominimal" CC_HAS_OMINIMAL)
check_compiler_flag("-ffunction-sections -fdata-sections" CC_HAS_SECTIONS)
check_compiler_flag("-ffast-math" CC_HAS_FASTMATH)
# Check for an omp support
set(CMAKE_REQUIRED_FLAGS "-fopenmp -Werror")
check_cxx_source_compiles("int main(void) {
#pragma omp parallel
return 0;
}" HAVE_OPENMP)
set(CMAKE_REQUIRED_FLAGS "")
endif()
# Check for LTO support by GCC
if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG})
unset(gcc_collect)
unset(gcc_lto_wrapper)
if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 4.7)
execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -v
OUTPUT_VARIABLE gcc_info_v ERROR_VARIABLE gcc_info_v)
string(REGEX MATCH "^(.+\nCOLLECT_GCC=)([^ \n]+)(\n.+)$" gcc_collect_valid ${gcc_info_v})
if(gcc_collect_valid)
string(REGEX REPLACE "^(.+\nCOLLECT_GCC=)([^ \n]+)(\n.+)$" "\\2" gcc_collect ${gcc_info_v})
endif()
string(REGEX MATCH "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" gcc_lto_wrapper_valid ${gcc_info_v})
if(gcc_lto_wrapper_valid)
string(REGEX REPLACE "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" "\\2" gcc_lto_wrapper ${gcc_info_v})
endif()
set(gcc_suffix "")
if(gcc_collect_valid AND gcc_collect)
string(REGEX MATCH "^(.*cc)(-.+)$" gcc_suffix_valid ${gcc_collect})
if(gcc_suffix_valid)
string(REGEX MATCH "^(.*cc)(-.+)$" "\\2" gcc_suffix ${gcc_collect})
endif()
endif()
get_filename_component(gcc_dir ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} DIRECTORY)
if(NOT CMAKE_GCC_AR)
find_program(CMAKE_GCC_AR NAMES gcc${gcc_suffix}-ar gcc-ar${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_GCC_NM)
find_program(CMAKE_GCC_NM NAMES gcc${gcc_suffix}-nm gcc-nm${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_GCC_RANLIB)
find_program(CMAKE_GCC_RANLIB NAMES gcc${gcc_suffix}-ranlib gcc-ranlib${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH)
endif()
unset(gcc_dir)
unset(gcc_suffix_valid)
unset(gcc_suffix)
unset(gcc_lto_wrapper_valid)
unset(gcc_collect_valid)
unset(gcc_collect)
unset(gcc_info_v)
endif()
if(CMAKE_GCC_AR AND CMAKE_GCC_NM AND CMAKE_GCC_RANLIB AND gcc_lto_wrapper)
message(STATUS "Found GCC's LTO toolset: ${gcc_lto_wrapper}, ${CMAKE_GCC_AR}, ${CMAKE_GCC_RANLIB}")
set(GCC_LTO_CFLAGS "-flto -fno-fat-lto-objects -fuse-linker-plugin")
set(GCC_LTO_AVAILABLE TRUE)
message(STATUS "Link-Time Optimization by GCC is available")
else()
set(GCC_LTO_AVAILABLE FALSE)
message(STATUS "Link-Time Optimization by GCC is NOT available")
endif()
unset(gcc_lto_wrapper)
endif()
# check for LTO by MSVC
if(MSVC)
if(NOT MSVC_VERSION LESS 1600)
set(MSVC_LTO_AVAILABLE TRUE)
message(STATUS "Link-Time Optimization by MSVC is available")
else()
set(MSVC_LTO_AVAILABLE FALSE)
message(STATUS "Link-Time Optimization by MSVC is NOT available")
endif()
endif()
# Check for LTO support by CLANG
if(CMAKE_COMPILER_IS_CLANG)
if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 3.5)
execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -print-search-dirs
OUTPUT_VARIABLE clang_search_dirs)
unset(clang_bindir)
unset(clang_libdir)
string(REGEX MATCH "^(.*programs: =)([^:]*:)*([^:]+/llvm[-.0-9]+/bin[^:]*)(:[^:]*)*(\n.+)$" clang_bindir_valid ${clang_search_dirs})
if(clang_bindir_valid)
string(REGEX REPLACE "^(.*programs: =)([^:]*:)*([^:]+/llvm[-.0-9]+/bin[^:]*)(:[^:]*)*(\n.+)$" "\\3" clang_bindir ${clang_search_dirs})
get_filename_component(clang_libdir "${clang_bindir}/../lib" REALPATH)
if(clang_libdir)
message(STATUS "Found CLANG/LLVM directories: ${clang_bindir}, ${clang_libdir}")
endif()
endif()
if(NOT (clang_bindir AND clang_libdir))
message(STATUS "Could NOT find CLANG/LLVM directories (bin and/or lib).")
endif()
if(NOT CMAKE_CLANG_LD AND clang_bindir)
find_program(CMAKE_CLANG_LD NAMES llvm-link link llvm-ld ld PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_CLANG_AR AND clang_bindir)
find_program(CMAKE_CLANG_AR NAMES llvm-ar ar PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_CLANG_NM AND clang_bindir)
find_program(CMAKE_CLANG_NM NAMES llvm-nm nm PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_CLANG_RANLIB AND clang_bindir)
find_program(CMAKE_CLANG_RANLIB NAMES llvm-ranlib ranlib PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
set(clang_lto_plugin_name "LLVMgold${CMAKE_SHARED_LIBRARY_SUFFIX}")
if(NOT CMAKE_LD_GOLD AND clang_bindir)
find_program(CMAKE_LD_GOLD NAMES ld.gold PATHS)
endif()
if(NOT CLANG_LTO_PLUGIN AND clang_libdir)
find_file(CLANG_LTO_PLUGIN ${clang_lto_plugin_name} PATH ${clang_libdir} NO_DEFAULT_PATH)
endif()
if(CLANG_LTO_PLUGIN)
message(STATUS "Found CLANG/LLVM's plugin for LTO: ${CLANG_LTO_PLUGIN}")
else()
message(STATUS "Could NOT find CLANG/LLVM's plugin (${clang_lto_plugin_name}) for LTO.")
endif()
if(CMAKE_CLANG_LD AND CMAKE_CLANG_AR AND CMAKE_CLANG_NM AND CMAKE_CLANG_RANLIB)
message(STATUS "Found CLANG/LLVM's binutils for LTO: ${CMAKE_CLANG_AR}, ${CMAKE_CLANG_RANLIB}")
else()
message(STATUS "Could NOT find CLANG/LLVM's binutils (ar, ranlib, nm) for LTO.")
endif()
unset(clang_lto_plugin_name)
unset(clang_libdir)
unset(clang_bindir_valid)
unset(clang_bindir)
unset(clang_search_dirs)
endif()
if((CLANG_LTO_PLUGIN AND CMAKE_LD_GOLD) AND
(CMAKE_CLANG_LD AND CMAKE_CLANG_AR AND CMAKE_CLANG_NM AND CMAKE_CLANG_RANLIB))
set(CLANG_LTO_AVAILABLE TRUE)
message(STATUS "Link-Time Optimization by CLANG/LLVM is available")
elseif(CMAKE_TOOLCHAIN_FILE AND NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 7.0)
set(CLANG_LTO_AVAILABLE TRUE)
if (NOT CMAKE_CLANG_AR)
set(CMAKE_CLANG_AR ${CMAKE_AR})
endif()
if (NOT CMAKE_CLANG_NM)
set(CMAKE_CLANG_NM ${CMAKE_NM})
endif()
if (NOT CMAKE_CLANG_RANLIB)
set(CMAKE_CLANG_RANLIB ${CMAKE_RANLIB })
endif()
message(STATUS "Assume Link-Time Optimization by CLANG/LLVM is available via ${CMAKE_TOOLCHAIN_FILE}")
else()
set(CLANG_LTO_AVAILABLE FALSE)
message(STATUS "Link-Time Optimization by CLANG/LLVM is NOT available")
endif()
endif()
# Perform build type specific configuration.
option(ENABLE_BACKTRACE "Enable output of fiber backtrace information in 'show
fiber' administrative command. Only works on x86 architectures, if compiled
with gcc. If GNU binutils and binutils-dev libraries are installed, backtrace
is output with resolved function (symbol) names. Otherwise only frame
addresses are printed." OFF)
set(HAVE_BFD False)
if(ENABLE_BACKTRACE)
if(NOT (X86_32 OR X86_64) OR NOT CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG})
# We only know this option to work with gcc
message(FATAL_ERROR "ENABLE_BACKTRACE option is set but the system
is not x86 based (${CMAKE_SYSTEM_PROCESSOR}) or the compiler
is not GNU GCC (${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER}).")
endif()
# Use GNU bfd if present.
find_library(BFD_LIBRARY NAMES libbfd.a)
if(BFD_LIBRARY)
check_library_exists(${BFD_LIBRARY} bfd_init "" HAVE_BFD_LIB)
endif()
find_library(IBERTY_LIBRARY NAMES libiberty.a)
if(IBERTY_LIBRARY)
check_library_exists(${IBERTY_LIBRARY} cplus_demangle "" HAVE_IBERTY_LIB)
endif()
set(CMAKE_REQUIRED_DEFINITIONS -DPACKAGE=${PACKAGE} -DPACKAGE_VERSION=${PACKAGE_VERSION})
check_include_files(bfd.h HAVE_BFD_H)
set(CMAKE_REQUIRED_DEFINITIONS)
find_package(ZLIB)
if(HAVE_BFD_LIB AND HAVE_BFD_H AND HAVE_IBERTY_LIB AND ZLIB_FOUND)
set(HAVE_BFD ON)
set(BFD_LIBRARIES ${BFD_LIBRARY} ${IBERTY_LIBRARY} ${ZLIB_LIBRARIES})
find_package_message(BFD_LIBRARIES "Found libbfd and dependencies"
${BFD_LIBRARIES})
if(TARGET_OS_FREEBSD AND NOT TARGET_OS_DEBIAN_FREEBSD)
set(BFD_LIBRARIES ${BFD_LIBRARIES} iconv)
endif()
endif()
endif()
macro(setup_compile_flags)
# LY: save initial C/CXX flags
if(NOT INITIAL_CMAKE_FLAGS_SAVED)
if(MSVC)
string(REGEX REPLACE "^(.*)(/EHsc)( *)(.*)$" "\\1/EHs\\3\\4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
set(INITIAL_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_C_FLAGS ${CMAKE_C_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_STATIC_LINKER_FLAGS ${CMAKE_STATIC_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_FLAGS_SAVED TRUE CACHE INTERNAL "State of initial CMake's flags" FORCE)
endif()
# LY: reset C/CXX flags
set(CXX_FLAGS ${INITIAL_CMAKE_CXX_FLAGS})
set(C_FLAGS ${INITIAL_CMAKE_C_FLAGS})
set(EXE_LINKER_FLAGS ${INITIAL_CMAKE_EXE_LINKER_FLAGS})
set(SHARED_LINKER_FLAGS ${INITIAL_CMAKE_SHARED_LINKER_FLAGS})
set(STATIC_LINKER_FLAGS ${INITIAL_CMAKE_STATIC_LINKER_FLAGS})
set(MODULE_LINKER_FLAGS ${INITIAL_CMAKE_MODULE_LINKER_FLAGS})
if(CC_HAS_FEXCEPTIONS)
add_compile_flags("C;CXX" "-fexceptions")
endif()
if(CC_HAS_FCXX_EXCEPTIONS)
add_compile_flags("CXX" "-fcxx-exceptions -frtti")
endif()
# In C a global variable without a storage specifier (static/extern) and
# without an initialiser is called a ’tentative definition’. The
# language permits multiple tentative definitions in the single
# translation unit; i.e. int foo; int foo; is perfectly ok. GNU
# toolchain goes even further, allowing multiple tentative definitions
# in *different* translation units. Internally, variables introduced via
# tentative definitions are implemented as ‘common’ symbols. Linker
# permits multiple definitions if they are common symbols, and it picks
# one arbitrarily for inclusion in the binary being linked.
#
# -fno-common forces GNU toolchain to behave in a more
# standard-conformant way in respect to tentative definitions and it
# prevents common symbols generation. Since we are a cross-platform
# project it really makes sense. There are toolchains that don’t
# implement GNU style handling of the tentative definitions and there
# are platforms lacking proper support for common symbols (osx).
if(CC_HAS_FNO_COMMON)
add_compile_flags("C;CXX" "-fno-common")
endif()
if(CC_HAS_GGDB)
add_compile_flags("C;CXX" "-ggdb")
endif()
if(CC_HAS_WNO_UNKNOWN_PRAGMAS AND NOT HAVE_OPENMP)
add_compile_flags("C;CXX" -Wno-unknown-pragmas)
endif()
if(CC_HAS_SECTIONS)
add_compile_flags("C;CXX" -ffunction-sections -fdata-sections)
elseif(MSVC)
add_compile_flags("C;CXX" /Gy)
endif()
# We must set -fno-omit-frame-pointer here, since we rely
# on frame pointer when getting a backtrace, and it must
# be used consistently across all object files.
# The same reasoning applies to -fno-stack-protector switch.
if(ENABLE_BACKTRACE)
if(CC_HAS_FNO_OMIT_FRAME_POINTER)
add_compile_flags("C;CXX" "-fno-omit-frame-pointer")
endif()
endif()
if(MSVC)
if (MSVC_VERSION LESS 1900)
message(FATAL_ERROR "At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.")
endif()
add_compile_flags("CXX" "/Zc:__cplusplus")
add_compile_flags("C;CXX" "/W4")
add_compile_flags("C;CXX" "/utf-8")
else()
if(CC_HAS_WALL)
add_compile_flags("C;CXX" "-Wall")
endif()
if(CC_HAS_WEXTRA)
add_compile_flags("C;CXX" "-Wextra")
endif()
endif()
if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}
AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 5)
# G++ bug. http://gcc.gnu.org/bugzilla/show_bug.cgi?id=31488
add_compile_flags("CXX" "-Wno-invalid-offsetof")
endif()
add_definitions("-D__STDC_FORMAT_MACROS=1")
add_definitions("-D__STDC_LIMIT_MACROS=1")
add_definitions("-D__STDC_CONSTANT_MACROS=1")
add_definitions("-D_HAS_EXCEPTIONS=1")
# Only add -Werror if it's a debug build, done by developers.
# Release builds should not cause extra trouble.
if(CC_HAS_WERROR AND (CI OR CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE STREQUAL "Debug"))
if(MSVC)
add_compile_flags("C;CXX" "/WX")
elseif(CMAKE_COMPILER_IS_CLANG)
if (NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 6)
add_compile_flags("C;CXX" "-Werror")
endif()
elseif(CMAKE_COMPILER_IS_GNUCC)
if (NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 6)
add_compile_flags("C;CXX" "-Werror")
endif()
else()
add_compile_flags("C;CXX" "-Werror")
endif()
endif()
if(HAVE_OPENMP)
add_compile_flags("C;CXX" "-fopenmp")
endif()
if (ENABLE_ASAN)
add_compile_flags("C;CXX" -fsanitize=address)
endif()
if(ENABLE_GCOV)
if(NOT HAVE_GCOV)
message(FATAL_ERROR
"ENABLE_GCOV option requested but gcov library is not found")
endif()
add_compile_flags("C;CXX" "-fprofile-arcs" "-ftest-coverage")
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
# add_library(gcov SHARED IMPORTED)
endif()
if(ENABLE_GPROF)
add_compile_flags("C;CXX" "-pg")
endif()
if(CMAKE_COMPILER_IS_GNUCC AND LTO_ENABLED)
add_compile_flags("C;CXX" ${GCC_LTO_CFLAGS})
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm -fwhole-program")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)
# Pass the same optimization flags to the linker
set(compile_flags "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}}")
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${compile_flags}")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${compile_flags}")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${compile_flags}")
unset(compile_flags)
else()
add_compile_flags("CXX" "-flto-odr-type-merging")
endif()
endif()
if(MSVC AND LTO_ENABLED)
add_compile_flags("C;CXX" "/GL")
foreach(linkmode IN ITEMS EXE SHARED STATIC MODULE)
set(${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS} /LTCG")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:NO *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:YES *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}")
string(STRIP "${${linkmode}_LINKER_FLAGS}" ${linkmode}_LINKER_FLAGS)
foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo Debug)
string(TOUPPER "${config}" config_uppercase)
if(DEFINED "CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:NO *)(.*)$" "\\1\\3" altered_flags "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:YES *)(.*)$" "\\1\\3" altered_flags "${altered_flags}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL *)(.*)$" "\\1\\3" altered_flags "${altered_flags}")
string(STRIP "${altered_flags}" altered_flags)
if(NOT "${altered_flags}" STREQUAL "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}")
set(CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/INCREMENTAL' removed for LTO" FORCE)
endif()
endif()
endforeach(config)
endforeach(linkmode)
unset(linkmode)
foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo)
foreach(lang IN ITEMS C CXX)
string(TOUPPER "${config}" config_uppercase)
if(DEFINED "CMAKE_${lang}_FLAGS_${config_uppercase}")
string(REPLACE "/O2" "/Ox" altered_flags "${CMAKE_${lang}_FLAGS_${config_uppercase}}")
if(NOT "${altered_flags}" STREQUAL "${CMAKE_${lang}_FLAGS_${config_uppercase}}")
set(CMAKE_${lang}_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/O2' replaced by '/Ox' for LTO" FORCE)
endif()
endif()
unset(config_uppercase)
endforeach(lang)
endforeach(config)
unset(altered_flags)
unset(lang)
unset(config)
endif()
if(CMAKE_COMPILER_IS_CLANG AND OSX_ARCHITECTURES)
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} -Wl,-keep_dwarf_unwind")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} -Wl,-keep_dwarf_unwind")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} -Wl,-keep_dwarf_unwind")
endif()
if(CMAKE_COMPILER_IS_CLANG AND LTO_ENABLED)
if(CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 3.9)
set(CLANG_LTO_FLAG "-flto")
else()
set(CLANG_LTO_FLAG "-flto=thin")
endif()
add_compile_flags("C;CXX" ${CLANG_LTO_FLAG})
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm -fwhole-program")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm")
endif()
# LY: push C/CXX flags into the cache
set(CMAKE_CXX_FLAGS ${CXX_FLAGS} CACHE STRING "Flags used by the C++ compiler during all build types" FORCE)
set(CMAKE_C_FLAGS ${C_FLAGS} CACHE STRING "Flags used by the C compiler during all build types" FORCE)
set(CMAKE_EXE_LINKER_FLAGS ${EXE_LINKER_FLAGS} CACHE STRING "Flags used by the linker" FORCE)
set(CMAKE_SHARED_LINKER_FLAGS ${SHARED_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of dll's" FORCE)
set(CMAKE_STATIC_LINKER_FLAGS ${STATIC_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of static libraries" FORCE)
set(CMAKE_MODULE_LINKER_FLAGS ${MODULE_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of modules" FORCE)
unset(CXX_FLAGS)
unset(C_FLAGS)
unset(EXE_LINKER_FLAGS)
unset(SHARED_LINKER_FLAGS)
unset(STATIC_LINKER_FLAGS)
unset(MODULE_LINKER_FLAGS)
endmacro(setup_compile_flags)
# determine library for for std::filesystem
set(LIBCXX_FILESYSTEM "")
if(CMAKE_COMPILER_IS_GNUCXX)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
set(LIBCXX_FILESYSTEM "stdc++fs")
endif()
elseif (CMAKE_COMPILER_IS_CLANG)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
set(LIBCXX_FILESYSTEM "c++experimental")
else()
set(LIBCXX_FILESYSTEM "stdc++fs")
endif()
endif()
cmake_policy(POP)

45
src/contrib/db/libmdbx/cmake/profile.cmake Normal file
View file

@ -0,0 +1,45 @@
## Copyright (c) 2012-2019 Leonid Yuriev <leo@yuriev.ru>.
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
include(CheckLibraryExists)
check_library_exists(gcov __gcov_flush "" HAVE_GCOV)
option(ENABLE_GCOV
"Enable integration with gcov, a code coverage program" OFF)
option(ENABLE_GPROF
"Enable integration with gprof, a performance analyzing tool" OFF)
if(CMAKE_CXX_COMPILER_LOADED)
include(CheckIncludeFileCXX)
check_include_file_cxx(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H)
else()
include(CheckIncludeFile)
check_include_file(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H)
endif()
option(MDBX_USE_VALGRIND "Enable integration with valgrind, a memory analyzing tool" OFF)
if(MDBX_USE_VALGRIND AND NOT HAVE_VALGRIND_MEMCHECK_H)
message(FATAL_ERROR "MDBX_USE_VALGRIND option is set but valgrind/memcheck.h is not found")
endif()
option(ENABLE_ASAN
"Enable AddressSanitizer, a fast memory error detector based on compiler instrumentation" OFF)
cmake_policy(POP)

183
src/contrib/db/libmdbx/cmake/utils.cmake Normal file
View file

@ -0,0 +1,183 @@
## Copyright (c) 2012-2019 Leonid Yuriev <leo@yuriev.ru>.
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
macro(add_compile_flags langs)
foreach(_lang ${langs})
string(REPLACE ";" " " _flags "${ARGN}")
if(CMAKE_CXX_COMPILER_LOADED AND _lang STREQUAL "CXX")
set("${_lang}_FLAGS" "${${_lang}_FLAGS} ${_flags}")
endif()
if(CMAKE_C_COMPILER_LOADED AND _lang STREQUAL "C")
set("${_lang}_FLAGS" "${${_lang}_FLAGS} ${_flags}")
endif()
endforeach()
unset(_lang)
unset(_flags)
endmacro(add_compile_flags)
macro(set_source_files_compile_flags)
foreach(file ${ARGN})
get_filename_component(_file_ext ${file} EXT)
set(_lang "")
if("${_file_ext}" STREQUAL ".m")
set(_lang OBJC)
# CMake believes that Objective C is a flavor of C++, not C,
# and uses g++ compiler for .m files.
# LANGUAGE property forces CMake to use CC for ${file}
set_source_files_properties(${file} PROPERTIES LANGUAGE C)
elseif("${_file_ext}" STREQUAL ".mm")
set(_lang OBJCXX)
endif()
if(_lang)
get_source_file_property(_flags ${file} COMPILE_FLAGS)
if("${_flags}" STREQUAL "NOTFOUND")
set(_flags "${CMAKE_${_lang}_FLAGS}")
else()
set(_flags "${_flags} ${CMAKE_${_lang}_FLAGS}")
endif()
# message(STATUS "Set (${file} ${_flags}")
set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS
"${_flags}")
endif()
endforeach()
unset(_file_ext)
unset(_lang)
endmacro(set_source_files_compile_flags)
macro(fetch_version name version_file)
set(${name}_VERSION "")
set(${name}_GIT_DESCRIBE "")
set(${name}_GIT_TIMESTAMP "")
set(${name}_GIT_TREE "")
set(${name}_GIT_COMMIT "")
set(${name}_GIT_REVISION 0)
set(${name}_GIT_VERSION "")
if(GIT)
execute_process(COMMAND ${GIT} describe --tags --long --dirty=-dirty
OUTPUT_VARIABLE ${name}_GIT_DESCRIBE
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_DESCRIBE" STREQUAL "")
message(FATAL_ERROR "Please fetch tags and/or install latest version of git ('describe --tags --long --dirty' failed)")
endif()
execute_process(COMMAND ${GIT} show --no-patch --format=%cI HEAD
OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%cI")
execute_process(COMMAND ${GIT} show --no-patch --format=%ci HEAD
OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%ci")
message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%cI HEAD' failed)")
endif()
endif()
execute_process(COMMAND ${GIT} show --no-patch --format=%T HEAD
OUTPUT_VARIABLE ${name}_GIT_TREE
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_TREE" STREQUAL "")
message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%T HEAD' failed)")
endif()
execute_process(COMMAND ${GIT} show --no-patch --format=%H HEAD
OUTPUT_VARIABLE ${name}_GIT_COMMIT
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_COMMIT" STREQUAL "")
message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%H HEAD' failed)")
endif()
execute_process(COMMAND ${GIT} rev-list --count --no-merges HEAD
OUTPUT_VARIABLE ${name}_GIT_REVISION
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_REVISION" STREQUAL "")
message(FATAL_ERROR "Please install latest version of git ('rev-list --count --no-merges HEAD' failed)")
endif()
string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}")
if(git_version_valid)
string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;\\4" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE})
else()
string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}")
if(git_version_valid)
string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;0" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE})
else()
message(AUTHOR_WARNING "Bad ${name} version \"${${name}_GIT_DESCRIBE}\"; falling back to 0.0.0 (have you made an initial release?)")
set(${name}_GIT_VERSION "0;0;0")
endif()
endif()
endif()
if(NOT ${name}_GIT_VERSION OR NOT ${name}_GIT_TIMESTAMP OR NOT ${name}_GIT_REVISION)
message(WARNING "Unable to retrive ${name} version from git.")
set(${name}_GIT_VERSION "0;0;0;0")
set(${name}_GIT_TIMESTAMP "")
set(${name}_GIT_REVISION 0)
# Try to get version from VERSION file
if(EXISTS "${version_file}")
file(STRINGS "${version_file}" ${name}_VERSION)
endif()
if(NOT ${name}_VERSION)
message(WARNING "Unable to retrive ${name} version from \"${version_file}\" file.")
set(${name}_VERSION_LIST ${${name}_GIT_VERSION})
string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}")
else()
string(REPLACE "." ";" ${name}_VERSION_LIST ${${name}_VERSION})
endif()
else()
list(APPEND ${name}_GIT_VERSION ${${name}_GIT_REVISION})
set(${name}_VERSION_LIST ${${name}_GIT_VERSION})
string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}")
endif()
list(GET ${name}_VERSION_LIST 0 "${name}_VERSION_MAJOR")
list(GET ${name}_VERSION_LIST 1 "${name}_VERSION_MINOR")
list(GET ${name}_VERSION_LIST 2 "${name}_VERSION_RELEASE")
list(GET ${name}_VERSION_LIST 3 "${name}_VERSION_REVISION")
set(${name}_VERSION_MAJOR ${${name}_VERSION_MAJOR} PARENT_SCOPE)
set(${name}_VERSION_MINOR ${${name}_VERSION_MINOR} PARENT_SCOPE)
set(${name}_VERSION_RELEASE ${${name}_VERSION_RELEASE} PARENT_SCOPE)
set(${name}_VERSION_REVISION ${${name}_VERSION_REVISION} PARENT_SCOPE)
set(${name}_VERSION ${${name}_VERSION} PARENT_SCOPE)
set(${name}_GIT_DESCRIBE ${${name}_GIT_DESCRIBE} PARENT_SCOPE)
set(${name}_GIT_TIMESTAMP ${${name}_GIT_TIMESTAMP} PARENT_SCOPE)
set(${name}_GIT_TREE ${${name}_GIT_TREE} PARENT_SCOPE)
set(${name}_GIT_COMMIT ${${name}_GIT_COMMIT} PARENT_SCOPE)
set(${name}_GIT_REVISION ${${name}_GIT_REVISION} PARENT_SCOPE)
set(${name}_GIT_VERSION ${${name}_GIT_VERSION} PARENT_SCOPE)
endmacro(fetch_version)
cmake_policy(POP)

6
src/contrib/db/libmdbx/example/CMakeLists.txt Normal file
View file

@ -0,0 +1,6 @@
set(TARGET mdbx_example)
project(${TARGET})
add_executable(${TARGET} example-mdbx.c)
target_link_libraries(${TARGET} mdbx)

1
src/contrib/db/libmdbx/example/README.md Normal file
View file

@ -0,0 +1 @@
See [example-mdbx.c](example-mdbx.c) as an example of using _libmdbx_, and do a line-by-line comparison of it with the [sample-bdb.txt](sample-bdb.txt) file.

112
src/contrib/db/libmdbx/example/example-mdbx.c Normal file
View file

@ -0,0 +1,112 @@
/* MDBX usage examle
*
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2017 Ilya Shipitsin <chipitsine@gmail.com>.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "mdbx.h"
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char *argv[]) {
(void)argc;
(void)argv;
int rc;
MDBX_env *env = NULL;
MDBX_dbi dbi = 0;
MDBX_val key, data;
MDBX_txn *txn = NULL;
MDBX_cursor *cursor = NULL;
char sval[32];
rc = mdbx_env_create(&env);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_open(env, "./example-db",
MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_dbi_open(txn, NULL, 0, &dbi);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
key.iov_len = sizeof(int);
key.iov_base = sval;
data.iov_len = sizeof(sval);
data.iov_base = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = mdbx_put(txn, dbi, &key, &data, 0);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
txn = NULL;
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_cursor_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
int found = 0;
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len,
(char *)key.iov_base, data.iov_base, (int)data.iov_len,
(char *)data.iov_base);
found += 1;
}
if (rc != MDBX_NOTFOUND || found == 0) {
fprintf(stderr, "mdbx_cursor_get: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
} else {
rc = MDBX_SUCCESS;
}
bailout:
if (cursor)
mdbx_cursor_close(cursor);
if (txn)
mdbx_txn_abort(txn);
if (dbi)
mdbx_dbi_close(env, dbi);
if (env)
mdbx_env_close(env);
return (rc != MDBX_SUCCESS) ? EXIT_FAILURE : EXIT_SUCCESS;
}

77
src/contrib/db/libmdbx/example/sample-bdb.txt Normal file
View file

@ -0,0 +1,77 @@
/* BerkeleyDB toy/sample
*
* Do a line-by-line comparison of this and example-mdbx.c
*/
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <string.h>
#include <db.h>
int main(int argc,char * argv[])
{
int rc;
DB_ENV *env;
DB *dbi;
DBT key, data;
DB_TXN *txn;
DBC *cursor;
char sval[32], kval[32];
/* Note: Most error checking omitted for simplicity */
#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD)
rc = db_env_create(&env, 0);
rc = env->open(env, "./testdb", FLAGS, 0664);
rc = db_create(&dbi, env, 0);
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664);
memset(&key, 0, sizeof(DBT));
memset(&data, 0, sizeof(DBT));
key.size = sizeof(int);
key.data = sval;
data.size = sizeof(sval);
data.data = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = dbi->put(dbi, txn, &key, &data, 0);
rc = txn->commit(txn, 0);
if (rc) {
fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc));
goto leave;
}
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->cursor(dbi, txn, &cursor, 0);
key.flags = DB_DBT_USERMEM;
key.data = kval;
key.ulen = sizeof(kval);
data.flags = DB_DBT_USERMEM;
data.data = sval;
data.ulen = sizeof(sval);
while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.data, (int) key.size, (char *) key.data,
data.data, (int) data.size, (char *) data.data);
}
rc = cursor->c_close(cursor);
rc = txn->abort(txn);
leave:
rc = dbi->close(dbi, 0);
rc = env->close(env, 0);
return rc;
}

3497
src/contrib/db/libmdbx/mdbx.h Normal file
View file

File diff suppressed because it is too large Load diff

184
src/contrib/db/libmdbx/packages/rpm/CMakeLists.txt Normal file
View file

@ -0,0 +1,184 @@
cmake_minimum_required(VERSION 2.8.7)
set(TARGET mdbx)
project(${TARGET})
set(MDBX_VERSION_MAJOR 0)
set(MDBX_VERSION_MINOR 3)
set(MDBX_VERSION_RELEASE 1)
set(MDBX_VERSION_REVISION 0)
set(MDBX_VERSION_STRING ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE})
enable_language(C)
enable_language(CXX)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED on)
add_definitions(-DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1 -D_GNU_SOURCE=1)
find_package(Threads REQUIRED)
get_directory_property(hasParent PARENT_DIRECTORY)
if(hasParent)
set(STANDALONE_BUILD 0)
else()
set(STANDALONE_BUILD 1)
enable_testing()
if (CMAKE_C_COMPILER_ID MATCHES GNU)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g3")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wextra")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread")
endif()
if (CMAKE_CXX_COMPILER_ID MATCHES GNU)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpointer-arith")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-security")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Woverloaded-virtual")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wwrite-strings")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=20")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wunused-function -Wunused-variable -Wunused-value -Wmissing-declarations")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-qual")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finline-functions-called-once")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat")
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g3")
endif()
if (COVERAGE)
if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
message(FATAL_ERROR "Coverage requires -DCMAKE_BUILD_TYPE=Debug Current value=${CMAKE_BUILD_TYPE}")
endif()
message(STATUS "Setting coverage compiler flags")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
add_definitions(-DCOVERAGE_TEST)
endif()
if (NOT TRAVIS)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fstack-protector-strong -static-libasan")
endif()
endif()
set(${TARGET}_SRC
mdbx.h
src/bits.h
src/defs.h
src/lck-linux.c
src/mdbx.c
src/osal.c
src/osal.h
src/version.c
)
add_library(${TARGET}_STATIC STATIC
${${TARGET}_SRC}
)
add_library(${TARGET} ALIAS ${TARGET}_STATIC)
add_library(${TARGET}_SHARED SHARED
${${TARGET}_SRC}
)
set_target_properties(${TARGET}_SHARED PROPERTIES
VERSION ${MDBX_VERSION_STRING}
SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
OUTPUT_NAME ${TARGET}
CLEAN_DIRECT_OUTPUT 1
)
set_target_properties(${TARGET}_STATIC PROPERTIES
VERSION ${MDBX_VERSION_STRING}
SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
OUTPUT_NAME ${TARGET}
CLEAN_DIRECT_OUTPUT 1
)
target_include_directories(${TARGET}_STATIC PUBLIC
${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(${TARGET}_SHARED PUBLIC
${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(${TARGET}_STATIC ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(${TARGET}_SHARED ${CMAKE_THREAD_LIBS_INIT})
if(UNIX AND NOT APPLE)
target_link_libraries(${TARGET}_STATIC rt)
target_link_libraries(${TARGET}_SHARED rt)
endif()
install(TARGETS ${TARGET}_STATIC DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
install(TARGETS ${TARGET}_SHARED DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
install(FILES mdbx.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT mdbx-devel)
add_subdirectory(src/tools)
add_subdirectory(test)
add_subdirectory(test/pcrf)
add_subdirectory(tutorial)
##############################################################################
set(CPACK_GENERATOR "RPM")
set(CPACK_RPM_COMPONENT_INSTALL ON)
# Version
if (NOT "$ENV{BUILD_NUMBER}" STREQUAL "")
set(CPACK_PACKAGE_RELEASE $ENV{BUILD_NUMBER})
else()
if (NOT "$ENV{CI_PIPELINE_ID}" STREQUAL "")
set(CPACK_PACKAGE_RELEASE $ENV{CI_PIPELINE_ID})
else()
set(CPACK_PACKAGE_RELEASE 1)
endif()
endif()
set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE})
set(CPACK_PACKAGE_VERSION ${MDBX_VERSION_STRING})
set(CPACK_PACKAGE_VERSION_FULL ${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE})
set(CPACK_RPM_mdbx-devel_PACKAGE_REQUIRES "mdbx = ${CPACK_PACKAGE_VERSION}")
set(CPACK_RPM_SPEC_INSTALL_POST "/bin/true")
set(CPACK_RPM_mdbx_PACKAGE_NAME mdbx)
set(CPACK_RPM_mdbx-devel_PACKAGE_NAME mdbx-devel)
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "The revised and extended descendant of Symas LMDB")
set(CPACK_PACKAGE_VENDOR "???")
set(CPACK_PACKAGE_CONTACT "Vladimir Romanov")
set(CPACK_PACKAGE_RELOCATABLE false)
set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
set(CPACK_RPM_PACKAGE_REQUIRES "")
set(CPACK_RPM_PACKAGE_GROUP "Applications/Database")
set(CPACK_RPM_mdbx_FILE_NAME "${CPACK_RPM_mdbx_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
set(CPACK_RPM_mdbx-devel_FILE_NAME "${CPACK_RPM_mdbx-devel_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/usr/local
/usr/local/bin
/usr/local/lib64
/usr/local/include
/usr/local/man
/usr/local/man/man1
)
include(CPack)

18
src/contrib/db/libmdbx/packages/rpm/build.sh Normal file
View file

@ -0,0 +1,18 @@
#!/bin/bash
set -e
CONFIG=$1
if [[ -z "${CONFIG}" ]]; then
CONFIG=Debug
fi
if [[ -r /opt/rh/devtoolset-6/enable ]]; then
source /opt/rh/devtoolset-6/enable
fi
#rm -f -r build || true
mkdir -p cmake-build-${CONFIG}
pushd cmake-build-${CONFIG} &> /dev/null
if [[ ! -r Makefile ]]; then
cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
fi
make -j8 || exit 1
popd &> /dev/null

25
src/contrib/db/libmdbx/packages/rpm/package.sh Normal file
View file

@ -0,0 +1,25 @@
#!/bin/bash
set -e
CONFIG=$1
if [[ -z "${CONFIG}" ]]; then
CONFIG=Debug
fi
DIRNAME=`dirname ${BASH_SOURCE[0]}`
DIRNAME=`readlink --canonicalize ${DIRNAME}`
if [[ -r /opt/rh/devtoolset-6/enable ]]; then
source /opt/rh/devtoolset-6/enable
fi
mkdir -p cmake-build-${CONFIG}
pushd cmake-build-${CONFIG} &> /dev/null
if [[ ! -r Makefile ]]; then
cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
fi
rm -f *.rpm
make -j8 package || exit 1
rm -f *-Unspecified.rpm
popd &> /dev/null

225
src/contrib/db/libmdbx/src/CMakeLists.txt Normal file
View file

@ -0,0 +1,225 @@
##
## Copyright 2019 Leonid Yuriev <leo@yuriev.ru>
## and other libmdbx authors: please see AUTHORS file.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
##
# Get version
fetch_version(MDBX "${CMAKE_CURRENT_SOURCE_DIR}/../VERSION")
message(STATUS "libmdbx version is ${MDBX_VERSION}")
if(MDBX_ALLOY_MODE)
set(LIBMDBX_SOURCES alloy.c)
else()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
set(LIBMDBX_OSAL windows)
else()
set(LIBMDBX_OSAL posix)
endif()
set(LIBMDBX_SOURCES
elements/defs.h elements/internals.h elements/osal.h
elements/core.c elements/osal.c elements/lck-${LIBMDBX_OSAL}.c)
endif()
list(APPEND LIBMDBX_SOURCES ../mdbx.h
"${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c"
"${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h")
if(MDBX_BUILD_SHARED_LIBRARY)
add_library(mdbx SHARED ${LIBMDBX_SOURCES})
target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS INTERFACE LIBMDBX_IMPORTS)
set(MDBX_LIBDEP_MODE PRIVATE)
else()
add_library(mdbx STATIC ${LIBMDBX_SOURCES})
set(MDBX_LIBDEP_MODE PUBLIC)
endif()
if(CC_HAS_VISIBILITY AND (LTO_ENABLED OR INTERPROCEDURAL_OPTIMIZATION))
set_target_properties(mdbx PROPERTIES LINK_FLAGS "-fvisibility=hidden")
endif()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
if(MSVC)
if(NOT MSVC_LIB_EXE)
# Find lib.exe
get_filename_component(CL_NAME ${CMAKE_C_COMPILER} NAME)
string(REPLACE cl.exe lib.exe MSVC_LIB_EXE ${CL_NAME})
find_program(MSVC_LIB_EXE ${MSVC_LIB_EXE})
endif()
if(MSVC_LIB_EXE)
message(STATUS "Found MSVC's lib tool: ${MSVC_LIB_EXE}")
set(MDBX_NTDLL_EXTRA_IMPLIB ${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.lib)
add_custom_command(OUTPUT ${MDBX_NTDLL_EXTRA_IMPLIB}
COMMENT "Create extra-import-library for ntdll.dll"
MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def"
COMMAND ${MSVC_LIB_EXE} /def:"${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" /out:"${MDBX_NTDLL_EXTRA_IMPLIB}" ${INITIAL_CMAKE_STATIC_LINKER_FLAGS})
else()
message(SEND_ERROR "MSVC's lib tool not found")
endif()
elseif(MINGW OR MINGW64)
if(NOT DLLTOOL)
# Find dlltool
get_filename_component(GCC_NAME ${CMAKE_C_COMPILER} NAME)
string(REPLACE gcc dlltool DLLTOOL_NAME ${GCC_NAME})
find_program(DLLTOOL NAMES ${DLLTOOL_NAME})
endif()
if(DLLTOOL)
message(STATUS "Found dlltool: ${DLLTOOL}")
set(MDBX_NTDLL_EXTRA_IMPLIB "${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.a")
add_custom_command(OUTPUT ${MDBX_NTDLL_EXTRA_IMPLIB}
COMMENT "Create extra-import-library for ntdll.dll"
MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def"
COMMAND ${DLLTOOL} -d "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" -l "${MDBX_NTDLL_EXTRA_IMPLIB}")
else()
message(SEND_ERROR "dlltool not found")
endif()
endif()
endif()
target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ${CMAKE_THREAD_LIBS_INIT})
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ntdll.lib)
if(MDBX_NTDLL_EXTRA_IMPLIB)
# LY: Sometimes Cmake requires a nightmarish magic for simple things.
# 1) create a target out of the library compilation result
add_custom_target(ntdll_extra_target DEPENDS ${MDBX_NTDLL_EXTRA_IMPLIB})
# 2) create an library target out of the library compilation result
add_library(ntdll_extra STATIC IMPORTED GLOBAL)
add_dependencies(ntdll_extra ntdll_extra_target)
# 3) specify where the library is (and where to find the headers)
set_target_properties(ntdll_extra
PROPERTIES
IMPORTED_LOCATION ${MDBX_NTDLL_EXTRA_IMPLIB})
target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ntdll_extra)
endif()
endif()
set_target_properties(mdbx PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
PUBLIC_HEADER "../mdbx.h")
if(CC_HAS_FASTMATH)
target_compile_options(mdbx PRIVATE "-ffast-math")
endif()
if(BUILD_FOR_NATIVE_CPU AND CC_HAS_ARCH_NATIVE)
target_compile_options(mdbx PUBLIC "-march=native")
endif()
if(CC_HAS_VISIBILITY)
target_compile_options(mdbx PRIVATE "-fvisibility=hidden")
endif()
install(TARGETS mdbx
LIBRARY DESTINATION lib COMPONENT runtime
RUNTIME DESTINATION bin COMPONENT runtime
ARCHIVE DESTINATION lib/static COMPONENT devel
PUBLIC_HEADER DESTINATION include
INCLUDES DESTINATION include COMPONENT devel)
################################################################################
#
# library build info (used in library version output)
#
set(MDBX_BUILD_FLAGS "")
# append cmake's build-type flags and defines
if(NOT CMAKE_CONFIGURATION_TYPES)
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}})
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES_${CMAKE_BUILD_TYPE_UPPERCASE}})
endif()
# append linker dll's options
if(LIBMDBX_TYPE STREQUAL "SHARED")
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_SHARED_LINKER_FLAGS})
endif()
# get definitions
get_target_property(defs_list mdbx COMPILE_DEFINITIONS)
if(defs_list)
list(APPEND MDBX_BUILD_FLAGS ${defs_list})
endif()
# get target compile options
get_target_property(options_list mdbx COMPILE_OPTIONS)
if(options_list)
list(APPEND MDBX_BUILD_FLAGS ${options_list})
endif()
list(REMOVE_DUPLICATES MDBX_BUILD_FLAGS)
string(REPLACE ";" " " MDBX_BUILD_FLAGS "${MDBX_BUILD_FLAGS}")
if(CMAKE_CONFIGURATION_TYPES)
# add dynamic part via per-configuration define
message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS} <AND CONFIGURATION DEPENDENT>")
add_definitions(-DMDBX_BUILD_FLAGS_CONFIG="$<$<CONFIG:Debug>:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$<CONFIG:Release>:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$<CONFIG:RelWithDebInfo>:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$<CONFIG:MinSizeRel>:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>")
else()
message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS}")
endif()
# get compiler info
execute_process(COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1"
OUTPUT_VARIABLE MDBX_BUILD_COMPILER
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
RESULT_VARIABLE rc)
if(rc OR NOT MDBX_BUILD_COMPILER)
string(STRIP "${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}" MDBX_BUILD_COMPILER)
endif()
# make a build-target triplet
if(CMAKE_C_COMPILER_TARGET)
set(MDBX_BUILD_TARGET "${CMAKE_C_COMPILER_TARGET}")
elseif(CMAKE_C_PLATFORM_ID AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_C_PLATFORM_ID}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_LIBRARY_ARCHITECTURE)
string(STRIP "${CMAKE_LIBRARY_ARCHITECTURE}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_GENERATOR_PLATFORM}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_SYSTEM_ARCH)
string(STRIP "${CMAKE_SYSTEM_ARCH}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
else()
string(STRIP "${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
endif()
if(CMAKE_CONFIGURATION_TYPES)
add_definitions(-DMDBX_BUILD_CONFIG="$<CONFIG>")
else()
set(MDBX_BUILD_CONFIG ${CMAKE_BUILD_TYPE})
endif()
# options
string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC)
set(options VERSION C_COMPILER CXX_COMPILER)
foreach(item IN LISTS options)
if(DEFINED ${item})
set(value "${${item}}")
elseif(DEFINED MDBX_${item})
set(item MDBX_${item})
set(value "${${item}}")
elseif(DEFINED CMAKE_${item})
set(item CMAKE_${item})
set(value "${${item}}")
else()
set(value "undefined")
endif()
message(STATUS "${item}: ${value}")
endforeach(item)
# generate version and config files
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c.in"
"${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c" ESCAPE_QUOTES)
file(SHA256 "${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c" MDBX_SOURCERY_DIGEST)
string(MAKE_C_IDENTIFIER "${MDBX_GIT_DESCRIBE}" MDBX_SOURCERY_SUFFIX)
set(MDBX_BUILD_SOURCERY "${MDBX_SOURCERY_DIGEST}_${MDBX_SOURCERY_SUFFIX}")
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h.in"
"${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h" ESCAPE_QUOTES)
add_definitions(-DMDBX_CONFIG_H="config.h")
add_subdirectory(tools)

26
src/contrib/db/libmdbx/src/alloy.c Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
/* Amalgamated build */
#define MDBX_ALLOY 1
#include "elements/internals.h" /* must be included fisrt */
#include "elements/core.c"
#include "elements/osal.c"
#include "elements/version.c"
#if defined(_WIN32) || defined(_WIN64)
#include "elements/lck-windows.c"
#else
#include "elements/lck-posix.c"
#endif

56
src/contrib/db/libmdbx/src/elements/config.h.in Normal file
View file

@ -0,0 +1,56 @@
/* This is CMake-template for libmdbx's config.h
******************************************************************************/
/* *INDENT-OFF* */
/* clang-format off */
#cmakedefine HAVE_VALGRIND_MEMCHECK_H
#cmakedefine HAS_RELAXED_CONSTEXPR
#cmakedefine LTO_ENABLED
#cmakedefine MDBX_USE_VALGRIND
#cmakedefine ENABLE_GPROF
#cmakedefine ENABLE_GCOV
#cmakedefine ENABLE_ASAN
/* Common */
#cmakedefine01 MDBX_TXN_CHECKPID
#cmakedefine01 MDBX_TXN_CHECKOWNER
#cmakedefine01 MDBX_BUILD_SHARED_LIBRARY
/* Windows */
#cmakedefine01 MDBX_CONFIG_MANUAL_TLS_CALLBACK
#cmakedefine01 MDBX_AVOID_CRT
/* MacOS */
#cmakedefine01 MDBX_OSX_SPEED_INSTEADOF_DURABILITY
/* POSIX */
#cmakedefine01 MDBX_USE_ROBUST
#cmakedefine01 MDBX_USE_OFDLOCKS
#cmakedefine01 MDBX_DISABLE_GNU_SOURCE
/* Simulate "AUTO" values of tristate options */
#cmakedefine MDBX_TXN_CHECKPID_AUTO
#ifdef MDBX_TXN_CHECKPID_AUTO
#undef MDBX_TXN_CHECKPID
#endif
#cmakedefine MDBX_USE_ROBUST_AUTO
#ifdef MDBX_USE_ROBUST_AUTO
#undef MDBX_USE_ROBUST
#endif
#cmakedefine MDBX_USE_OFDLOCKS_AUTO
#ifdef MDBX_USE_OFDLOCKS_AUTO
#undef MDBX_USE_OFDLOCKS
#endif
/* Build Info */
#cmakedefine MDBX_BUILD_TIMESTAMP "@MDBX_BUILD_TIMESTAMP@"
#cmakedefine MDBX_BUILD_TARGET "@MDBX_BUILD_TARGET@"
#cmakedefine MDBX_BUILD_CONFIG "@MDBX_BUILD_CONFIG@"
#cmakedefine MDBX_BUILD_COMPILER "@MDBX_BUILD_COMPILER@"
#cmakedefine MDBX_BUILD_FLAGS "@MDBX_BUILD_FLAGS@"
#cmakedefine MDBX_BUILD_SOURCERY @MDBX_BUILD_SOURCERY@
/* *INDENT-ON* */
/* clang-format on */

16709
src/contrib/db/libmdbx/src/elements/core.c Normal file
View file

File diff suppressed because it is too large Load diff

450
src/contrib/db/libmdbx/src/elements/defs.h Normal file
View file

@ -0,0 +1,450 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/* *INDENT-OFF* */
/* clang-format off */
#ifndef __GNUC_PREREQ
# if defined(__GNUC__) && defined(__GNUC_MINOR__)
# define __GNUC_PREREQ(maj, min) \
((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GNUC_PREREQ(maj, min) (0)
# endif
#endif /* __GNUC_PREREQ */
#ifndef __CLANG_PREREQ
# ifdef __clang__
# define __CLANG_PREREQ(maj,min) \
((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
# else
# define __CLANG_PREREQ(maj,min) (0)
# endif
#endif /* __CLANG_PREREQ */
#ifndef __GLIBC_PREREQ
# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
# define __GLIBC_PREREQ(maj, min) \
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GLIBC_PREREQ(maj, min) (0)
# endif
#endif /* __GLIBC_PREREQ */
#ifndef __has_attribute
# define __has_attribute(x) (0)
#endif
#ifndef __has_feature
# define __has_feature(x) (0)
#endif
#ifndef __has_extension
# define __has_extension(x) (0)
#endif
#ifndef __has_builtin
# define __has_builtin(x) (0)
#endif
#ifndef __has_warning
# define __has_warning(x) (0)
#endif
#ifndef __has_include
# define __has_include(x) (0)
#endif
#if __has_feature(thread_sanitizer)
# define __SANITIZE_THREAD__ 1
#endif
#if __has_feature(address_sanitizer)
# define __SANITIZE_ADDRESS__ 1
#endif
/*----------------------------------------------------------------------------*/
#ifndef __extern_C
# ifdef __cplusplus
# define __extern_C extern "C"
# else
# define __extern_C
# endif
#endif /* __extern_C */
#ifndef __cplusplus
# ifndef bool
# define bool _Bool
# endif
# ifndef true
# define true (1)
# endif
# ifndef false
# define false (0)
# endif
#endif
#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
# define nullptr NULL
#endif
/*----------------------------------------------------------------------------*/
#ifndef __always_inline
# if defined(__GNUC__) || __has_attribute(__always_inline__)
# define __always_inline __inline __attribute__((__always_inline__))
# elif defined(_MSC_VER)
# define __always_inline __forceinline
# else
# define __always_inline
# endif
#endif /* __always_inline */
#ifndef __noinline
# if defined(__GNUC__) || __has_attribute(__noinline__)
# define __noinline __attribute__((__noinline__))
# elif defined(_MSC_VER)
# define __noinline __declspec(noinline)
# elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
# define __noinline inline
# elif !defined(__INTEL_COMPILER)
# define __noinline /* FIXME ? */
# endif
#endif /* __noinline */
#ifndef __must_check_result
# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
# define __must_check_result __attribute__((__warn_unused_result__))
# else
# define __must_check_result
# endif
#endif /* __must_check_result */
#ifndef __maybe_unused
# if defined(__GNUC__) || __has_attribute(__unused__)
# define __maybe_unused __attribute__((__unused__))
# else
# define __maybe_unused
# endif
#endif /* __maybe_unused */
#ifndef __deprecated
# if defined(__GNUC__) || __has_attribute(__deprecated__)
# define __deprecated __attribute__((__deprecated__))
# elif defined(_MSC_VER)
# define __deprecated __declspec(deprecated)
# else
# define __deprecated
# endif
#endif /* __deprecated */
#if !defined(__noop) && !defined(_MSC_VER)
# define __noop(...) do {} while(0)
#endif /* __noop */
#ifndef __fallthrough
# if __GNUC_PREREQ(7, 0) || __has_attribute(__fallthrough__)
# define __fallthrough __attribute__((__fallthrough__))
# else
# define __fallthrough __noop()
# endif
#endif /* __fallthrough */
#ifndef __unreachable
# if __GNUC_PREREQ(4,5)
# define __unreachable() __builtin_unreachable()
# elif defined(_MSC_VER)
# define __unreachable() __assume(0)
# else
# define __unreachable() __noop()
# endif
#endif /* __unreachable */
#ifndef __prefetch
# if defined(__GNUC__) || defined(__clang__)
# define __prefetch(ptr) __builtin_prefetch(ptr)
# else
# define __prefetch(ptr) __noop(ptr)
# endif
#endif /* __prefetch */
#ifndef __noreturn
# if defined(__GNUC__) || __has_attribute(__noreturn__)
# define __noreturn __attribute__((__noreturn__))
# elif defined(_MSC_VER)
# define __noreturn __declspec(noreturn)
# else
# define __noreturn
# endif
#endif /* __noreturn */
#ifndef __nothrow
# if defined(__cplusplus)
# if __cplusplus < 201703L
# define __nothrow throw()
# else
# define __nothrow noexcept(true)
# endif /* __cplusplus */
# elif defined(__GNUC__) || __has_attribute(__nothrow__)
# define __nothrow __attribute__((__nothrow__))
# elif defined(_MSC_VER) && defined(__cplusplus)
# define __nothrow __declspec(nothrow)
# else
# define __nothrow
# endif
#endif /* __nothrow */
#ifndef __pure_function
/* Many functions have no effects except the return value and their
* return value depends only on the parameters and/or global variables.
* Such a function can be subject to common subexpression elimination
* and loop optimization just as an arithmetic operator would be.
* These functions should be declared with the attribute pure. */
# if defined(__GNUC__) || __has_attribute(__pure__)
# define __pure_function __attribute__((__pure__))
# else
# define __pure_function
# endif
#endif /* __pure_function */
#ifndef __const_function
/* Many functions do not examine any values except their arguments,
* and have no effects except the return value. Basically this is just
* slightly more strict class than the PURE attribute, since function
* is not allowed to read global memory.
*
* Note that a function that has pointer arguments and examines the
* data pointed to must not be declared const. Likewise, a function
* that calls a non-const function usually must not be const.
* It does not make sense for a const function to return void. */
# if defined(__GNUC__) || __has_attribute(__const__)
# define __const_function __attribute__((__const__))
# else
# define __const_function
# endif
#endif /* __const_function */
#ifndef __hidden
# if defined(__GNUC__) || __has_attribute(__visibility__)
# define __hidden __attribute__((__visibility__("hidden")))
# else
# define __hidden
# endif
#endif /* __hidden */
#ifndef __optimize
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(__optimize__)
# define __optimize(ops)
# elif defined(__GNUC__) || __has_attribute(__optimize__)
# define __optimize(ops) __attribute__((__optimize__(ops)))
# else
# define __optimize(ops)
# endif
# else
# define __optimize(ops)
# endif
#endif /* __optimize */
#ifndef __hot
# if defined(__OPTIMIZE__)
# if defined(__e2k__)
# define __hot __attribute__((__hot__)) __optimize(3)
# elif defined(__clang__) && !__has_attribute(__hot_) \
&& __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
/* just put frequently used functions in separate section */
# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
# elif defined(__GNUC__) || __has_attribute(__hot__)
# define __hot __attribute__((__hot__)) __optimize("O3")
# else
# define __hot __optimize("O3")
# endif
# else
# define __hot
# endif
#endif /* __hot */
#ifndef __cold
# if defined(__OPTIMIZE__)
# if defined(__e2k__)
# define __cold __attribute__((__cold__)) __optimize(1)
# elif defined(__clang__) && !__has_attribute(cold) \
&& __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
/* just put infrequently used functions in separate section */
# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
# elif defined(__GNUC__) || __has_attribute(cold)
# define __cold __attribute__((__cold__)) __optimize("Os")
# else
# define __cold __optimize("Os")
# endif
# else
# define __cold
# endif
#endif /* __cold */
#ifndef __flatten
# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
# define __flatten __attribute__((__flatten__))
# else
# define __flatten
# endif
#endif /* __flatten */
#ifndef likely
# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__)
# define likely(cond) __builtin_expect(!!(cond), 1)
# else
# define likely(x) (x)
# endif
#endif /* likely */
#ifndef unlikely
# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__)
# define unlikely(cond) __builtin_expect(!!(cond), 0)
# else
# define unlikely(x) (x)
# endif
#endif /* unlikely */
/* Workaround for Coverity Scan */
#if defined(__COVERITY__) && __GNUC_PREREQ(7, 0) && !defined(__cplusplus)
typedef float _Float32;
typedef double _Float32x;
typedef double _Float64;
typedef long double _Float64x;
typedef float _Float128 __attribute__((__mode__(__TF__)));
typedef __complex__ float __cfloat128 __attribute__ ((__mode__ (__TC__)));
typedef _Complex float __cfloat128 __attribute__ ((__mode__ (__TC__)));
#endif /* Workaround for Coverity Scan */
#ifndef __printf_args
# if defined(__GNUC__) || __has_attribute(__format__)
# define __printf_args(format_index, first_arg) \
__attribute__((__format__(printf, format_index, first_arg)))
# else
# define __printf_args(format_index, first_arg)
# endif
#endif /* __printf_args */
#ifndef __anonymous_struct_extension__
# if defined(__GNUC__)
# define __anonymous_struct_extension__ __extension__
# else
# define __anonymous_struct_extension__
# endif
#endif /* __anonymous_struct_extension__ */
#ifndef __Wpedantic_format_voidptr
static __inline const void* __pure_function
__Wpedantic_format_voidptr(const void* ptr) {return ptr;}
# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
#endif /* __Wpedantic_format_voidptr */
/*----------------------------------------------------------------------------*/
#if defined(MDBX_USE_VALGRIND)
# include <valgrind/memcheck.h>
# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
/* LY: available since Valgrind 3.10 */
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# endif
#elif !defined(RUNNING_ON_VALGRIND)
# define VALGRIND_CREATE_MEMPOOL(h,r,z)
# define VALGRIND_DESTROY_MEMPOOL(h)
# define VALGRIND_MEMPOOL_TRIM(h,a,s)
# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
# define VALGRIND_MEMPOOL_FREE(h,a)
# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
# define VALGRIND_MAKE_MEM_DEFINED(a,s)
# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
# define RUNNING_ON_VALGRIND (0)
#endif /* MDBX_USE_VALGRIND */
#ifdef __SANITIZE_ADDRESS__
# include <sanitizer/asan_interface.h>
#elif !defined(ASAN_POISON_MEMORY_REGION)
# define ASAN_POISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
#endif /* __SANITIZE_ADDRESS__ */
/*----------------------------------------------------------------------------*/
#ifndef ARRAY_LENGTH
# ifdef __cplusplus
template <typename T, size_t N>
char (&__ArraySizeHelper(T (&array)[N]))[N];
# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
# else
# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
# endif
#endif /* ARRAY_LENGTH */
#ifndef ARRAY_END
# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
#endif /* ARRAY_END */
#ifndef STRINGIFY
# define STRINGIFY_HELPER(x) #x
# define STRINGIFY(x) STRINGIFY_HELPER(x)
#endif /* STRINGIFY */
#define CONCAT(a,b) a##b
#define XCONCAT(a,b) CONCAT(a,b)
#ifndef offsetof
# define offsetof(type, member) __builtin_offsetof(type, member)
#endif /* offsetof */
#ifndef container_of
# define container_of(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
#endif /* container_of */
#define MDBX_TETRAD(a, b, c, d) \
((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__)
#ifndef STATIC_ASSERT_MSG
# if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
|| __has_feature(c_static_assert)
# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
# elif defined(static_assert)
# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
# elif defined(_MSC_VER)
# include <crtdbg.h>
# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
# else
# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
# endif
#endif /* STATIC_ASSERT */
#ifndef STATIC_ASSERT
# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
#endif
/* *INDENT-ON* */
/* clang-format on */

1367
src/contrib/db/libmdbx/src/elements/internals.h Normal file
View file

File diff suppressed because it is too large Load diff

551
src/contrib/db/libmdbx/src/elements/lck-posix.c Normal file
View file

@ -0,0 +1,551 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "internals.h"
/*----------------------------------------------------------------------------*/
/* global constructor/destructor */
#if defined(__linux__) || defined(__gnu_linux__)
#include <sys/utsname.h>
#ifndef MDBX_ALLOY
uint32_t mdbx_linux_kernel_version;
#endif /* MDBX_ALLOY */
#endif /* Linux */
static __cold __attribute__((__constructor__)) void
mdbx_global_constructor(void) {
#if defined(__linux__) || defined(__gnu_linux__)
struct utsname buffer;
if (uname(&buffer) == 0) {
int i = 0;
char *p = buffer.release;
while (*p && i < 4) {
if (*p >= '0' && *p <= '9') {
long number = strtol(p, &p, 10);
if (number > 0) {
if (number > 255)
number = 255;
mdbx_linux_kernel_version += number << (24 - i * 8);
}
++i;
} else {
++p;
}
}
}
#endif /* Linux */
mdbx_rthc_global_init();
}
static __cold __attribute__((__destructor__)) void
mdbx_global_destructor(void) {
mdbx_rthc_global_dtor();
}
/*----------------------------------------------------------------------------*/
/* lck */
/* Описание реализации блокировок для POSIX & Linux:
*
* lck-файл отображается в память, в нём организуется таблица читателей и
* размещаются совместно используемые posix-мьютексы (futex). Посредством
* этих мьютексов (см struct MDBX_lockinfo) реализуются:
* - Блокировка таблицы читателей для регистрации,
* т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock().
* - Блокировка БД для пишущих транзакций,
* т.е. функции mdbx_txn_lock() и mdbx_txn_unlock().
*
* Остальной функционал реализуется отдельно посредством файловых блокировок:
* - Первоначальный захват БД в режиме exclusive/shared и последующий перевод
* в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade().
* - Проверка присутствие процессов-читателей,
* т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check().
*
* Для блокировки файлов используется fcntl(F_SETLK), так как:
* - lockf() оперирует только эксклюзивной блокировкой и требует
* открытия файла в RW-режиме.
* - flock() не гарантирует атомарности при смене блокировок
* и оперирует только всем файлом целиком.
* - Для контроля процессов-читателей используются однобайтовые
* range-блокировки lck-файла посредством fcntl(F_SETLK). При этом
* в качестве позиции используется pid процесса-читателя.
* - Для первоначального захвата и shared/exclusive выполняется блокировка
* основного файла БД и при успехе lck-файла.
*
* ----------------------------------------------------------------------------
* УДЕРЖИВАЕМЫЕ БЛОКИРОВКИ В ЗАВИСИМОСТИ ОТ РЕЖИМА И СОСТОЯНИЯ
*
* Эксклюзивный режим без lck-файла:
* = заблокирован весь dxb-файл посредством F_RDLCK или F_WRLCK,
* в зависимости от MDBX_RDONLY.
*
* Не-операционный режим на время пере-инициализации и разрушении lck-файла:
* = F_WRLCK блокировка первого байта lck-файла, другие процессы ждут её
* снятия при получении F_RDLCK через F_SETLKW.
* - блокировки dxb-файла могут меняться до снятие эксклюзивной блокировки
* lck-файла:
* + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
* + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
*
* ОПЕРАЦИОННЫЙ режим с lck-файлом:
* = F_RDLCK блокировка первого байта lck-файла, другие процессы не могут
* получить F_WRLCK и таким образом видят что БД используется.
* + F_WRLCK блокировка pid-байта в clk-файле после первой транзакции чтения.
* + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
* + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
*/
#if MDBX_USE_OFDLOCKS
static int op_setlk, op_setlkw, op_getlk;
static void __cold choice_fcntl() {
assert(!op_setlk && !op_setlkw && !op_getlk);
if ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0
#if defined(__linux__) || defined(__gnu_linux__)
&& mdbx_linux_kernel_version >
0x030f0000 /* OFD locks are available since 3.15, but engages here
only for 3.16 and larer kernels (LTS) for reliability reasons */
#endif /* linux */
) {
op_setlk = F_OFD_SETLK;
op_setlkw = F_OFD_SETLKW;
op_getlk = F_OFD_GETLK;
return;
}
op_setlk = F_SETLK;
op_setlkw = F_SETLKW;
op_getlk = F_GETLK;
}
#else
#define op_setlk F_SETLK
#define op_setlkw F_SETLKW
#define op_getlk F_GETLK
#endif /* MDBX_USE_OFDLOCKS */
#ifndef OFF_T_MAX
#define OFF_T_MAX \
((sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff)
#endif
static int lck_op(mdbx_filehandle_t fd, int cmd, int lck, off_t offset,
off_t len) {
mdbx_jitter4testing(true);
for (;;) {
struct flock lock_op;
memset(&lock_op, 0, sizeof(lock_op));
lock_op.l_type = lck;
lock_op.l_whence = SEEK_SET;
lock_op.l_start = offset;
lock_op.l_len = len;
int rc = fcntl(fd, cmd, &lock_op);
mdbx_jitter4testing(true);
if (rc != -1) {
if (cmd == op_getlk) {
/* Checks reader by pid. Returns:
* MDBX_RESULT_TRUE - if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE - if pid is dead (lock acquired). */
return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE
: MDBX_RESULT_TRUE;
}
return MDBX_SUCCESS;
}
rc = errno;
if (rc != EINTR || cmd == op_setlkw) {
mdbx_assert(nullptr, MDBX_IS_ERROR(rc));
return rc;
}
}
}
MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0);
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1);
}
MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0);
return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1);
}
MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(pid > 0);
return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1);
}
/*---------------------------------------------------------------------------*/
MDBX_INTERNAL_FUNC int __cold mdbx_lck_seize(MDBX_env *env) {
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
#if MDBX_USE_OFDLOCKS
if (unlikely(op_setlk == 0))
choice_fcntl();
#endif /* MDBX_USE_OFDLOCKS */
int rc;
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
rc =
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "without-lck", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
}
/* Firstly try to get exclusive locking. */
rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
if (rc == MDBX_SUCCESS) {
continue_dxb_exclusive:
rc =
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
if (rc == MDBX_SUCCESS)
return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
/* the cause may be a collision with POSIX's file-lock recovery. */
if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
rc == EDEADLK)) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "dxb-exclusive", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Fallback to lck-shared */
rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "fallback-shared", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Done: return with shared locking. */
return MDBX_RESULT_FALSE;
}
/* Wait for lck-shared now. */
/* Here may be await during transient processes, for instance until another
* competing process doesn't call lck_downgrade(). */
rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "try-shared", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Lock against another process operating in without-lck or exclusive mode. */
rc =
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__,
"lock-against-without-lck", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* got shared, retry exclusive */
rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
if (rc == MDBX_SUCCESS)
goto continue_dxb_exclusive;
if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
rc == EDEADLK)
return MDBX_RESULT_FALSE /* Done: exclusive is unavailable,
but shared locks are alive. */
;
mdbx_error("%s(%s) failed: errcode %u", __func__, "try-exclusive", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
int rc = MDBX_SUCCESS;
if ((env->me_flags & MDBX_EXCLUSIVE) == 0) {
rc = lck_op(env->me_fd, op_setlk, F_UNLCK, 0, env->me_pid);
if (rc == MDBX_SUCCESS)
rc = lck_op(env->me_fd, op_setlk, F_UNLCK, env->me_pid + 1,
OFF_T_MAX - env->me_pid - 1);
}
if (rc == MDBX_SUCCESS)
rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1);
if (unlikely(rc != 0)) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "lck", rc);
assert(MDBX_IS_ERROR(rc));
}
return rc;
}
MDBX_INTERNAL_FUNC int __cold mdbx_lck_destroy(MDBX_env *env,
MDBX_env *inprocess_neighbor) {
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
int rc = MDBX_SUCCESS;
if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor &&
env->me_lck &&
/* try get exclusive access */
lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 &&
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX)) {
mdbx_verbose("%s: got exclusive, drown mutexes", __func__);
rc = pthread_mutex_destroy(&env->me_lck->mti_rmutex);
if (rc == 0)
rc = pthread_mutex_destroy(&env->me_lck->mti_wmutex);
mdbx_assert(env, rc == 0);
if (rc == 0) {
memset(env->me_lck, 0x81, sizeof(MDBX_lockinfo));
msync(env->me_lck, env->me_os_psize, MS_ASYNC);
}
mdbx_jitter4testing(false);
}
/* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored
* after file was closed.
*
* 2) File locks would be released (by kernel) while the file-descriptors will
* be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel,
* locks should be released here explicitly with properly order. */
/* close dxb and restore lock */
if (env->me_fd != INVALID_HANDLE_VALUE) {
if (unlikely(close(env->me_fd) != 0) && rc == MDBX_SUCCESS)
rc = errno;
env->me_fd = INVALID_HANDLE_VALUE;
if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
/* restore file-lock */
rc = lck_op(
inprocess_neighbor->me_fd, F_SETLKW,
(inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
(inprocess_neighbor->me_flags & MDBX_EXCLUSIVE)
? 0
: inprocess_neighbor->me_pid,
(inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1);
}
}
/* close clk and restore locks */
if (env->me_lfd != INVALID_HANDLE_VALUE) {
if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS)
rc = errno;
env->me_lfd = INVALID_HANDLE_VALUE;
if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
/* restore file-locks */
rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1);
if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader)
rc = mdbx_rpid_set(inprocess_neighbor);
}
}
if (inprocess_neighbor && rc != MDBX_SUCCESS)
inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR;
return rc;
}
/*---------------------------------------------------------------------------*/
static int mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int rc);
MDBX_INTERNAL_FUNC int __cold mdbx_lck_init(MDBX_env *env,
MDBX_env *inprocess_neighbor,
int global_uniqueness_flag) {
if (inprocess_neighbor)
return MDBX_SUCCESS /* currently don't need any initialization
if LCK already opened/used inside current process */
;
/* FIXME: Unfortunately, there is no other reliable way but to long testing
* on each platform. On the other hand, behavior like FreeBSD is incorrect
* and we can expect it to be rare. Moreover, even on FreeBSD without
* additional in-process initialization, the probability of an problem
* occurring is vanishingly small, and the symptom is a return of EINVAL
* while locking a mutex. In other words, in the worst case, the problem
* results in an EINVAL error at the start of the transaction, but NOT data
* loss, nor database corruption, nor other fatal troubles. Thus, the code
* below I am inclined to think the workaround for erroneous platforms (like
* FreeBSD), rather than a defect of libmdbx. */
#if defined(__FreeBSD__)
/* seems that shared mutexes on FreeBSD required in-process initialization */
(void)global_uniqueness_flag;
#else
/* shared mutexes on many other platforms (including Darwin and Linux's
* futexes) doesn't need any addition in-process initialization */
if (global_uniqueness_flag != MDBX_RESULT_TRUE)
return MDBX_SUCCESS;
#endif
pthread_mutexattr_t ma;
int rc = pthread_mutexattr_init(&ma);
if (rc)
return rc;
rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
if (rc)
goto bailout;
#if MDBX_USE_ROBUST
#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \
!defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L
rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#else
rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#endif
if (rc)
goto bailout;
#endif /* MDBX_USE_ROBUST */
#if _POSIX_C_SOURCE >= 199506L && !defined(MDBX_SAFE4QEMU)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
if (rc == ENOTSUP)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
if (rc)
goto bailout;
#endif /* PTHREAD_PRIO_INHERIT */
rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_rmutex, &ma);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_wmutex, &ma);
bailout:
pthread_mutexattr_destroy(&ma);
return rc;
}
static int mdbx_robust_lock(MDBX_env *env, pthread_mutex_t *mutex) {
mdbx_jitter4testing(true);
int rc = pthread_mutex_lock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
static int mdbx_robust_trylock(MDBX_env *env, pthread_mutex_t *mutex) {
mdbx_jitter4testing(true);
int rc = pthread_mutex_trylock(mutex);
if (unlikely(rc != 0 && rc != EBUSY))
rc = mdbx_mutex_failed(env, mutex, rc);
return (rc != EBUSY) ? rc : MDBX_BUSY;
}
static int mdbx_robust_unlock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_unlock(mutex);
mdbx_jitter4testing(true);
if (unlikely(rc != 0))
env->me_flags |= MDBX_FATAL_ERROR;
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
mdbx_trace("%s", ">>");
int rc = mdbx_robust_lock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
return rc;
}
MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
mdbx_trace("%s", ">>");
int rc = mdbx_robust_unlock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", __func__, rc);
}
int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
mdbx_trace("%s", ">>");
int rc = dontwait ? mdbx_robust_trylock(env, env->me_wmutex)
: mdbx_robust_lock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS;
}
void mdbx_txn_unlock(MDBX_env *env) {
mdbx_trace("%s", ">>");
int rc = mdbx_robust_unlock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", __func__, rc);
}
static int __cold mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int err) {
int rc = err;
#if MDBX_USE_ROBUST
if (err == EOWNERDEAD) {
/* We own the mutex. Clean up after dead previous owner. */
int rlocked = (env->me_lck && mutex == &env->me_lck->mti_rmutex);
rc = MDBX_SUCCESS;
if (!rlocked) {
if (unlikely(env->me_txn)) {
/* env is hosed if the dead thread was ours */
env->me_flags |= MDBX_FATAL_ERROR;
env->me_txn = NULL;
rc = MDBX_PANIC;
}
}
mdbx_notice("%cmutex owner died, %s", (rlocked ? 'r' : 'w'),
(rc ? "this process' env is hosed" : "recovering"));
int check_rc = mdbx_reader_check0(env, rlocked, NULL);
check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \
!defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L
int mreco_rc = pthread_mutex_consistent_np(mutex);
#else
int mreco_rc = pthread_mutex_consistent(mutex);
#endif
check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
if (unlikely(mreco_rc))
mdbx_error("mutex recovery failed, %s", mdbx_strerror(mreco_rc));
rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
if (MDBX_IS_ERROR(rc))
pthread_mutex_unlock(mutex);
return rc;
}
#else
(void)mutex;
#endif /* MDBX_USE_ROBUST */
mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err));
if (rc != EDEADLK)
env->me_flags |= MDBX_FATAL_ERROR;
return rc;
}

777
src/contrib/db/libmdbx/src/elements/lck-windows.c Normal file
View file

@ -0,0 +1,777 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "internals.h"
/* PREAMBLE FOR WINDOWS:
*
* We are not concerned for performance here.
* If you are running Windows a performance could NOT be the goal.
* Otherwise please use Linux.
*
* Regards,
* LY
*/
static void mdbx_winnt_import(void);
#if MDBX_BUILD_SHARED_LIBRARY
#if MDBX_AVOID_CRT && defined(NDEBUG)
/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks.
*
* Define dll's entry point only for Release build when NDEBUG is defined and
* MDBX_AVOID_CRT=ON. if the entry point isn't defined then MSVC's will
* automatically use DllMainCRTStartup() from CRT library, which also
* automatically call DllMain() from our mdbx.dll */
#pragma comment(linker, "/ENTRY:DllMain")
#endif /* MDBX_AVOID_CRT */
BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved)
#else
#if !MDBX_CONFIG_MANUAL_TLS_CALLBACK
static
#endif /* !MDBX_CONFIG_MANUAL_TLS_CALLBACK */
void NTAPI
mdbx_dll_handler(PVOID module, DWORD reason, PVOID reserved)
#endif /* MDBX_BUILD_SHARED_LIBRARY */
{
(void)reserved;
switch (reason) {
case DLL_PROCESS_ATTACH:
mdbx_winnt_import();
mdbx_rthc_global_init();
break;
case DLL_PROCESS_DETACH:
mdbx_rthc_global_dtor();
break;
case DLL_THREAD_ATTACH:
break;
case DLL_THREAD_DETACH:
mdbx_rthc_thread_dtor(module);
break;
}
#if MDBX_BUILD_SHARED_LIBRARY
return TRUE;
#endif
}
#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK
/* *INDENT-OFF* */
/* clang-format off */
#if defined(_MSC_VER)
# pragma const_seg(push)
# pragma data_seg(push)
# ifdef _WIN64
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:_tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor")
/* specific const-segment for WIN64 */
# pragma const_seg(".CRT$XLB")
const
# else
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:__tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor")
/* specific data-segment for WIN32 */
# pragma data_seg(".CRT$XLB")
# endif
__declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_dll_handler;
# pragma data_seg(pop)
# pragma const_seg(pop)
#elif defined(__GNUC__)
# ifdef _WIN64
const
# endif
PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_dll_handler;
#else
# error FIXME
#endif
/* *INDENT-ON* */
/* clang-format on */
#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK */
/*----------------------------------------------------------------------------*/
#define LCK_SHARED 0
#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK
#define LCK_WAITFOR 0
#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY
static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset,
size_t bytes) {
OVERLAPPED ov;
ov.hEvent = 0;
ov.Offset = (DWORD)offset;
ov.OffsetHigh = HIGH_DWORD(offset);
return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov);
}
static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset,
size_t bytes) {
return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes,
HIGH_DWORD(bytes));
}
/*----------------------------------------------------------------------------*/
/* global `write` lock for write-txt processing,
* exclusive locking both meta-pages) */
#define LCK_MAXLEN (1u + (size_t)(MAXSSIZE_T))
#define LCK_META_OFFSET 0
#define LCK_META_LEN 0x10000u
#define LCK_BODY_OFFSET LCK_META_LEN
#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET)
#define LCK_META LCK_META_OFFSET, LCK_META_LEN
#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN
#define LCK_WHOLE 0, LCK_MAXLEN
int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
if (dontwait) {
if (!TryEnterCriticalSection(&env->me_windowsbug_lock))
return MDBX_BUSY;
} else {
EnterCriticalSection(&env->me_windowsbug_lock);
}
if ((env->me_flags & MDBX_EXCLUSIVE) ||
flock(env->me_fd,
dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
: (LCK_EXCLUSIVE | LCK_WAITFOR),
LCK_BODY))
return MDBX_SUCCESS;
int rc = GetLastError();
LeaveCriticalSection(&env->me_windowsbug_lock);
return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY;
}
void mdbx_txn_unlock(MDBX_env *env) {
int rc =
(env->me_flags & MDBX_EXCLUSIVE) ? TRUE : funlock(env->me_fd, LCK_BODY);
LeaveCriticalSection(&env->me_windowsbug_lock);
if (!rc)
mdbx_panic("%s failed: errcode %u", __func__, GetLastError());
}
/*----------------------------------------------------------------------------*/
/* global `read` lock for readers registration,
* exclusive locking `mti_numreaders` (second) cacheline */
#define LCK_LO_OFFSET 0
#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders)
#define LCK_UP_OFFSET LCK_LO_LEN
#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET)
#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN
#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN
MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
mdbx_srwlock_AcquireShared(&env->me_remap_guard);
if (env->me_lfd == INVALID_HANDLE_VALUE)
return MDBX_SUCCESS; /* readonly database in readonly filesystem */
/* transite from S-? (used) to S-E (locked), e.g. exclusive lock upper-part */
if ((env->me_flags & MDBX_EXCLUSIVE) ||
flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER))
return MDBX_SUCCESS;
int rc = GetLastError();
mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
return rc;
}
MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* transite from S-E (locked) to S-? (used), e.g. unlock upper-part */
if ((env->me_flags & MDBX_EXCLUSIVE) == 0 &&
!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s failed: errcode %u", __func__, GetLastError());
}
mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
}
static int suspend_and_append(mdbx_handle_array_t **array,
const DWORD ThreadId) {
const unsigned limit = (*array)->limit;
if ((*array)->count == limit) {
void *ptr = mdbx_realloc(
(limit > ARRAY_LENGTH((*array)->handles))
? *array
: /* don't free initial array on the stack */ NULL,
sizeof(mdbx_handle_array_t) +
sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles)));
if (!ptr)
return MDBX_ENOMEM;
if (limit == ARRAY_LENGTH((*array)->handles))
memcpy(ptr, *array, sizeof(mdbx_handle_array_t));
*array = (mdbx_handle_array_t *)ptr;
(*array)->limit = limit * 2;
}
HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION,
FALSE, ThreadId);
if (hThread == NULL)
return GetLastError();
if (SuspendThread(hThread) == -1) {
int err = GetLastError();
DWORD ExitCode;
if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED ||
!GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE)
err = MDBX_SUCCESS;
CloseHandle(hThread);
return err;
}
(*array)->handles[(*array)->count++] = hThread;
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int
mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
const size_t CurrentTid = GetCurrentThreadId();
int rc;
if (env->me_lck) {
/* Scan LCK for threads of the current process */
const MDBX_reader *const begin = env->me_lck->mti_readers;
const MDBX_reader *const end = begin + env->me_lck->mti_numreaders;
const size_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0;
for (const MDBX_reader *reader = begin; reader < end; ++reader) {
if (reader->mr_pid != env->me_pid || !reader->mr_tid) {
skip_lck:
continue;
}
if (reader->mr_tid == CurrentTid || reader->mr_tid == WriteTxnOwner)
goto skip_lck;
if (env->me_flags & MDBX_NOTLS) {
/* Skip duplicates in no-tls mode */
for (const MDBX_reader *scan = reader; --scan >= begin;)
if (scan->mr_tid == reader->mr_tid)
goto skip_lck;
}
rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid);
if (rc != MDBX_SUCCESS) {
bailout_lck:
(void)mdbx_resume_threads_after_remap(*array);
return rc;
}
}
if (WriteTxnOwner && WriteTxnOwner != CurrentTid) {
rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner);
if (rc != MDBX_SUCCESS)
goto bailout_lck;
}
} else {
/* Without LCK (i.e. read-only mode).
* Walk thougth a snapshot of all running threads */
mdbx_assert(env,
env->me_txn0 == NULL || (env->me_flags & MDBX_EXCLUSIVE) != 0);
const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
if (hSnapshot == INVALID_HANDLE_VALUE)
return GetLastError();
THREADENTRY32 entry;
entry.dwSize = sizeof(THREADENTRY32);
if (!Thread32First(hSnapshot, &entry)) {
rc = GetLastError();
bailout_toolhelp:
CloseHandle(hSnapshot);
(void)mdbx_resume_threads_after_remap(*array);
return rc;
}
do {
if (entry.th32OwnerProcessID != env->me_pid ||
entry.th32ThreadID == CurrentTid)
continue;
rc = suspend_and_append(array, entry.th32ThreadID);
if (rc != MDBX_SUCCESS)
goto bailout_toolhelp;
} while (Thread32Next(hSnapshot, &entry));
rc = GetLastError();
if (rc != ERROR_NO_MORE_FILES)
goto bailout_toolhelp;
CloseHandle(hSnapshot);
}
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int
mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) {
int rc = MDBX_SUCCESS;
for (unsigned i = 0; i < array->count; ++i) {
const HANDLE hThread = array->handles[i];
if (ResumeThread(hThread) == -1) {
const int err = GetLastError();
DWORD ExitCode;
if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED &&
GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE)
rc = err;
}
CloseHandle(hThread);
}
return rc;
}
/*----------------------------------------------------------------------------*/
/* global `initial` lock for lockfile initialization,
* exclusive/shared locking first cacheline */
/* Briefly descritpion of locking schema/algorithm:
* - Windows does not support upgrading or downgrading for file locking.
* - Therefore upgrading/downgrading is emulated by shared and exclusive
* locking of upper and lower halves.
* - In other words, we have FSM with possible 9 states,
* i.e. free/shared/exclusive x free/shared/exclusive == 9.
* Only 6 states of FSM are used, which 2 of ones are transitive.
*
* The mdbx_lck_seize() moves the locking-FSM from the initial free/unlocked
* state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible,
* or to the "used" (and returns MDBX_RESULT_FALSE).
*
* The mdbx_lck_downgrade() moves the locking-FSM from "exclusive write"
* state to the "used" (i.e. shared) state.
*
* States:
* ?-? = free, i.e. unlocked
* S-? = used, i.e. shared lock
* E-? = exclusive-read, i.e. operational exclusive
* ?-S
* ?-E = middle (transitive state)
* S-S
* S-E = locked (transitive state)
* E-S
* E-E = exclusive-write, i.e. exclusive due (re)initialization
*/
static void lck_unlock(MDBX_env *env) {
int rc;
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* double `unlock` for robustly remove overlapped shared/exclusive locks */
while (funlock(env->me_lfd, LCK_LOWER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_lfd, LCK_UPPER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
if (env->me_fd != INVALID_HANDLE_VALUE) {
/* explicitly unlock to avoid latency for other processes (windows kernel
* releases such locks via deferred queues) */
while (funlock(env->me_fd, LCK_BODY))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_META))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_WHOLE))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
}
MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
MDBX_env *inprocess_neighbor,
int global_uniqueness_flag) {
(void)env;
(void)inprocess_neighbor;
(void)global_uniqueness_flag;
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
MDBX_env *inprocess_neighbor) {
(void)inprocess_neighbor;
/* LY: should unmap before releasing the locks to avoid race condition and
* STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */
if (env->me_map)
mdbx_munmap(&env->me_dxb_mmap);
if (env->me_lck)
mdbx_munmap(&env->me_lck_mmap);
lck_unlock(env);
return MDBX_SUCCESS;
}
/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE)
* or as 'used' (S-? and returns MDBX_RESULT_FALSE).
* Oherwise returns an error. */
static int internal_seize_lck(HANDLE lfd) {
int rc;
assert(lfd != INVALID_HANDLE_VALUE);
/* 1) now on ?-? (free), get ?-E (middle) */
mdbx_jitter4testing(false);
if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) {
rc = GetLastError() /* 2) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", __func__,
"?-?(free) >> ?-E(middle)", rc);
return rc;
}
/* 3) now on ?-E (middle), try E-E (exclusive-write) */
mdbx_jitter4testing(false);
if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER))
return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */;
/* 5) still on ?-E (middle) */
rc = GetLastError();
mdbx_jitter4testing(false);
if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
/* 6) something went wrong, give up */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"?-E(middle) >> ?-?(free)", GetLastError());
return rc;
}
/* 7) still on ?-E (middle), try S-E (locked) */
mdbx_jitter4testing(false);
rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE
: GetLastError();
mdbx_jitter4testing(false);
if (rc != MDBX_RESULT_FALSE)
mdbx_error("%s(%s) failed: errcode %u", __func__,
"?-E(middle) >> S-E(locked)", rc);
/* 8) now on S-E (locked) or still on ?-E (middle),
* transite to S-? (used) or ?-? (free) */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"X-E(locked/middle) >> X-?(used/free)", GetLastError());
/* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) {
int rc;
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (env->me_flags & MDBX_EXCLUSIVE)
return MDBX_RESULT_TRUE /* nope since files were must be opened
non-shareable */
;
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. on read-only filesystem) */
mdbx_jitter4testing(false);
if (!flock(env->me_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", __func__, "without-lck", rc);
return rc;
}
return MDBX_RESULT_FALSE;
}
rc = internal_seize_lck(env->me_lfd);
mdbx_jitter4testing(false);
if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
/* Check that another process don't operates in without-lck mode.
* Doing such check by exclusive locking the body-part of db. Should be
* noted:
* - we need an exclusive lock for do so;
* - we can't lock meta-pages, otherwise other process could get an error
* while opening db in valid (non-conflict) mode. */
if (!flock(env->me_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", __func__,
"lock-against-without-lck", rc);
mdbx_jitter4testing(false);
lck_unlock(env);
} else {
mdbx_jitter4testing(false);
if (!funlock(env->me_fd, LCK_BODY))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"unlock-against-without-lck", GetLastError());
}
}
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
/* Transite from exclusive state (E-?) to used (S-?) */
assert(env->me_fd != INVALID_HANDLE_VALUE);
assert(env->me_lfd != INVALID_HANDLE_VALUE);
#if 1
if (env->me_flags & MDBX_EXCLUSIVE)
return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
;
#else
/* 1) must be at E-E (exclusive-write) */
if (env->me_flags & MDBX_EXCLUSIVE) {
/* transite from E-E to E_? (exclusive-read) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"E-E(exclusive-write) >> E-?(exclusive-read)", GetLastError());
return MDBX_SUCCESS /* 2) now at E-? (exclusive-read), done */;
}
#endif
/* 3) now at E-E (exclusive-write), transite to ?_E (middle) */
if (!funlock(env->me_lfd, LCK_LOWER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"E-E(exclusive-write) >> ?-E(middle)", GetLastError());
/* 4) now at ?-E (middle), transite to S-E (locked) */
if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) {
int rc = GetLastError() /* 5) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", __func__,
"?-E(middle) >> S-E(locked)", rc);
return rc;
}
/* 6) got S-E (locked), continue transition to S-? (used) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"S-E(locked) >> S-?(used)", GetLastError());
return MDBX_SUCCESS /* 7) now at S-? (used), done */;
}
/*----------------------------------------------------------------------------*/
/* reader checking (by pid) */
MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
/* Checks reader by pid.
*
* Returns:
* MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE, if pid is dead (lock acquired)
* or otherwise the errcode. */
MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
(void)env;
HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid);
int rc;
if (likely(hProcess)) {
rc = WaitForSingleObject(hProcess, 0);
if (unlikely(rc == WAIT_FAILED))
rc = GetLastError();
CloseHandle(hProcess);
} else {
rc = GetLastError();
}
switch (rc) {
case ERROR_INVALID_PARAMETER:
/* pid seems invalid */
return MDBX_RESULT_FALSE;
case WAIT_OBJECT_0:
/* process just exited */
return MDBX_RESULT_FALSE;
case WAIT_TIMEOUT:
/* pid running */
return MDBX_RESULT_TRUE;
default:
/* failure */
return rc;
}
}
//----------------------------------------------------------------------------
// Stub for slim read-write lock
// Copyright (C) 1995-2002 Brad Wilson
static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) {
srwl->readerCount = srwl->writerCount = 0;
}
static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) {
while (true) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
// If there's a writer already, spin without unnecessarily
// interlocking the CPUs
if (srwl->writerCount != 0) {
YieldProcessor();
continue;
}
// Add to the readers list
_InterlockedIncrement(&srwl->readerCount);
// Check for writers again (we may have been pre-empted). If
// there are no writers writing or waiting, then we're done.
if (srwl->writerCount == 0)
break;
// Remove from the readers list, spin, try again
_InterlockedDecrement(&srwl->readerCount);
YieldProcessor();
}
}
static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) {
assert(srwl->readerCount > 0);
_InterlockedDecrement(&srwl->readerCount);
}
static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) {
while (true) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
// If there's a writer already, spin without unnecessarily
// interlocking the CPUs
if (srwl->writerCount != 0) {
YieldProcessor();
continue;
}
// See if we can become the writer (expensive, because it inter-
// locks the CPUs, so writing should be an infrequent process)
if (_InterlockedExchange(&srwl->writerCount, 1) == 0)
break;
}
// Now we're the writer, but there may be outstanding readers.
// Spin until there aren't any more; new readers will wait now
// that we're the writer.
while (srwl->readerCount != 0) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
YieldProcessor();
}
}
static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) {
assert(srwl->writerCount == 1 && srwl->readerCount >= 0);
srwl->writerCount = 0;
}
MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared,
mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive,
mdbx_srwlock_ReleaseExclusive;
/*----------------------------------------------------------------------------*/
#if 0 /* LY: unused for now */
static DWORD WINAPI stub_DiscardVirtualMemory(PVOID VirtualAddress,
SIZE_T Size) {
return VirtualAlloc(VirtualAddress, Size, MEM_RESET, PAGE_NOACCESS)
? ERROR_SUCCESS
: GetLastError();
}
#endif /* unused for now */
static uint64_t WINAPI stub_GetTickCount64(void) {
LARGE_INTEGER Counter, Frequency;
return (QueryPerformanceFrequency(&Frequency) &&
QueryPerformanceCounter(&Counter))
? Counter.QuadPart * 1000ul / Frequency.QuadPart
: 0;
}
/*----------------------------------------------------------------------------*/
#ifndef MDBX_ALLOY
MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx;
MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW;
MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle;
MDBX_NtFsControlFile mdbx_NtFsControlFile;
MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
MDBX_GetTickCount64 mdbx_GetTickCount64;
#if 0 /* LY: unused for now */
MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory;
MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory;
MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory;
#endif /* unused for now */
#endif /* MDBX_ALLOY */
static void mdbx_winnt_import(void) {
const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll");
const MDBX_srwlock_function init =
(MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock");
if (init != NULL) {
mdbx_srwlock_Init = init;
mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockShared");
mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockShared");
mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockExclusive");
mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockExclusive");
} else {
mdbx_srwlock_Init = stub_srwlock_Init;
mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared;
mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared;
mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive;
mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive;
}
#define GET_KERNEL32_PROC(ENTRY) \
mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(hKernel32dll, #ENTRY)
GET_KERNEL32_PROC(GetFileInformationByHandleEx);
GET_KERNEL32_PROC(GetVolumeInformationByHandleW);
GET_KERNEL32_PROC(GetFinalPathNameByHandleW);
GET_KERNEL32_PROC(SetFileInformationByHandle);
GET_KERNEL32_PROC(PrefetchVirtualMemory);
GET_KERNEL32_PROC(GetTickCount64);
if (!mdbx_GetTickCount64)
mdbx_GetTickCount64 = stub_GetTickCount64;
#if 0 /* LY: unused for now */
GET_KERNEL32_PROC(DiscardVirtualMemory);
if (!mdbx_DiscardVirtualMemory)
mdbx_DiscardVirtualMemory = stub_DiscardVirtualMemory;
GET_KERNEL32_PROC(OfferVirtualMemory);
GET_KERNEL32_PROC(ReclaimVirtualMemory);
#endif /* unused for now */
#undef GET_KERNEL32_PROC
const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll");
mdbx_NtFsControlFile =
(MDBX_NtFsControlFile)GetProcAddress(hNtdll, "NtFsControlFile");
}

1244
src/contrib/db/libmdbx/src/elements/ntdll.def Normal file
View file

File diff suppressed because it is too large Load diff

1899
src/contrib/db/libmdbx/src/elements/osal.c Normal file
View file

File diff suppressed because it is too large Load diff

959
src/contrib/db/libmdbx/src/elements/osal.h Normal file
View file

@ -0,0 +1,959 @@
/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/*----------------------------------------------------------------------------*/
/* Microsoft compiler generates a lot of warning for self includes... */
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
#if defined(_WIN32) || defined(_WIN64)
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif
#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
!defined(MDBX_TOOLS)
#define _NO_CRT_STDIO_INLINE
#endif
#endif /* Windows */
/*----------------------------------------------------------------------------*/
/* C99 includes */
#include <inttypes.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
/* C11 stdalign.h */
#if __has_include(<stdalign.h>)
#include <stdalign.h>
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
#define alignas(N) _Alignas(N)
#elif defined(_MSC_VER)
#define alignas(N) __declspec(align(N))
#elif __has_attribute(__aligned__) || defined(__GNUC__)
#define alignas(N) __attribute__((__aligned__(N)))
#else
#error "FIXME: Required _alignas() or equivalent."
#endif
/*----------------------------------------------------------------------------*/
/* Systems includes */
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
defined(__BSD__) || defined(__NETBSD__) || defined(__bsdi__) || \
defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__)
#include <sys/cdefs.h>
#include <sys/sysctl.h>
#include <sys/types.h>
#if defined(__FreeBSD__) || defined(__DragonFly__)
#include <vm/vm_param.h>
#elif defined(__OpenBSD__) || defined(__NetBSD__)
#include <uvm/uvm_param.h>
#else
#define SYSCTL_LEGACY_NONCONST_MIB
#endif
#include <sys/vmmeter.h>
#else
#include <malloc.h>
#ifndef _POSIX_C_SOURCE
#ifdef _POSIX_SOURCE
#define _POSIX_C_SOURCE 1
#else
#define _POSIX_C_SOURCE 0
#endif
#endif
#endif /* !xBSD */
#if defined(__FreeBSD__) || defined(__OpenBSD__) || __has_include(<malloc_np.h>)
#include <malloc_np.h>
#endif
#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
#include <malloc/malloc.h>
#endif /* MacOS */
#if defined(__MACH__)
#include <mach/host_info.h>
#include <mach/mach_host.h>
#include <mach/mach_port.h>
#include <uuid/uuid.h>
#undef P_DIRTY
#endif
#if defined(__linux__) || defined(__gnu_linux__)
#include <linux/sysctl.h>
#include <sys/sendfile.h>
#include <sys/statvfs.h>
#endif /* Linux */
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 0
#endif
#ifndef _XOPEN_SOURCE_EXTENDED
#define _XOPEN_SOURCE_EXTENDED 0
#else
#include <utmpx.h>
#endif /* _XOPEN_SOURCE_EXTENDED */
#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
#include <kstat.h>
#endif /* SunOS/Solaris */
#if defined(_WIN32) || defined(_WIN64)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <tlhelp32.h>
#include <windows.h>
#include <winnt.h>
#include <winternl.h>
#define HAVE_SYS_STAT_H
#define HAVE_SYS_TYPES_H
typedef HANDLE mdbx_thread_t;
typedef unsigned mdbx_thread_key_t;
#define MDBX_OSAL_SECTION HANDLE
#define MAP_FAILED NULL
#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
#define THREAD_CALL WINAPI
#define THREAD_RESULT DWORD
typedef struct {
HANDLE mutex;
HANDLE event;
} mdbx_condmutex_t;
typedef CRITICAL_SECTION mdbx_fastmutex_t;
#if MDBX_AVOID_CRT
#ifndef mdbx_malloc
static inline void *mdbx_malloc(size_t bytes) {
return LocalAlloc(LMEM_FIXED, bytes);
}
#endif /* mdbx_malloc */
#ifndef mdbx_calloc
static inline void *mdbx_calloc(size_t nelem, size_t size) {
return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size);
}
#endif /* mdbx_calloc */
#ifndef mdbx_realloc
static inline void *mdbx_realloc(void *ptr, size_t bytes) {
return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE);
}
#endif /* mdbx_realloc */
#ifndef mdbx_free
#define mdbx_free LocalFree
#endif /* mdbx_free */
#else
#define mdbx_malloc malloc
#define mdbx_calloc calloc
#define mdbx_realloc realloc
#define mdbx_free free
#define mdbx_strdup _strdup
#endif /* MDBX_AVOID_CRT */
#ifndef snprintf
#define snprintf _snprintf /* ntdll */
#endif
#ifndef vsnprintf
#define vsnprintf _vsnprintf /* ntdll */
#endif
#else /*----------------------------------------------------------------------*/
#include <pthread.h>
#include <signal.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <unistd.h>
typedef pthread_t mdbx_thread_t;
typedef pthread_key_t mdbx_thread_key_t;
#define INVALID_HANDLE_VALUE (-1)
#define THREAD_CALL
#define THREAD_RESULT void *
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
} mdbx_condmutex_t;
typedef pthread_mutex_t mdbx_fastmutex_t;
#define mdbx_malloc malloc
#define mdbx_calloc calloc
#define mdbx_realloc realloc
#define mdbx_free free
#define mdbx_strdup strdup
#endif /* Platform */
#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
/* malloc_usable_size() already provided */
#elif defined(__APPLE__)
#define malloc_usable_size(ptr) malloc_size(ptr)
#elif defined(_MSC_VER) && !MDBX_AVOID_CRT
#define malloc_usable_size(ptr) _msize(ptr)
#endif /* malloc_usable_size */
/* *INDENT-OFF* */
/* clang-format off */
#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
#include <sys/stat.h>
#endif
#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
#include <sys/types.h>
#endif
#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
#include <sys/file.h>
#endif
/* *INDENT-ON* */
/* clang-format on */
#ifndef SSIZE_MAX
#define SSIZE_MAX INTPTR_MAX
#endif
#if !defined(MADV_DODUMP) && defined(MADV_CORE)
#define MADV_DODUMP MADV_CORE
#endif /* MADV_CORE -> MADV_DODUMP */
#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
#define MADV_DONTDUMP MADV_NOCORE
#endif /* MADV_NOCORE -> MADV_DONTDUMP */
#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
defined(i486) || defined(__i486) || defined(__i486__) || \
defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
#ifndef __ia32__
/* LY: define neutral __ia32__ for x86 and x86-64 archs */
#define __ia32__ 1
#endif /* __ia32__ */
#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64) || defined(_M_X64))
/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
#define __amd64__ 1
#endif /* __amd64__ */
#endif /* all x86 */
#if !defined(MDBX_UNALIGNED_OK)
#if defined(_MSC_VER)
#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */
#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */
#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \
!defined(__ALIGNED__)
#define MDBX_UNALIGNED_OK 1
#else
#define MDBX_UNALIGNED_OK 0
#endif
#endif /* MDBX_UNALIGNED_OK */
#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
#error \
"Sanity checking failed: Two's complement, reasonably sized integer types"
#endif
/*----------------------------------------------------------------------------*/
/* Compiler's includes for builtins/intrinsics */
#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
#include <intrin.h>
#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
#if defined(__ia32__) || defined(__e2k__)
#include <x86intrin.h>
#endif /* __ia32__ */
#if defined(__ia32__)
#include <cpuid.h>
#endif /* __ia32__ */
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
#include <mbarrier.h>
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#include <machine/sys/inline.h>
#elif defined(__IBMC__) && defined(__powerpc)
#include <atomic.h>
#elif defined(_AIX)
#include <builtins.h>
#include <sys/atomic_op.h>
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
#include <c_asm.h>
#include <machine/builtins.h>
#elif defined(__MWERKS__)
/* CodeWarrior - troubles ? */
#pragma gcc_extensions
#elif defined(__SNC__)
/* Sony PS3 - troubles ? */
#elif defined(__hppa__) || defined(__hppa)
#include <machine/inline.h>
#else
#error Unsupported C compiler, please use GNU C 4.4 or newer
#endif /* Compiler */
/*----------------------------------------------------------------------------*/
/* Byteorder */
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
/* *INDENT-OFF* */
/* clang-format off */
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \
defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
#include <endian.h>
#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
#include <machine/endian.h>
#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
#include <sys/isa_defs.h>
#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
(__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
#include <sys/endian.h>
#include <sys/types.h>
#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
defined(__NETBSD__) || defined(__NetBSD__) || \
defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
#include <sys/param.h>
#endif /* OS */
/* *INDENT-ON* */
/* clang-format on */
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
#define __BYTE_ORDER__ __BYTE_ORDER
#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
#define __BYTE_ORDER__ _BYTE_ORDER
#else
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || \
(defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
defined(__WINDOWS__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || \
(defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
#error __BYTE_ORDER__ should be defined.
#endif /* Arch */
#endif
#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
/*----------------------------------------------------------------------------*/
/* Memory/Compiler barriers, cache coherence */
static __maybe_unused __inline void mdbx_compiler_barrier(void) {
#if defined(__clang__) || defined(__GNUC__)
__asm__ __volatile__("" ::: "memory");
#elif defined(_MSC_VER)
_ReadWriteBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
__memory_barrier();
if (type > MDBX_BARRIER_COMPILER)
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__compiler_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__fence();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
static __maybe_unused __inline void mdbx_memory_barrier(void) {
#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__clang__) || defined(__GNUC__)
__sync_synchronize();
#elif defined(_MSC_VER)
MemoryBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
/*----------------------------------------------------------------------------*/
/* Cache coherence and invalidation */
#ifndef MDBX_CPU_WRITEBACK_IS_COHERENT
#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
defined(__hppa__)
#define MDBX_CPU_WRITEBACK_IS_COHERENT 1
#else
#define MDBX_CPU_WRITEBACK_IS_COHERENT 0
#endif
#endif /* MDBX_CPU_WRITEBACK_IS_COHERENT */
#ifndef MDBX_CACHELINE_SIZE
#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
#define MDBX_CACHELINE_SIZE 128
#else
#define MDBX_CACHELINE_SIZE 64
#endif
#endif /* MDBX_CACHELINE_SIZE */
#if MDBX_CPU_WRITEBACK_IS_COHERENT
#define mdbx_flush_noncoherent_cpu_writeback() mdbx_compiler_barrier()
#else
#define mdbx_flush_noncoherent_cpu_writeback() mdbx_memory_barrier()
#endif
#if __has_include(<sys/cachectl.h>)
#include <sys/cachectl.h>
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
/* MIPS should have explicit cache control */
#include <sys/cachectl.h>
#endif
#ifndef MDBX_CPU_CACHE_MMAP_NONCOHERENT
#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
/* MIPS has cache coherency issues. */
#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 1
#else
/* LY: assume no relevant mmap/dcache issues. */
#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 0
#endif
#endif /* ndef MDBX_CPU_CACHE_MMAP_NONCOHERENT */
static __maybe_unused __inline void
mdbx_invalidate_mmap_noncoherent_cache(void *addr, size_t nbytes) {
#if MDBX_CPU_CACHE_MMAP_NONCOHERENT
#ifdef DCACHE
/* MIPS has cache coherency issues.
* Note: for any nbytes >= on-chip cache size, entire is flushed. */
cacheflush(addr, nbytes, DCACHE);
#else
#error "Oops, cacheflush() not available"
#endif /* DCACHE */
#else /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */
(void)addr;
(void)nbytes;
#endif /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */
}
/*----------------------------------------------------------------------------*/
/* libc compatibility stuff */
#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
(defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
#define mdbx_asprintf asprintf
#define mdbx_vasprintf vasprintf
#else
MDBX_INTERNAL_FUNC __printf_args(2, 3) int __maybe_unused
mdbx_asprintf(char **strp, const char *fmt, ...);
MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
#endif
/*----------------------------------------------------------------------------*/
/* OS abstraction layer stuff */
/* max bytes to write in one call */
#if defined(_WIN32) || defined(_WIN64)
#define MAX_WRITE UINT32_C(0x01000000)
#else
#define MAX_WRITE UINT32_C(0x3fff0000)
#endif
#if defined(__linux__) || defined(__gnu_linux__)
MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
#endif /* Linux */
/* Get the size of a memory page for the system.
* This is the basic size that the platform's memory manager uses, and is
* fundamental to the use of memory-mapped files. */
static __maybe_unused __inline size_t mdbx_syspagesize(void) {
#if defined(_WIN32) || defined(_WIN64)
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwPageSize;
#else
return sysconf(_SC_PAGE_SIZE);
#endif
}
#ifndef mdbx_strdup
LIBMDBX_API char *mdbx_strdup(const char *str);
#endif
static __maybe_unused __inline int mdbx_get_errno(void) {
#if defined(_WIN32) || defined(_WIN64)
DWORD rc = GetLastError();
#else
int rc = errno;
#endif
return rc;
}
#ifndef mdbx_memalign_alloc
MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
void **result);
#endif
#ifndef mdbx_memalign_free
MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
#endif
MDBX_INTERNAL_FUNC int mdbx_condmutex_init(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_lock(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_unlock(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_signal(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_wait(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_destroy(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
int iovcnt, uint64_t offset,
size_t expected_written);
MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
uint64_t offset);
MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
size_t count, uint64_t offset);
MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
size_t count);
MDBX_INTERNAL_FUNC int
mdbx_thread_create(mdbx_thread_t *thread,
THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg);
MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
enum mdbx_syncmode_bits {
MDBX_SYNC_DATA = 1,
MDBX_SYNC_SIZE = 2,
MDBX_SYNC_IODQ = 4
};
MDBX_INTERNAL_FUNC int mdbx_filesync(mdbx_filehandle_t fd,
enum mdbx_syncmode_bits mode_bits);
MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
MDBX_INTERNAL_FUNC int mdbx_openfile(const char *pathname, int flags,
mode_t mode, mdbx_filehandle_t *fd,
bool exclusive);
MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
typedef struct mdbx_mmap_param {
union {
void *address;
uint8_t *dxb;
struct MDBX_lockinfo *lck;
};
mdbx_filehandle_t fd;
size_t limit; /* mapping length, but NOT a size of file nor DB */
size_t current; /* mapped region size, i.e. the size of file and DB */
#if defined(_WIN32) || defined(_WIN64)
uint64_t filesize /* in-process cache of a file size. */;
#endif
#ifdef MDBX_OSAL_SECTION
MDBX_OSAL_SECTION section;
#endif
} mdbx_mmap_t;
MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
const size_t must, const size_t limit,
const bool truncate);
MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current,
size_t wanna);
#if defined(_WIN32) || defined(_WIN64)
typedef struct {
unsigned limit, count;
HANDLE handles[31];
} mdbx_handle_array_t;
MDBX_INTERNAL_FUNC int
mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
MDBX_INTERNAL_FUNC int
mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
#endif /* Windows */
MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
size_t length, int async);
MDBX_INTERNAL_FUNC int mdbx_check4nonlocal(mdbx_filehandle_t handle, int flags);
static __maybe_unused __inline uint32_t mdbx_getpid(void) {
STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentProcessId();
#else
return getpid();
#endif
}
static __maybe_unused __inline size_t mdbx_thread_self(void) {
STATIC_ASSERT(sizeof(mdbx_tid_t) <= sizeof(size_t));
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentThreadId();
#else
return (size_t)pthread_self();
#endif
}
MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny);
MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
MDBX_INTERNAL_FUNC uint64_t
mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
typedef union bin128 {
__anonymous_struct_extension__ struct { uint64_t x, y; };
__anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
} bin128_t;
MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
/*----------------------------------------------------------------------------*/
/* lck stuff */
#if defined(_WIN32) || defined(_WIN64)
#undef MDBX_OSAL_LOCK
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0xF10C)
#else
#define MDBX_OSAL_LOCK pthread_mutex_t
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0x8017)
#endif /* MDBX_OSAL_LOCK */
/// \brief Initialization of synchronization primitives linked with MDBX_env
/// instance both in LCK-file and within the current process.
/// \param
/// global_uniqueness_flag = true - denotes that there are no other processes
/// working with DB and LCK-file. Thus the function MUST initialize
/// shared synchronization objects in memory-mapped LCK-file.
/// global_uniqueness_flag = false - denotes that at least one process is
/// already working with DB and LCK-file, including the case when DB
/// has already been opened in the current process. Thus the function
/// MUST NOT initialize shared synchronization objects in memory-mapped
/// LCK-file that are already in use.
/// \return Error code or zero on success.
MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
MDBX_env *inprocess_neighbor,
int global_uniqueness_flag);
/// \brief Disconnects from shared interprocess objects and destructs
/// synchronization objects linked with MDBX_env instance
/// within the current process.
/// \param
/// inprocess_neighbor = NULL - if the current process does not have other
/// instances of MDBX_env linked with the DB being closed.
/// Thus the function MUST check for other processes working with DB or
/// LCK-file, and keep or destroy shared synchronization objects in
/// memory-mapped LCK-file depending on the result.
/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
/// (anyone of there is several) working with DB or LCK-file within the
/// current process. Thus the function MUST NOT try to acquire exclusive
/// lock and/or try to destruct shared synchronization objects linked with
/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
/// of other instances of MDBX_env within the current process, e.g.
/// restore POSIX-fcntl locks after the closing of file descriptors.
/// \return Error code (MDBX_PANIC) or zero on success.
MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
MDBX_env *inprocess_neighbor);
/// \brief Connects to shared interprocess locking objects and tries to acquire
/// the maximum lock level (shared if exclusive is not available)
/// Depending on implementation or/and platform (Windows) this function may
/// acquire the non-OS super-level lock (e.g. for shared synchronization
/// objects initialization), which will be downgraded to OS-exclusive or
/// shared via explicit calling of mdbx_lck_downgrade().
/// \return
/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
/// the current process is the first and only after the last use of DB.
/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
/// DB has already been opened and now is used by other processes.
/// Otherwise (not 0 and not -1) - error code.
MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
/// \brief Downgrades the level of initially acquired lock to
/// operational level specified by agrument. The reson for such downgrade:
/// - unblocking of other processes that are waiting for access, i.e.
/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
/// should be made aware that access is unavailable rather than
/// wait for it.
/// - freeing locks that interfere file operation (expecially for Windows)
/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
/// operational lock.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
/// \brief Locks LCK-file or/and table of readers for (de)registering.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
/// \brief Acquires lock for DB change (on writing transaction start)
/// Reading transactions will not be blocked.
/// Declared as LIBMDBX_API because it is used in mdbx_chk.
/// \return Error code or zero on success
LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dontwait);
/// \brief Releases lock once DB changes is made (after writing transaction
/// has finished).
/// Declared as LIBMDBX_API because it is used in mdbx_chk.
LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
/// the current process. The function does no more than needed for
/// the correct working of mdbx_rpid_check() in other processes.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
/// \brief Resets alive-flag of reader presence (indicative lock)
/// for PID of the current process. The function does no more than needed
/// for the correct working of mdbx_rpid_check() in other processes.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
/// \brief Checks for reading process status with the given pid with help of
/// alive-flag of presence (indicative lock) or using another way.
/// \return
/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
/// and working with DB (indicative lock is present).
/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
/// or not working with DB (indicative lock is not present).
/// Otherwise (not 0 and not -1) - error code.
MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
#if defined(_WIN32) || defined(_WIN64)
typedef union MDBX_srwlock {
struct {
long volatile readerCount;
long volatile writerCount;
};
RTL_SRWLOCK native;
} MDBX_srwlock;
typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
mdbx_GetFileInformationByHandleEx;
typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
_In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
_In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
_Out_opt_ LPDWORD lpMaximumComponentLength,
_Out_opt_ LPDWORD lpFileSystemFlags,
_Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
mdbx_GetVolumeInformationByHandleW;
typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
_Out_ LPWSTR lpszFilePath,
_In_ DWORD cchFilePath,
_In_ DWORD dwFlags);
MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
mdbx_SetFileInformationByHandle;
typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
IN HANDLE FileHandle, IN OUT HANDLE Event,
IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
typedef struct _WIN32_MEMORY_RANGE_ENTRY {
PVOID VirtualAddress;
SIZE_T NumberOfBytes;
} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
#endif /* Windows 8.x */
typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
HANDLE hProcess, ULONG_PTR NumberOfEntries,
PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
#if 0 /* LY: unused for now */
#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81
typedef enum OFFER_PRIORITY {
VmOfferPriorityVeryLow = 1,
VmOfferPriorityLow,
VmOfferPriorityBelowNormal,
VmOfferPriorityNormal
} OFFER_PRIORITY;
#endif /* Windows 8.1 */
typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress,
SIZE_T Size);
MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory;
typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress,
SIZE_T Size);
MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory;
typedef DWORD(WINAPI *MDBX_OfferVirtualMemory(
PVOID VirtualAddress,
SIZE_T Size,
OFFER_PRIORITY Priority
);
MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory;
#endif /* unused for now */
#endif /* Windows */
/*----------------------------------------------------------------------------*/
/* Atomics */
#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \
!defined(__STDC_NO_ATOMICS__) && \
(__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
!(defined(__GNUC__) || defined(__clang__)))
#include <stdatomic.h>
#elif defined(__GNUC__) || defined(__clang__)
/* LY: nothing required */
#elif defined(_MSC_VER)
#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
#pragma warning(disable : 4133) /* 'function': incompatible types - from \
'size_t' to 'LONGLONG' */
#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
'std::size_t', possible loss of data */
#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
'long', possible loss of data */
#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
#elif defined(__APPLE__)
#include <libkern/OSAtomic.h>
#else
#error FIXME atomic-ops
#endif
/*----------------------------------------------------------------------------*/
#if defined(_MSC_VER) && _MSC_VER >= 1900
/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
* for internal format-args checker. */
#undef PRIuPTR
#undef PRIiPTR
#undef PRIdPTR
#undef PRIxPTR
#define PRIuPTR "Iu"
#define PRIiPTR "Ii"
#define PRIdPTR "Id"
#define PRIxPTR "Ix"
#define PRIuSIZE "zu"
#define PRIiSIZE "zi"
#define PRIdSIZE "zd"
#define PRIxSIZE "zx"
#endif /* fix PRI*PTR for _MSC_VER */
#ifndef PRIuSIZE
#define PRIuSIZE PRIuPTR
#define PRIiSIZE PRIiPTR
#define PRIdSIZE PRIdPTR
#define PRIxSIZE PRIxPTR
#endif /* PRI*SIZE macros for MSVC */
#ifdef _MSC_VER
#pragma warning(pop)
#endif

46
src/contrib/db/libmdbx/src/elements/version.c.in Normal file
View file

@ -0,0 +1,46 @@
/* This is CMake-template for libmdbx's version.c
******************************************************************************/
#include "internals.h"
#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || \
MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR}
#error "API version mismatch! Had `git fetch --tags` done?"
#endif
static const char sourcery[] = STRINGIFY(MDBX_BUILD_SOURCERY);
__dll_export
#ifdef __attribute_used__
__attribute_used__
#elif defined(__GNUC__) || __has_attribute(__used__)
__attribute__((__used__))
#endif
#ifdef __attribute_externally_visible__
__attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || \
__has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const mdbx_version_info mdbx_version = {
${MDBX_VERSION_MAJOR},
${MDBX_VERSION_MINOR},
${MDBX_VERSION_RELEASE},
${MDBX_VERSION_REVISION},
{"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@",
"@MDBX_GIT_DESCRIBE@"},
sourcery};
__dll_export
#ifdef __attribute_used__
__attribute_used__
#elif defined(__GNUC__) || __has_attribute(__used__)
__attribute__((__used__))
#endif
#ifdef __attribute_externally_visible__
__attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || \
__has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const char *const mdbx_sourcery_anchor = sourcery;

87
src/contrib/db/libmdbx/src/man1/mdbx_chk.1 Normal file
View file

@ -0,0 +1,87 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_CHK 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_chk \- MDBX checking tool
.SH SYNOPSIS
.B mdbx_chk
[\c
.BR \-V ]
[\c
.BR \-v [ v [ v ]]]
[\c
.BR \-n ]
[\c
.BR \-q ]
[\c
.BR \-w ]
[\c
.BR \-d ]
[\c
.BI \-s \ subdb\fR]
[\c
.BR \-c ]
[\c
.BR \-i ]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_chk
utility intended to check an MDBX database file.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-v
Produce verbose output, including summarize space and page usage statistics.
If \fB\-vv\fP is given, be more verbose, show summarized B-tree info
and space allocation.
If \fB\-vvv\fP is given, be more verbose, include summarized statistics
of leaf B-tree pages.
If \fB\-vvvv\fP is given, be even more verbose, show info of each page
during B-tree traversal and basic info of each GC record.
If \fB\-vvvvv\fP is given, turn maximal verbosity, display the full list
of page IDs in the GC records and size of each key-value pair of database(s).
.TP
.BR \-n
Open MDBX environment(s) which do not use subdirectories.
.TP
.BR \-q
Be quiet; do not output anything even if an error was detected.
.TP
.BR \-w
Open environment in read-write mode and lock for writing while checking.
This could be impossible if environment already used by another process(s)
in an incompatible read-write mode. This allow rollback to last steady commit
(in case environment was not closed properly) and then check transaction IDs
of meta-pages. Otherwise, without \fB\-w\fP option environment will be
opened in read-only mode.
.TP
.BR \-d
Disable page-by-page traversal of B-tree. In this case, without B-tree
traversal, it is unable to check for lost-unused pages nor for double-used
pages.
.TP
.BR \-s \ subdb
Verify and show info only for a specific subdatabase.
.TP
.BR \-c
Force using cooperative mode while opening environment, i.e. don't try to open
in exclusive/monopolistic mode. Only exclusive/monopolistic mode allow complete
check, including full check of all meta-pages and actual size of database file.
.TP
.BR \-i
Ignore wrong order errors, which will likely false-positive if custom
comparator(s) was used.
.SH DIAGNOSTICS
Exit status is zero if no errors occur. Errors result in a non-zero exit status
and a diagnostic message being written to standard error
if no quiet mode was requested.
.SH "SEE ALSO"
.BR mdbx_stat (1),
.BR mdbx_copy (1),
.BR mdbx_dump (1),
.BR mdbx_load (1)
.SH AUTHOR
Leonid Yuriev <https://github.com/leo-yuriev>

60
src/contrib/db/libmdbx/src/man1/mdbx_copy.1 Normal file
View file

@ -0,0 +1,60 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_COPY 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_copy \- MDBX environment copy tool
.SH SYNOPSIS
.B mdbx_copy
[\c
.BR \-V ]
[\c
.BR \-c ]
[\c
.BR \-n ]
.B srcpath
[\c
.BR dstpath ]
.SH DESCRIPTION
The
.B mdbx_copy
utility copies an MDBX environment. The environment can
be copied regardless of whether it is currently in use.
No lockfile is created, since it gets recreated at need.
If
.I dstpath
is specified it must be the path of an empty directory
for storing the backup. Otherwise, the backup will be
written to stdout.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-c
Compact while copying. Only current data pages will be copied; freed
or unused pages will be omitted from the copy. This option will
slow down the backup process as it is more CPU-intensive.
Currently it fails if the environment has suffered a page leak.
.TP
.BR \-n
Open MDBX environment(s) which do not use subdirectories.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH CAVEATS
This utility can trigger significant file size growth if run
in parallel with write transactions, because pages which they
free during copying cannot be reused until the copy is done.
.SH "SEE ALSO"
.BR mdbx_dump (1),
.BR mdbx_chk (1),
.BR mdbx_stat (1),
.BR mdbx_load (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

80
src/contrib/db/libmdbx/src/man1/mdbx_dump.1 Normal file
View file

@ -0,0 +1,80 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_DUMP 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_dump \- MDBX environment export tool
.SH SYNOPSIS
.B mdbx_dump
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-l ]
[\c
.BR \-n ]
[\c
.BR \-p ]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_dump
utility reads a database and writes its contents to the
standard output using a portable flat-text format
understood by the
.BR mdbx_load (1)
utility.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-f \ file
Write to the specified file instead of to the standard output.
.TP
.BR \-l
List the databases stored in the environment. Just the
names will be listed, no data will be output.
.TP
.BR \-n
Dump an MDBX database which does not use subdirectories.
.TP
.BR \-p
If characters in either the key or data items are printing characters (as
defined by isprint(3)), output them directly. This option permits users to
use standard text editors and tools to modify the contents of databases.
Note: different systems may have different notions about what characters
are considered printing characters, and databases dumped in this manner may
be less portable to external systems.
.TP
.BR \-a
Dump all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Dump a specific subdatabase. If no database is specified, only the main database is dumped.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
Dumping and reloading databases that use user-defined comparison functions
will result in new databases that use the default comparison functions.
\fBIn this case it is quite likely that the reloaded database will be
damaged beyond repair permitting neither record storage nor retrieval.\fP
The only available workaround is to modify the source for the
.BR mdbx_load (1)
utility to load the database using the correct comparison functions.
.SH "SEE ALSO"
.BR mdbx_load (1),
.BR mdbx_copy (1),
.BR mdbx_chk (1),
.BR mdbx_stat (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

89
src/contrib/db/libmdbx/src/man1/mdbx_load.1 Normal file
View file

@ -0,0 +1,89 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_LOAD 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_load \- MDBX environment import tool
.SH SYNOPSIS
.B mdbx_load
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-n ]
[\c
.BI \-s \ subdb\fR]
[\c
.BR \-N ]
[\c
.BR \-T ]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_load
utility reads from the standard input and loads it into the
MDBX environment
.BR envpath .
The input to
.B mdbx_load
must be in the output format specified by the
.BR mdbx_dump (1)
utility or as specified by the
.B -T
option below.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-a
Append all records in the order they appear in the input. The input is assumed to already be
in correctly sorted order and no sorting or checking for redundant values will be performed.
This option must be used to reload data that was produced by running
.B mdbx_dump
on a database that uses custom compare functions.
.TP
.BR \-f \ file
Read from the specified file instead of from the standard input.
.TP
.BR \-n
Load an MDBX database which does not use subdirectories.
.TP
.BR \-s \ subdb
Load a specific subdatabase. If no database is specified, data is loaded into the main database.
.TP
.BR \-N
Don't overwrite existing records when loading into an already existing database; just skip them.
.TP
.BR \-T
Load data from simple text files. The input must be paired lines of text, where the first
line of the pair is the key item, and the second line of the pair is its corresponding
data item.
A simple escape mechanism, where newline and backslash (\\) characters are special, is
applied to the text input. Newline characters are interpreted as record separators.
Backslash characters in the text will be interpreted in one of two ways: If the backslash
character precedes another backslash character, the pair will be interpreted as a literal
backslash. If the backslash character precedes any other character, the two characters
following the backslash will be interpreted as a hexadecimal specification of a single
character; for example, \\0a is a newline character in the ASCII character set.
For this reason, any backslash or newline characters that naturally occur in the text
input must be escaped to avoid misinterpretation by
.BR mdbx_load .
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdbx_dump (1),
.BR mdbx_chk (1),
.BR mdbx_stat (1),
.BR mdbx_copy (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

69
src/contrib/db/libmdbx/src/man1/mdbx_stat.1 Normal file
View file

@ -0,0 +1,69 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_STAT 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_stat \- MDBX environment status tool
.SH SYNOPSIS
.B mdbx_stat
[\c
.BR \-V ]
[\c
.BR \-e ]
[\c
.BR \-f [ f [ f ]]]
[\c
.BR \-n ]
[\c
.BR \-r [ r ]]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_stat
utility displays the status of an MDBX environment.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-e
Display information about the database environment.
.TP
.BR \-f
Display information about the environment freelist.
If \fB\-ff\fP is given, summarize each freelist entry.
If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
.TP
.BR \-n
Display the status of an MDBX database which does not use subdirectories.
.TP
.BR \-r
Display information about the environment reader table.
Shows the process ID, thread ID, and transaction ID for each active
reader slot. The process ID and transaction ID are in decimal, the
thread ID is in hexadecimal. The transaction ID is displayed as "-"
if the reader does not currently have a read transaction open.
If \fB\-rr\fP is given, check for stale entries in the reader
table and clear them. The reader table will be printed again
after the check is performed.
.TP
.BR \-a
Display the status of all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Display the status of a specific subdatabase.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdbx_chk (1),
.BR mdbx_copy (1),
.BR mdbx_dump (1),
.BR mdbx_load (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

42
src/contrib/db/libmdbx/src/tools/CMakeLists.txt Normal file
View file

@ -0,0 +1,42 @@
set(MDBX_TOOLS mdbx_chk mdbx_copy mdbx_dump mdbx_load mdbx_stat)
# use, i.e. don't skip the full RPATH for the build tree
set(CMAKE_SKIP_BUILD_RPATH FALSE)
# when building, don't use the install RPATH already (but later on when installing)
set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
# add the automatically determined parts of the RPATH
# which point to directories outside the build tree to the install RPATH
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
# the RPATH to be used when installing, but only if it's not a system directory
list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
if(isSystemDir EQUAL -1)
if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
else()
set(CMAKE_INSTALL_RPATH "\$ORIGIN/../lib")
endif()
endif()
foreach(TOOL ${MDBX_TOOLS})
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
add_executable(${TOOL} ${TOOL}.c wingetopt.c wingetopt.h)
else()
add_executable(${TOOL} ${TOOL}.c)
endif()
target_link_libraries(${TOOL} mdbx ${CMAKE_THREAD_LIBS_INIT})
set_target_properties(${TOOL} PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
install(TARGETS ${TOOL} DESTINATION ${CMAKE_INSTALL_PREFIX}/bin COMPONENT mdbx)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../man1/${TOOL}.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/man/man1 COMPONENT mdbx)
endforeach()
if(LIB_MATH)
target_link_libraries(mdbx_chk ${LIB_MATH})
target_link_libraries(mdbx_stat ${LIB_MATH})
endif()

1430
src/contrib/db/libmdbx/src/tools/mdbx_chk.c Normal file
View file

File diff suppressed because it is too large Load diff

130
src/contrib/db/libmdbx/src/tools/mdbx_copy.c Normal file
View file

@ -0,0 +1,130 @@
/* mdbx_copy.c - memory-mapped database backup tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
int main(int argc, char *argv[]) {
int rc;
MDBX_env *env = NULL;
const char *progname = argv[0], *act;
unsigned flags = MDBX_RDONLY;
unsigned cpflags = 0;
bool quiet = false;
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDBX_NOSUBDIR;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDBX_CP_COMPACT;
else if (argv[1][1] == 'q' && argv[1][2] == '\0')
quiet = true;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("mdbx_copy version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
} else
argc = 0;
}
if (argc < 2 || argc > 3) {
fprintf(stderr, "usage: %s [-V] [-q] [-c] [-n] srcpath [dstpath]\n",
progname);
exit(EXIT_FAILURE);
}
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
if (!quiet) {
fprintf((argc == 2) ? stderr : stdout,
"mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]);
fflush(NULL);
}
act = "opening environment";
rc = mdbx_env_create(&env);
if (rc == MDBX_SUCCESS) {
rc = mdbx_env_open(env, argv[1], flags, 0640);
}
if (rc == MDBX_SUCCESS) {
act = "copying";
if (argc == 2) {
mdbx_filehandle_t fd;
#if defined(_WIN32) || defined(_WIN64)
fd = GetStdHandle(STD_OUTPUT_HANDLE);
#else
fd = fileno(stdout);
#endif
rc = mdbx_env_copy2fd(env, fd, cpflags);
} else
rc = mdbx_env_copy(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc,
mdbx_strerror(rc));
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

352
src/contrib/db/libmdbx/src/tools/mdbx_dump.c Normal file
View file

@ -0,0 +1,352 @@
/* mdbx_dump.c - memory-mapped database dump tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#include <ctype.h>
#define PRINT 1
static int mode;
typedef struct flagbit {
int bit;
char *name;
} flagbit;
flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"},
{MDBX_DUPSORT, "dupsort"},
{MDBX_INTEGERKEY, "integerkey"},
{MDBX_DUPFIXED, "dupfixed"},
{MDBX_INTEGERDUP, "integerdup"},
{MDBX_REVERSEDUP, "reversedup"},
{0, NULL}};
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static const char hexc[] = "0123456789abcdef";
static void dumpbyte(unsigned char c) {
putchar(hexc[c >> 4]);
putchar(hexc[c & 0xf]);
}
static void text(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
if (isprint(*c) && *c != '\\') {
putchar(*c);
} else {
putchar('\\');
dumpbyte(*c);
}
c++;
}
putchar('\n');
}
static void dumpval(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
dumpbyte(*c++);
}
putchar('\n');
}
/* Dump in BDB-compatible format */
static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
MDBX_cursor *mc;
MDBX_stat ms;
MDBX_val key, data;
MDBX_envinfo info;
unsigned int flags;
int rc, i;
rc = mdbx_dbi_flags(txn, dbi, &flags);
if (rc)
return rc;
rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms));
if (rc)
return rc;
rc = mdbx_env_info(mdbx_txn_env(txn), &info, sizeof(info));
if (rc)
return rc;
printf("VERSION=3\n");
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%" PRIu64 "\n", info.mi_mapsize);
printf("maxreaders=%u\n", info.mi_maxreaders);
for (i = 0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
printf("%s=1\n", dbflags[i].name);
printf("db_pagesize=%d\n", ms.ms_psize);
printf("HEADER=END\n");
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc)
return rc;
while ((rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
if (mode & PRINT) {
text(&key);
text(&data);
} else {
dumpval(&key);
dumpval(&data);
}
}
printf("DATA=END\n");
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
return rc;
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envflags = 0, list = 0;
if (argc < 2)
usage(prog);
/* -a: dump main DB and all subDBs
* -s: dump only the named subDB
* -n: use NOSUBDIR flag on env_open
* -p: use printable characters
* -f: write to file instead of stdout
* -V: print version and exit
* (default) dump only the main DB
*/
while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) {
switch (i) {
case 'V':
printf("mdbx_dump version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
case 'l':
list = 1;
/*FALLTHROUGH*/;
__fallthrough;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'f':
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'p':
mode |= PRINT;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
printf("mdbx_dump %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(NULL);
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdbx_env_set_maxdbs(env, 2);
}
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
int count = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
count++;
str = mdbx_malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS) {
if (list) {
printf("%s\n", str);
list++;
} else {
rc = dumpit(txn, db2, str);
if (rc)
break;
}
mdbx_dbi_close(env, db2);
}
mdbx_free(str);
if (rc)
continue;
}
mdbx_cursor_close(cursor);
if (!count) {
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog,
envname);
rc = MDBX_NOTFOUND;
} else if (rc == MDBX_INCOMPATIBLE) {
/* LY: the record it not a named sub-db. */
rc = MDBX_SUCCESS;
}
} else {
rc = dumpit(txn, dbi, subname);
}
if (rc && rc != MDBX_NOTFOUND)
fprintf(stderr, "%s: %s: %s\n", prog, envname, mdbx_strerror(rc));
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

567
src/contrib/db/libmdbx/src/tools/mdbx_load.c Normal file
View file

@ -0,0 +1,567 @@
/* mdbx_load.c - memory-mapped database load tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#include <ctype.h>
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
#define PRINT 1
#define NOHDR 2
static int mode;
static char *subname = NULL;
static size_t lineno;
static int version;
static int dbi_flags;
static char *prog;
static int Eof;
static MDBX_envinfo envinfo;
static MDBX_val kbuf, dbuf;
static MDBX_val k0buf;
#define STRLENOF(s) (sizeof(s) - 1)
typedef struct flagbit {
int bit;
char *name;
int len;
} flagbit;
#define S(s) s, STRLENOF(s)
flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")},
{MDBX_DUPSORT, S("dupsort")},
{MDBX_INTEGERKEY, S("integerkey")},
{MDBX_DUPFIXED, S("dupfixed")},
{MDBX_INTEGERDUP, S("integerdup")},
{MDBX_REVERSEDUP, S("reversedup")},
{0, NULL, 0}};
static void readhdr(void) {
char *ptr;
dbi_flags = 0;
while (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
lineno++;
if (!strncmp(dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) ||
!strncmp(dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
/* LY: silently ignore information fields. */
continue;
} else if (!strncmp(dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n",
prog, lineno, version);
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) {
break;
} else if (!strncmp(dbuf.iov_base, "format=", STRLENOF("format="))) {
if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print",
STRLENOF("print")))
mode |= PRINT;
else if (strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "bytevalue",
STRLENOF("bytevalue"))) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported FORMAT %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "database=", STRLENOF("database="))) {
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
if (subname)
mdbx_free(subname);
subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database="));
} else if (!strncmp(dbuf.iov_base, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree",
STRLENOF("btree"))) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("type="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
void *unused;
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapaddr="), "%p", &unused);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapaddr %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapsize="), "%" PRIu64,
&envinfo.mi_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapsize %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base,
"maxreaders=", STRLENOF("maxreaders="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("maxreaders="), "%u",
&envinfo.mi_maxreaders);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid maxreaders %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
} else {
int i;
for (i = 0; dbflags[i].bit; i++) {
if (!strncmp(dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
dbi_flags |= dbflags[i].bit;
break;
}
}
if (!dbflags[i].bit) {
ptr = memchr(dbuf.iov_base, '=', dbuf.iov_len);
if (!ptr) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog,
lineno);
exit(EXIT_FAILURE);
} else {
*ptr = '\0';
fprintf(stderr,
"%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.iov_base);
}
}
}
}
}
static void badend(void) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog,
lineno);
}
static int unhex(unsigned char *c2) {
int x, c;
x = *c2++ & 0x4f;
if (x & 0x40)
x -= 55;
c = x << 4;
x = *c2 & 0x4f;
if (x & 0x40)
x -= 55;
c |= x;
return c;
}
static int readline(MDBX_val *out, MDBX_val *buf) {
unsigned char *c1, *c2, *end;
size_t len, l2;
int c;
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
Eof = 1;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
badend:
Eof = 1;
badend();
return EOF;
}
if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END")))
return EOF;
goto badend;
}
}
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
Eof = 1;
return EOF;
}
lineno++;
c1 = buf->iov_base;
len = strlen((char *)c1);
l2 = len;
/* Is buffer too short? */
while (c1[len - 1] != '\n') {
buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2);
if (!buf->iov_base) {
Eof = 1;
fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n",
prog, lineno);
return EOF;
}
c1 = buf->iov_base;
c1 += l2;
if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) {
Eof = 1;
badend();
return EOF;
}
buf->iov_len *= 2;
len = strlen((char *)c1);
l2 += len;
}
c1 = c2 = buf->iov_base;
len = l2;
c1[--len] = '\0';
end = c1 + len;
if (mode & PRINT) {
while (c2 < end) {
if (unlikely(*c2 == '\\')) {
if (c2[1] == '\\') {
*c1++ = '\\';
} else {
if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(++c2);
}
c2 += 2;
} else {
/* copies are redundant when no escapes were used */
*c1++ = *c2++;
}
}
} else {
/* odd length not allowed */
if (len & 1) {
Eof = 1;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(c2);
c2 += 2;
}
}
c2 = out->iov_base = buf->iov_base;
out->iov_len = c1 - c2;
return 0;
}
static void usage(void) {
fprintf(stderr,
"usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
static int anyway_greater(const MDBX_val *a, const MDBX_val *b) {
(void)a;
(void)b;
return 1;
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env = NULL;
MDBX_txn *txn = NULL;
MDBX_cursor *mc = NULL;
MDBX_dbi dbi;
char *envname = NULL;
int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0;
int append = 0;
MDBX_val prevk;
prog = argv[0];
if (argc < 2)
usage();
/* -a: append records in input order
* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
switch (i) {
case 'V':
printf("mdbx_load version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
case 'a':
append = 1;
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 's':
subname = mdbx_strdup(optarg);
break;
case 'N':
putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(NULL);
dbuf.iov_len = 4096;
dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
/* read first header for mapsize= */
if (!(mode & NOHDR))
readhdr();
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
mdbx_env_set_maxdbs(env, 2);
if (envinfo.mi_maxreaders)
mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
if (envinfo.mi_mapsize) {
if (envinfo.mi_mapsize > SIZE_MAX) {
fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc,
mdbx_strerror(MDBX_TOO_LARGE));
return EXIT_FAILURE;
}
mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize);
}
#ifdef MDBX_FIXEDMAP
if (info.mi_mapaddr)
envflags |= MDBX_FIXEDMAP;
#endif
rc = mdbx_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
kbuf.iov_len = mdbx_env_get_maxkeysize(env);
if (kbuf.iov_len >= SIZE_MAX / 4) {
fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n",
kbuf.iov_len);
goto env_close;
}
kbuf.iov_len = (kbuf.iov_len + 1) * 2;
kbuf.iov_base = malloc(kbuf.iov_len * 2);
k0buf.iov_len = kbuf.iov_len;
k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len;
prevk.iov_base = k0buf.iov_base;
while (!Eof) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
append ? anyway_greater : NULL,
append ? anyway_greater : NULL);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
int batch = 0;
prevk.iov_len = 0;
while (1) {
MDBX_val key;
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
MDBX_val data;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
prog, lineno);
goto txn_abort;
}
int appflag = 0;
if (append) {
appflag = MDBX_APPEND;
if (dbi_flags & MDBX_DUPSORT) {
if (prevk.iov_len == key.iov_len &&
memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0)
appflag = MDBX_APPEND | MDBX_APPENDDUP;
else
memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len);
}
}
rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag);
if (rc == MDBX_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog,
lineno, mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdbx_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog, lineno,
mdbx_strerror(rc));
goto env_close;
}
mdbx_dbi_close(env, dbi);
/* try read next header */
if (!(mode & NOHDR))
readhdr();
}
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

436
src/contrib/db/libmdbx/src/tools/mdbx_stat.c Normal file
View file

@ -0,0 +1,436 @@
/* mdbx_stat.c - memory-mapped database status tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static void prstat(MDBX_stat *ms) {
printf(" Pagesize: %u\n", ms->ms_psize);
printf(" Tree depth: %u\n", ms->ms_depth);
printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages);
printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages);
printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages);
printf(" Entries: %" PRIu64 "\n", ms->ms_entries);
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid,
mdbx_tid_t thread, uint64_t txnid, uint64_t lag,
size_t bytes_used, size_t bytes_retained) {
(void)ctx;
if (num == 1)
printf("Reader Table Status\n"
" #\tslot\t%6s %*s %20s %10s %13s %13s\n",
"pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used",
"retained");
printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxSIZE, num, slot, (size_t)pid,
(int)sizeof(size_t) * 2, (size_t)thread);
if (txnid)
printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag,
bytes_used / 1048576.0, bytes_retained / 1048576.0);
else
printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0");
return user_break ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
}
int main(int argc, char *argv[]) {
int o, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
MDBX_stat mst;
MDBX_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2)
usage(prog);
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((o = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch (o) {
case 'V':
printf("mdbx_stat version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
envname = argv[optind];
printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(NULL);
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname)
mdbx_env_set_maxdbs(env, 4);
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (envinfo || freinfo) {
(void)mdbx_env_info(env, &mei, sizeof(mei));
} else {
/* LY: zap warnings from gcc */
memset(&mei, 0, sizeof(mei));
}
if (envinfo) {
(void)mdbx_env_stat(env, &mst, sizeof(mst));
printf("Environment Info\n");
printf(" Pagesize: %u\n", mst.ms_psize);
if (mei.mi_geo.lower != mei.mi_geo.upper) {
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
"/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
"/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize,
mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize,
mei.mi_geo.shrink / mst.ms_psize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
} else {
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
}
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize);
printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1);
printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
mei.mi_latter_reader_txnid,
mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
printf(" Max readers: %u\n", mei.mi_maxreaders);
printf(" Number of readers used: %u\n", mei.mi_numreaders);
} else {
/* LY: zap warnings from gcc */
memset(&mst, 0, sizeof(mst));
}
if (rdrinfo) {
rc = mdbx_reader_list(env, reader_list_func, nullptr);
if (rc == MDBX_RESULT_TRUE)
printf("Reader Table is empty\n");
else if (rc == MDBX_SUCCESS && rdrinfo > 1) {
int dead;
rc = mdbx_reader_check(env, &dead);
if (rc == MDBX_RESULT_TRUE) {
printf(" %d stale readers cleared.\n", dead);
rc = mdbx_reader_list(env, reader_list_func, nullptr);
if (rc == MDBX_RESULT_TRUE)
printf(" Now Reader Table is empty\n");
} else
printf(" No stale readers.\n");
}
if (MDBX_IS_ERROR(rc)) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (freinfo) {
MDBX_cursor *cursor;
MDBX_val key, data;
pgno_t pages = 0, *iptr;
pgno_t reclaimable = 0;
printf("Freelist Status\n");
dbi = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) ==
MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
iptr = data.iov_base;
const pgno_t number = *iptr++;
pages += number;
if (envinfo && mei.mi_latter_reader_txnid > *(size_t *)key.iov_base)
reclaimable += number;
if (freinfo > 1) {
char *bad = "";
pgno_t prev =
MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1;
pgno_t span = 1;
for (unsigned i = 0; i < number; ++i) {
pgno_t pg = iptr[i];
if (MDBX_PNL_DISORDERED(prev, pg))
bad = " [bad sequence]";
prev = pg;
while (i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span)))
++span;
}
printf(" Transaction %" PRIaTXN ", %" PRIaPGNO
" pages, maxspan %" PRIaPGNO "%s\n",
*(txnid_t *)key.iov_base, number, span, bad);
if (freinfo > 2) {
for (unsigned i = 0; i < number; i += span) {
const pgno_t pg = iptr[i];
for (span = 1;
i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span));
++span)
;
if (span > 1)
printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span);
else
printf(" %9" PRIaPGNO "\n", pg);
}
}
}
}
mdbx_cursor_close(cursor);
switch (rc) {
case MDBX_SUCCESS:
case MDBX_NOTFOUND:
break;
case MDBX_EINTR:
fprintf(stderr, "Interrupted by signal/user\n");
goto txn_abort;
default:
fprintf(stderr, "mdbx_cursor_get failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
if (envinfo) {
uint64_t value = mei.mi_mapsize / mst.ms_psize;
double percent = value / 100.0;
printf("Page Allocation Info\n");
printf(" Max pages: %" PRIu64 " 100%%\n", value);
value = mei.mi_last_pgno + 1;
printf(" Pages used: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1);
printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_last_pgno + 1 - pages;
printf(" Used now: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages;
printf(" Unallocated: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages - reclaimable;
printf(" Detained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = reclaimable;
printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent);
value =
mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1) + reclaimable;
printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent);
} else
printf(" Free pages: %" PRIaPGNO "\n", pages);
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
str = mdbx_malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS)
printf("Status of %s\n", str);
mdbx_free(str);
if (rc)
continue;
rc = mdbx_dbi_stat(txn, db2, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdbx_dbi_close(env, db2);
}
mdbx_cursor_close(cursor);
}
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

95
src/contrib/db/libmdbx/src/tools/wingetopt.c Normal file
View file

@ -0,0 +1,95 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
/*----------------------------------------------------------------------------*/
/* Microsoft compiler generates a lot of warning for self includes... */
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif
#endif /* _MSC_VER (warnings) */
#include "wingetopt.h"
#include <stdio.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(pop)
#endif
/*----------------------------------------------------------------------------*/
#ifndef NULL
#define NULL 0
#endif
#ifndef EOF
#define EOF (-1)
#endif
#define ERR(s, c) \
if (opterr) { \
fputs(argv[0], stderr); \
fputs(s, stderr); \
fputc(c, stderr); \
}
int opterr = 1;
int optind = 1;
int optopt;
char *optarg;
int getopt(int argc, char *const argv[], const char *opts) {
static int sp = 1;
int c;
const char *cp;
if (sp == 1) {
if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
return EOF;
else if (strcmp(argv[optind], "--") == 0) {
optind++;
return EOF;
}
}
optopt = c = argv[optind][sp];
if (c == ':' || (cp = strchr(opts, c)) == NULL) {
ERR(": illegal option -- ", c);
if (argv[optind][++sp] == '\0') {
optind++;
sp = 1;
}
return '?';
}
if (*++cp == ':') {
if (argv[optind][sp + 1] != '\0')
optarg = &argv[optind++][sp + 1];
else if (++optind >= argc) {
ERR(": option requires an argument -- ", c);
sp = 1;
return '?';
} else
optarg = argv[optind++];
sp = 1;
} else {
if (argv[optind][++sp] == '\0') {
sp = 1;
optind++;
}
optarg = NULL;
}
return c;
}

30
src/contrib/db/libmdbx/src/tools/wingetopt.h Normal file
View file

@ -0,0 +1,30 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
#ifndef _WINGETOPT_H_
#define _WINGETOPT_H_
/* Bit of madness for Windows console */
#define mdbx_strerror mdbx_strerror_ANSI2OEM
#define mdbx_strerror_r mdbx_strerror_r_ANSI2OEM
#ifdef __cplusplus
extern "C" {
#endif
extern int opterr;
extern int optind;
extern int optopt;
extern char *optarg;
int getopt(int argc, char *const argv[], const char *optstring);
#ifdef __cplusplus
}
#endif
#endif /* _GETOPT_H_ */

53
src/contrib/db/libmdbx/test/CMakeLists.txt Normal file
View file

@ -0,0 +1,53 @@
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
set(TEST_OSAL windows)
else()
set(TEST_OSAL unix)
endif()
add_executable(mdbx_test
base.h
cases.cc
chrono.cc
chrono.h
config.cc
config.h
copy.cc
dead.cc
hill.cc
jitter.cc
keygen.cc
keygen.h
log.cc
log.h
main.cc
osal.h
osal-${TEST_OSAL}.cc
test.cc
test.h
try.cc
utils.cc
utils.h
append.cc
ttl.cc
nested.cc
)
set_target_properties(mdbx_test PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>
CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON)
if(CC_HAS_FASTMATH)
target_compile_options(mdbx_test PRIVATE "-ffast-math")
endif()
if(CC_HAS_VISIBILITY AND (LTO_ENABLED OR INTERPROCEDURAL_OPTIMIZATION))
set_target_properties(mdbx_test PROPERTIES LINK_FLAGS "-fvisibility=hidden")
endif()
target_link_libraries(mdbx_test mdbx ${LIB_MATH} ${CMAKE_THREAD_LIBS_INIT})
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
target_link_libraries(mdbx_test winmm.lib)
endif()
if(UNIX AND NOT SUBPROJECT)
add_subdirectory(pcrf)
endif()

164
src/contrib/db/libmdbx/test/append.cc Normal file
View file

@ -0,0 +1,164 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_append::run() {
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
/* LY: тест наполнения таблиц в append-режиме,
* при котором записи добавляются строго в конец (в порядке сортировки) */
const unsigned flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_APPEND | MDBX_APPENDDUP
: MDBX_APPEND;
keyvalue_maker.make_ordered();
key = keygen::alloc(config.params.keylen_max);
data = keygen::alloc(config.params.datalen_max);
keygen::buffer last_key = keygen::alloc(config.params.keylen_max);
keygen::buffer last_data = keygen::alloc(config.params.datalen_max);
last_key->value.iov_base = last_key->bytes;
last_key->value.iov_len = 0;
last_data->value.iov_base = last_data->bytes;
last_data->value.iov_len = 0;
simple_checksum inserted_checksum;
uint64_t inserted_number = 0;
uint64_t serial_count = 0;
unsigned txn_nops = 0;
uint64_t commited_inserted_number = inserted_number;
simple_checksum commited_inserted_checksum = inserted_checksum;
while (should_continue()) {
const keygen::serial_t serial = serial_count;
if (!keyvalue_maker.increment(serial_count, 1)) {
// дошли до границы пространства ключей
break;
}
log_trace("append: append-a %" PRIu64, serial);
generate_pair(serial, key, data);
int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value,
&last_key->value)
: 1;
if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT))
cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value);
err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags);
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("append: bailout-insert due '%s'", mdbx_strerror(err));
txn_end(true);
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
if (cmp > 0) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_put(appenda-a)", err);
memcpy(last_key->value.iov_base, key->value.iov_base,
last_key->value.iov_len = key->value.iov_len);
memcpy(last_data->value.iov_base, data->value.iov_base,
last_data->value.iov_len = data->value.iov_len);
++inserted_number;
inserted_checksum.push((uint32_t)inserted_number, key->value);
inserted_checksum.push(10639, data->value);
} else {
if (unlikely(err != MDBX_EKEYMISMATCH))
failure_perror("mdbx_put(appenda-a) != MDBX_EKEYMISMATCH", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
commited_inserted_number = inserted_number;
commited_inserted_checksum = inserted_checksum;
txn_nops = 0;
}
report(1);
}
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
}
}
//----------------------------------------------------------------------------
txn_begin(true);
cursor_open(dbi);
MDBX_val check_key, check_data;
err =
mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST);
if (likely(inserted_number)) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_get(MDBX_FIRST)", err);
}
simple_checksum read_checksum;
uint64_t read_count = 0;
while (err == MDBX_SUCCESS) {
++read_count;
read_checksum.push((uint32_t)read_count, check_key);
read_checksum.push(10639, check_data);
err =
mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_NEXT);
}
if (unlikely(err != MDBX_NOTFOUND))
failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err);
if (unlikely(read_count != inserted_number))
failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")",
read_count, inserted_number);
if (unlikely(read_checksum.value != inserted_checksum.value))
failure("read_checksum(0x%016" PRIu64 ") "
"!= inserted_checksum(0x%016" PRIu64 ")",
read_checksum.value, inserted_checksum.value);
cursor_close();
txn_end(true);
//----------------------------------------------------------------------------
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}
return true;
}

116
src/contrib/db/libmdbx/test/base.h Normal file
View file

@ -0,0 +1,116 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#ifndef NOMINMAX
#define NOMINMAX
#endif
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#ifdef _MSC_VER
#define _CRT_SECURE_NO_WARNINGS
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
mode specified; termination on exception \
is not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
/* If you wish to build your application for a previous Windows platform,
* include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you
* wish to support before including SDKDDKVer.h.
*
* TODO: #define _WIN32_WINNT WIN32_MUSTDIE */
#include <SDKDDKVer.h>
#endif /* WINDOWS */
#ifdef __APPLE__
#define _DARWIN_C_SOURCE
#endif
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#include <io.h>
#else
#include <fcntl.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#endif
#ifdef _BSD_SOURCE
#include <endian.h>
#endif
#include <algorithm>
#include <cassert>
#include <cinttypes> // for PRId64, PRIu64
#include <cstdarg>
#include <cstddef>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#define MDBX_INTERNAL_FUNC
#define MDBX_INTERNAL_VAR extern
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../mdbx.h"
#include "../src/elements/defs.h"
#include "../src/elements/osal.h"
#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__))
#define __thread __declspec(thread)
#endif /* __thread */
#ifdef _MSC_VER
#pragma warning(pop)
#pragma warning(disable : 4201) /* nonstandard extension used : \
nameless struct / union */
#pragma warning(disable : 4127) /* conditional expression is constant */
#if _MSC_VER < 1900
#pragma warning(disable : 4510) /* default constructor could \
not be generated */
#pragma warning(disable : 4512) /* assignment operator could \
not be generated */
#pragma warning(disable : 4610) /* user-defined constructor required */
#ifndef snprintf
#define snprintf(buffer, buffer_size, format, ...) \
_snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__)
#endif
#ifndef vsnprintf
#define vsnprintf(buffer, buffer_size, format, args) \
_vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args)
#endif
#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \
may be unsafe */
#endif
#endif /* _MSC_VER */

99
src/contrib/db/libmdbx/test/cases.cc Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, const actor_params &params) {
unsigned wait4id = 0;
if (params.waitfor_nops) {
for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) {
if (i->is_waitable(params.waitfor_nops)) {
if (i->signal_nops && i->signal_nops != params.waitfor_nops)
failure("Previous waitable actor (id=%u) already linked on %u-ops\n",
i->actor_id, i->signal_nops);
wait4id = i->actor_id;
i->signal_nops = params.waitfor_nops;
break;
}
}
if (!wait4id)
failure("No previous waitable actor for %u-ops\n", params.waitfor_nops);
}
unsigned space_id = 0;
if (!space_id_cstr || strcmp(space_id_cstr, "auto") == 0)
space_id = last_space_id + 1;
else {
char *end = nullptr;
errno = 0;
space_id = strtoul(space_id_cstr, &end, 0);
if (errno)
failure_perror("Expects an integer value for space-id\n", errno);
if (end && *end)
failure("The '%s' is unexpected for space-id\n", end);
}
if (space_id > ACTOR_ID_MAX)
failure("Invalid space-id %u\n", space_id);
last_space_id = space_id;
log_trace("configure_actor: space %u for %s", space_id,
testcase2str(testcase));
global::actors.emplace_back(
actor_config(testcase, params, space_id, wait4id));
global::databases.insert(params.pathname_db);
}
void testcase_setup(const char *casename, actor_params &params,
unsigned &last_space_id) {
if (strcmp(casename, "basic") == 0) {
log_notice(">>> testcase_setup(%s)", casename);
configure_actor(last_space_id, ac_nested, nullptr, params);
configure_actor(last_space_id, ac_hill, nullptr, params);
configure_actor(last_space_id, ac_ttl, nullptr, params);
configure_actor(last_space_id, ac_copy, nullptr, params);
configure_actor(last_space_id, ac_append, nullptr, params);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_try, nullptr, params);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_try, nullptr, params);
log_notice("<<< testcase_setup(%s): done", casename);
} else {
failure("unknown testcase `%s`", casename);
}
}
void keycase_setup(const char *casename, actor_params &params) {
if (strcmp(casename, "random") == 0 || strcmp(casename, "prng") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_random;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "dashes") == 0 ||
strcmp(casename, "aside") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_dashes;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "custom") == 0) {
log_notice("=== keycase_setup(%s): skip", casename);
params.keygen.keycase = kc_custom;
} else {
failure("unknown keycase `%s`", casename);
}
}
/* TODO */

136
src/contrib/db/libmdbx/test/chrono.cc Normal file
View file

@ -0,0 +1,136 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace chrono {
#ifndef NSEC_PER_SEC
#define NSEC_PER_SEC 1000000000u
#endif /* NSEC_PER_SEC */
uint32_t ns2fractional(uint32_t ns) {
assert(ns < NSEC_PER_SEC);
/* LY: здесь и далее используется "длинное деление", которое
* для ясности кода оставлено как есть (без ручной оптимизации). Так как
* GCC, Clang и даже MSVC сами давно умеют конвертировать деление на
* константу в быструю reciprocal-форму. */
return ((uint64_t)ns << 32) / NSEC_PER_SEC;
}
uint32_t fractional2ns(uint32_t fractional) {
return (fractional * (uint64_t)NSEC_PER_SEC) >> 32;
}
#ifndef USEC_PER_SEC
#define USEC_PER_SEC 1000000u
#endif /* USEC_PER_SEC */
uint32_t us2fractional(uint32_t us) {
assert(us < USEC_PER_SEC);
return ((uint64_t)us << 32) / USEC_PER_SEC;
}
uint32_t fractional2us(uint32_t fractional) {
return (fractional * (uint64_t)USEC_PER_SEC) >> 32;
}
#ifndef MSEC_PER_SEC
#define MSEC_PER_SEC 1000u
#endif /* MSEC_PER_SEC */
uint32_t ms2fractional(uint32_t ms) {
assert(ms < MSEC_PER_SEC);
return ((uint64_t)ms << 32) / MSEC_PER_SEC;
}
uint32_t fractional2ms(uint32_t fractional) {
return (fractional * (uint64_t)MSEC_PER_SEC) >> 32;
}
time from_ns(uint64_t ns) {
time result;
result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) |
ns2fractional((uint32_t)(ns % NSEC_PER_SEC));
return result;
}
time from_us(uint64_t us) {
time result;
result.fixedpoint = ((us / USEC_PER_SEC) << 32) |
us2fractional((uint32_t)(us % USEC_PER_SEC));
return result;
}
time from_ms(uint64_t ms) {
time result;
result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) |
ms2fractional((uint32_t)(ms % MSEC_PER_SEC));
return result;
}
time now_realtime() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static void(WINAPI * query_time)(LPFILETIME);
if (!query_time) {
query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(
GetModuleHandle(TEXT("kernel32.dll")),
"GetSystemTimePreciseAsFileTime");
if (!query_time)
query_time = GetSystemTimeAsFileTime;
}
FILETIME filetime;
query_time(&filetime);
uint64_t ns100 =
(uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime;
return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u);
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_REALTIME, &ts)))
failure_perror("clock_gettime(CLOCK_REALTIME", errno);
return from_timespec(ts);
#endif
}
time now_motonic() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static uint64_t reciprocal;
static LARGE_INTEGER Frequency;
if (reciprocal == 0) {
if (!QueryPerformanceFrequency(&Frequency))
failure_perror("QueryPerformanceFrequency()", GetLastError());
reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) /
Frequency.QuadPart);
assert(reciprocal);
}
LARGE_INTEGER Counter;
if (!QueryPerformanceCounter(&Counter))
failure_perror("QueryPerformanceCounter()", GetLastError());
time result;
result.fixedpoint = (Counter.QuadPart / Frequency.QuadPart) << 32;
uint64_t mod = Counter.QuadPart % Frequency.QuadPart;
result.fixedpoint += (mod * reciprocal) >> 16;
return result;
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_MONOTONIC, &ts)))
failure_perror("clock_gettime(CLOCK_MONOTONIC)", errno);
return from_timespec(ts);
#endif
}
} /* namespace chrono */

99
src/contrib/db/libmdbx/test/chrono.h Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "utils.h"
namespace chrono {
#pragma pack(push, 1)
typedef union time {
uint64_t fixedpoint;
__anonymous_struct_extension__ struct {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint32_t fractional;
union {
uint32_t utc;
uint32_t integer;
};
#else
union {
uint32_t utc;
uint32_t integer;
};
uint32_t fractional;
#endif
};
void reset() { fixedpoint = 0; }
uint32_t seconds() const { return utc; }
} time;
#pragma pack(pop)
uint32_t ns2fractional(uint32_t);
uint32_t fractional2ns(uint32_t);
uint32_t us2fractional(uint32_t);
uint32_t fractional2us(uint32_t);
uint32_t ms2fractional(uint32_t);
uint32_t fractional2ms(uint32_t);
time from_ns(uint64_t us);
time from_us(uint64_t ns);
time from_ms(uint64_t ms);
inline time from_seconds(uint64_t seconds) {
assert(seconds < UINT32_MAX);
time result;
result.fixedpoint = seconds << 32;
return result;
}
inline time from_utc(time_t utc) {
assert(utc >= 0);
return from_seconds((uint64_t)utc);
}
inline time infinite() {
time result;
result.fixedpoint = UINT64_MAX;
return result;
}
#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \
defined(CLOCK_REALTIME)
inline time from_timespec(const struct timespec &ts) {
time result;
result.fixedpoint =
((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec);
return result;
}
#endif /* HAVE_TIMESPEC_TV_NSEC */
#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL)
inline time from_timeval(const struct timeval &tv) {
time result;
result.fixedpoint =
((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec);
return result;
}
#endif /* HAVE_TIMEVAL_TV_USEC */
time now_realtime();
time now_motonic();
} /* namespace chrono */

602
src/contrib/db/libmdbx/test/config.cc Normal file
View file

@ -0,0 +1,602 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#if defined(_MSC_VER) && !defined(strcasecmp)
#define strcasecmp(str, len) _stricmp(str, len)
#endif /* _MSC_VER && strcasecmp() */
namespace config {
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value) {
assert(narg < argc);
const char *current = argv[narg];
const size_t optlen = strlen(option);
if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen))
return false;
if (!value) {
if (current[optlen + 2] == '=')
failure("Option '--%s' doen't accept any value\n", option);
return true;
}
*value = nullptr;
if (current[optlen + 2] == '=') {
*value = &current[optlen + 3];
return true;
}
if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) {
*value = argv[narg + 1];
if (strcmp(*value, "default") == 0) {
if (!default_value)
failure("Option '--%s' doen't accept default value\n", option);
*value = default_value;
}
++narg;
return true;
}
if (default_value) {
*value = default_value;
return true;
}
failure("No value given for '--%s' option\n", option);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty) {
return parse_option(argc, argv, narg, option, value, allow_empty,
allow_empty ? "" : nullptr);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
const char *default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr, default_value))
return false;
if (!allow_empty && strlen(value_cstr) == 0)
failure("Value for option '--%s' could't be empty\n", option);
value = value_cstr;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs) {
const char *list;
if (!parse_option(argc, argv, narg, option, &list))
return false;
unsigned clear = 0;
while (*list) {
if (*list == ',' || *list == ' ' || *list == '\t') {
++list;
continue;
}
const char *const comma = strchr(list, ',');
const bool strikethrough = *list == '-' || *list == '~';
if (strikethrough || *list == '+')
++list;
else
mask = clear;
const size_t len = (comma) ? comma - list : strlen(list);
const option_verb *scan = verbs;
while (true) {
if (!scan->verb)
failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list,
option);
if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) {
mask = strikethrough ? mask & ~scan->mask : mask | scan->mask;
clear = strikethrough ? clear & ~scan->mask : clear | scan->mask;
list += len;
break;
}
++scan;
}
}
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval, const uint64_t maxval,
const uint64_t default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr))
return false;
if (default_value && strcmp(value_cstr, "default") == 0) {
value = default_value;
return true;
}
if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0) {
value = minval;
return true;
}
if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0) {
value = maxval;
return true;
}
char *suffix = nullptr;
errno = 0;
unsigned long long raw = strtoull(value_cstr, &suffix, 0);
if ((suffix && *suffix) || errno) {
suffix = nullptr;
errno = 0;
raw = strtoull(value_cstr, &suffix, 10);
}
if (errno)
failure("Option '--%s' expects a numeric value (%s)\n", option,
test_strerror(errno));
uint64_t multipler = 1;
if (suffix && *suffix) {
if (scale == no_scale)
failure("Option '--%s' doen't accepts suffixes, so '%s' is unexpected\n",
option, suffix);
if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024);
else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0)
multipler =
(scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024;
else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024;
else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024 * 1024;
else if (scale == duration &&
(strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0))
multipler = 1;
else if (scale == duration &&
(strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0))
multipler = 60;
else if (scale == duration &&
(strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0))
multipler = 3600;
else if (scale == duration &&
(strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0))
multipler = 3600 * 24;
else
failure(
"Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s"
"suffixes, but '%s' is unexpected\n",
option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "",
suffix);
}
if (raw >= UINT64_MAX / multipler)
failure("The value for option '--%s' is too huge\n", option);
value = raw * multipler;
if (maxval && value > maxval)
failure("The maximal value for option '--%s' is %" PRIu64 "\n", option,
maxval);
if (value < minval)
failure("The minimal value for option '--%s' is %" PRIu64 "\n", option,
minval);
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval, const unsigned maxval,
const unsigned default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval,
default_value))
return false;
value = (unsigned)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval, const uint8_t maxval,
const uint8_t default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval,
default_value))
return false;
value = (uint8_t)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value) {
uint64_t proxy = (uint64_t)value;
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int64_t)proxy;
return true;
}
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value) {
uint64_t proxy = (uint64_t)value;
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int32_t)proxy;
return true;
}
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value) {
const char *value_cstr = nullptr;
if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) {
const char *current = argv[narg];
if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) {
value = false;
return true;
}
if (strncmp(current, "--dont-", 7) == 0 &&
strcmp(current + 7, option) == 0) {
value = false;
return true;
}
return false;
}
if (!value_cstr) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) {
value = false;
return true;
}
failure(
"Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n",
option, value_cstr);
}
//-----------------------------------------------------------------------------
const struct option_verb mode_bits[] = {
{"rdonly", MDBX_RDONLY}, {"mapasync", MDBX_MAPASYNC},
{"utterly", MDBX_UTTERLY_NOSYNC}, {"nosubdir", MDBX_NOSUBDIR},
{"nosync", MDBX_NOSYNC}, {"nometasync", MDBX_NOMETASYNC},
{"writemap", MDBX_WRITEMAP}, {"notls", MDBX_NOTLS},
{"nordahead", MDBX_NORDAHEAD}, {"nomeminit", MDBX_NOMEMINIT},
{"coalesce", MDBX_COALESCE}, {"lifo", MDBX_LIFORECLAIM},
{"perturb", MDBX_PAGEPERTURB}, {nullptr, 0}};
const struct option_verb table_bits[] = {
{"key.reverse", MDBX_REVERSEKEY},
{"key.integer", MDBX_INTEGERKEY},
{"data.integer", MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.fixed", MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.reverse", MDBX_REVERSEDUP | MDBX_DUPSORT},
{"data.dups", MDBX_DUPSORT},
{nullptr, 0}};
static void dump_verbs(const char *caption, size_t bits,
const struct option_verb *verbs) {
log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits);
const char *comma = "";
while (verbs->mask && bits) {
if ((bits & verbs->mask) == verbs->mask) {
logging::feed("%s%s", comma, verbs->verb);
bits -= verbs->mask;
comma = ", ";
}
++verbs;
}
logging::feed("%s\n", (*comma == '\0') ? "none" : "");
}
static void dump_duration(const char *caption, unsigned duration) {
log_verbose("%s: ", caption);
if (duration) {
if (duration > 24 * 3600)
logging::feed("%u_", duration / (24 * 3600));
if (duration > 3600)
logging::feed("%02u:", (duration % (24 * 3600)) / 3600);
logging::feed("%02u:%02u", (duration % 3600) / 60, duration % 60);
} else {
logging::feed("INFINITE");
}
logging::feed("\n");
}
void dump(const char *title) {
logging::local_suffix indent(title);
for (auto i = global::actors.begin(); i != global::actors.end(); ++i) {
log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id,
testcase2str(i->testcase), i->space_id);
indent.push();
if (i->params.loglevel) {
log_verbose("log: level %u, %s\n", i->params.loglevel,
i->params.pathname_log.empty()
? "console"
: i->params.pathname_log.c_str());
}
log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR
", %i %i, %i]\n",
i->params.pathname_db.c_str(), i->params.size_now,
i->params.size_lower, i->params.size_upper,
i->params.shrink_threshold, i->params.growth_step,
i->params.pagesize);
dump_verbs("mode", i->params.mode_flags, mode_bits);
dump_verbs("table", i->params.table_flags, table_bits);
if (i->params.test_nops)
log_verbose("iterations/records %u\n", i->params.test_nops);
else
dump_duration("duration", i->params.test_duration);
if (i->params.nrepeat)
log_verbose("repeat %u\n", i->params.nrepeat);
else
log_verbose("repeat ETERNALLY\n");
log_verbose("threads %u\n", i->params.nthreads);
log_verbose(
"keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64
", split %u/%u\n",
keygencase2str(i->params.keygen.keycase), i->params.keygen.width,
i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset,
i->params.keygen.split,
i->params.keygen.width - i->params.keygen.split);
log_verbose("keygen.seed: %u\n", i->params.keygen.seed);
log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min,
i->params.keylen_max);
log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min,
i->params.datalen_max);
log_verbose("batch: read %u, write %u\n", i->params.batch_read,
i->params.batch_write);
if (i->params.waitfor_nops)
log_verbose("wait: actor %u for %u ops\n", i->wait4id,
i->params.waitfor_nops);
else if (i->params.delaystart)
dump_duration("delay", i->params.delaystart);
else
log_verbose("no-delay\n");
if (i->params.inject_writefaultn)
log_verbose("inject-writefault on %u ops\n",
i->params.inject_writefaultn);
else
log_verbose("no-inject-writefault\n");
log_verbose("limits: readers %u, tables %u\n", i->params.max_readers,
i->params.max_tables);
log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
log_verbose("ignore MDBX_MAP_FULL error: %s\n",
i->params.ignore_dbfull ? "Yes" : "No");
log_verbose("verifying by speculum: %s\n",
i->params.speculum ? "Yes" : "No");
indent.pop();
}
dump_duration("timeout", global::config::timeout_duration_seconds);
log_verbose("cleanup: before %s, after %s\n",
global::config::cleanup_before ? "Yes" : "No",
global::config::cleanup_after ? "Yes" : "No");
log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No");
log_verbose("progress indicator: %s\n",
global::config::progress_indicator ? "Yes" : "No");
log_verbose("console mode: %s\n",
global::config::console_mode ? "Yes" : "No");
}
} /* namespace config */
//-----------------------------------------------------------------------------
using namespace config;
actor_config::actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id)
: params(params) {
this->space_id = space_id;
this->actor_id = 1 + (unsigned)global::actors.size();
this->testcase = testcase;
this->wait4id = wait4id;
signal_nops = 0;
}
const std::string actor_config::serialize(const char *prefix) const {
simple_checksum checksum;
std::string result;
if (prefix)
result.append(prefix);
checksum.push(params.pathname_db);
result.append(params.pathname_db);
result.push_back('|');
checksum.push(params.pathname_log);
result.append(params.pathname_log);
result.push_back('|');
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
result.append(data2hex(static_cast<const actor_params_pod *>(&params),
sizeof(actor_params_pod), checksum));
result.push_back('|');
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
result.append(data2hex(static_cast<const actor_config_pod *>(this),
sizeof(actor_config_pod), checksum));
result.push_back('|');
result.push_back(global::config::progress_indicator ? 'Y' : 'N');
checksum.push(global::config::progress_indicator);
result.push_back(global::config::console_mode ? 'Y' : 'N');
checksum.push(global::config::console_mode);
result.push_back('|');
result.append(osal_serialize(checksum));
result.push_back('|');
result.append(std::to_string(checksum.value));
return result;
}
bool actor_config::deserialize(const char *str, actor_config &config) {
simple_checksum checksum;
TRACE(">> actor_config::deserialize: %s\n", str);
const char *slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-1\n");
return false;
}
config.params.pathname_db.assign(str, slash - str);
checksum.push(config.params.pathname_db);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-2\n");
return false;
}
config.params.pathname_log.assign(str, slash - str);
checksum.push(config.params.pathname_log);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-3\n");
return false;
}
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params),
sizeof(actor_params_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-4\n");
return false;
}
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config),
sizeof(actor_config_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-5\n");
return false;
}
if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N')) {
global::config::progress_indicator = str[0] == 'Y';
checksum.push(global::config::progress_indicator);
global::config::console_mode = str[1] == 'Y';
checksum.push(global::config::console_mode);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-6\n");
return false;
}
}
if (!config.osal_deserialize(str, slash, checksum)) {
TRACE("<< actor_config::deserialize: osal\n");
return false;
}
str = slash + 1;
uint64_t verify = std::stoull(std::string(str));
if (checksum.value != verify) {
TRACE("<< actor_config::deserialize: checksum mismatch\n");
return false;
}
TRACE("<< actor_config::deserialize: OK\n");
return true;
}
unsigned actor_params::mdbx_keylen_min() const {
return (table_flags & MDBX_INTEGERKEY) ? 4 : 0;
}
unsigned actor_params::mdbx_keylen_max() const {
return (table_flags & MDBX_INTEGERKEY)
? 8
: std::min((unsigned)mdbx_limits_keysize_max(pagesize),
(unsigned)UINT16_MAX);
}
unsigned actor_params::mdbx_datalen_min() const {
return (table_flags & MDBX_INTEGERDUP) ? 4 : 0;
}
unsigned actor_params::mdbx_datalen_max() const {
return (table_flags & MDBX_INTEGERDUP)
? 8
: std::min((table_flags & MDBX_DUPSORT)
? (unsigned)mdbx_limits_keysize_max(pagesize)
: (unsigned)MDBX_MAXDATASIZE,
(unsigned)UINT16_MAX);
}

326
src/contrib/db/libmdbx/test/config.h Normal file
View file

@ -0,0 +1,326 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "log.h"
#include "utils.h"
#define ACTOR_ID_MAX INT16_MAX
enum actor_testcase {
ac_none,
ac_hill,
ac_deadread,
ac_deadwrite,
ac_jitter,
ac_try,
ac_copy,
ac_append,
ac_ttl,
ac_nested
};
enum actor_status {
as_unknown,
as_debuging,
as_running,
as_successful,
as_killed,
as_failed,
as_coredump,
};
const char *testcase2str(const actor_testcase);
const char *status2str(actor_status status);
enum keygen_case {
kc_random, /* [ 6.. 2.. 7.. 4.. 0.. 1.. 5.. 3.. ] */
kc_dashes, /* [ 0123.. 4567.. ] */
kc_custom,
/* TODO: more cases */
};
const char *keygencase2str(const keygen_case);
//-----------------------------------------------------------------------------
namespace config {
enum scale_mode { no_scale, decimal, binary, duration };
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value = nullptr);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty = false);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
const char *default_value);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value);
struct option_verb {
const char *const verb;
unsigned mask;
};
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval = 0, const uint64_t maxval = INT64_MAX,
const uint64_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval = 0, const unsigned maxval = INT32_MAX,
const unsigned default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval = 0,
const uint8_t maxval = 255, const uint8_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value = -1);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value = -1);
inline bool parse_option_intptr(int argc, char *const argv[], int &narg,
const char *option, intptr_t &value,
const intptr_t minval, const intptr_t maxval,
const intptr_t default_value = -1) {
static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?");
if (sizeof(intptr_t) == 8)
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int64_t *>(&value), int64_t(minval),
int64_t(maxval), int64_t(default_value));
else
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int32_t *>(&value), int32_t(minval),
int32_t(maxval), int32_t(default_value));
}
//-----------------------------------------------------------------------------
#pragma pack(push, 1)
struct keygen_params_pod {
/* Параметры генератора пар key-value.
*
* Ключи и значения генерируются по задаваемым параметрам на основе "плоской"
* исходной координаты. При этом, в общем случае, в процессе тестов исходная
* координата последовательно итерируется в заданном диапазоне, а необходимые
* паттерны/последовательности/узоры получаются за счет преобразования
* исходной координаты, согласно описанным ниже параметрам.
*
* Стоит отметить, что порядок описания параметров для удобства совпадает с
* порядком их использования, т.е. с порядком соответствующих преобразований.
*
* Второе важное замечание касается ограничений одновременной координированной
* генерации паттеров как для ключей, так и для значений. Суть в том, что
* такая возможность не нужна по следующим причинам:
* - libmdbx поддерживает два существенно различающихся вида таблиц,
* "уникальные" (без дубликатов и без multi-value), и так называемые
* "с дубликатами" (c multi-value).
* - Для таблиц "без дубликатов" только размер связанных к ключами значений
* (данных) оказывает влияния на работу движка, непосредственно содержимое
* данных не анализируется движком и не оказывает влияния на его работу.
* - Для таблиц "с дубликатами", при наличии более одного значения для
* некоторого ключа, формируется дочернее btree-поддерево. Это дерево
* формируется в отдельном "кусте" страниц и обслуживается независимо
* от окружения родительского ключа.
* - Таким образом, паттерн генерации значений имеет смысл только для
* таблиц "с дубликатами" и только в контексте одного значения ключа.
* Иначе говоря, нет смысла в со-координации генерации паттернов для
* ключей и значений. Более того, генерацию значений всегда необходимо
* рассматривать в контексте связки с одним значением ключа.
* - Тем не менее, во всех случаях достаточно важным является равномерная
* всех возможных сочетаний длин ключей и данных.
*
* width:
* Большинство тестов предполагают создание или итерирование некоторого
* количества записей. При этом требуется итерирование или генерация
* значений и ключей из некоторого ограниченного пространства вариантов.
*
* Параметр width задает такую ширину пространства вариантов в битах.
* Таким образом мощность пространства вариантов (пока) всегда равна
* степени двойки. Это ограничение можно снять, но ценой увеличения
* вычислительной сложности, включая потерю простоты и прозрачности.
*
* С другой стороны, не-битовый width может быть полезен:
* - Позволит генерировать ключи/значения в точно задаваемом диапазоне.
* Например, перебрать в псевдо-случайном порядке 10001 значение.
* - Позволит поровну разделять заданное пространство (диапазон)
* ключей/значений между количеством потоков некратным степени двойки.
*
* mesh и seed:
* Позволяют получить псевдо-случайные последовательности ключей/значений.
* Параметр mesh задает сколько младших бит исходной плоской координаты
* будет "перемешано" (инъективно отображено), а параметр seed позволяет
* выбрать конкретный вариант "перемешивания".
*
* Перемешивание выполняется при ненулевом значении mesh. Перемешивание
* реализуется посредством применения двух инъективных функций для
* заданного количества бит:
* - применяется первая инъективная функция;
* - к результату добавляется salt полученный из seed;
* - применяется вторая инъективная функция;
*
* Следует отметить, что mesh умышленно позволяет перемешать только младшую
* часть, что при ненулевом значении split (см далее) не позволяет получать
* псевдо-случайные значений ключей без псевдо-случайности в значениях.
*
* Такое ограничение соответствуют внутренней алгоритмике libmdbx. Проще
* говоря, мы можем проверить движок псевдо-случайной последовательностью
* ключей на таблицах без дубликатов (без multi-value), а затем проверить
* корректность работу псевдо-случайной последовательностью значений на
* таблицах с дубликатами (с multi-value), опционально добавляя
* псевдо-случайности к последовательности ключей. Однако, нет смысла
* генерировать псевдо-случайные ключи, одновременно с формированием
* какого-либо паттерна в значениях, так как содержимое в данных либо
* не будет иметь значения (для таблиц без дубликатов), либо будет
* обрабатываться в отдельных btree-поддеревьях.
*
* rotate и offset:
* Для проверки слияния и разделения страниц внутри движка требуются
* генерация ключей/значений в виде не-смежных последовательностей, как-бы
* в виде "пунктира", который постепенно заполняет весь заданных диапазон.
*
* Параметры позволяют генерировать такой "пунктир". Соответственно rotate
* задает циклический сдвиг вправо, а offset задает смещение, точнее говоря
* сложение по модулю внутри диапазона заданного посредством width.
*
* Например, при rotate равном 1 (циклический сдвиг вправо на 1 бит),
* четные и нечетные исходные значения сложатся в две линейные
* последовательности, которые постепенно закроют старшую и младшую
* половины диапазона.
*
* split:
* Для таблиц без дубликатов (без multi-value ключей) фактически требуется
* генерация только ключей, а данные могут быть постоянным. Но для таблиц с
* дубликатами (с multi-value ключами) также требуется генерация значений.
*
* Ненулевое значение параметра split фактически включает генерацию значений,
* при этом значение split определяет сколько бит исходного абстрактного
* номера будет отрезано для генерации значения.
*/
uint8_t width;
uint8_t mesh;
uint8_t rotate;
uint8_t split;
uint32_t seed;
uint64_t offset;
keygen_case keycase;
};
struct actor_params_pod {
unsigned mode_flags;
unsigned table_flags;
intptr_t size_lower;
intptr_t size_now;
intptr_t size_upper;
int shrink_threshold;
int growth_step;
int pagesize;
unsigned test_duration;
unsigned test_nops;
unsigned nrepeat;
unsigned nthreads;
unsigned keylen_min, keylen_max;
unsigned datalen_min, datalen_max;
unsigned batch_read;
unsigned batch_write;
unsigned delaystart;
unsigned waitfor_nops;
unsigned inject_writefaultn;
unsigned max_readers;
unsigned max_tables;
keygen_params_pod keygen;
uint8_t loglevel;
bool drop_table;
bool ignore_dbfull;
bool speculum;
};
struct actor_config_pod {
unsigned actor_id, space_id;
actor_testcase testcase;
unsigned wait4id;
unsigned signal_nops;
};
#pragma pack(pop)
extern const struct option_verb mode_bits[];
extern const struct option_verb table_bits[];
void dump(const char *title = "config-dump: ");
} /* namespace config */
struct actor_params : public config::actor_params_pod {
std::string pathname_log;
std::string pathname_db;
void set_defaults(const std::string &tmpdir);
unsigned mdbx_keylen_min() const;
unsigned mdbx_keylen_max() const;
unsigned mdbx_datalen_min() const;
unsigned mdbx_datalen_max() const;
};
struct actor_config : public config::actor_config_pod {
actor_params params;
bool wanna_event4signalling() const { return true /* TODO ? */; }
actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id);
actor_config(const char *str) {
if (!deserialize(str, *this))
failure("Invalid internal parameter '%s'\n", str);
}
const std::string osal_serialize(simple_checksum &) const;
bool osal_deserialize(const char *str, const char *end, simple_checksum &);
const std::string serialize(const char *prefix) const;
static bool deserialize(const char *str, actor_config &config);
bool is_waitable(size_t nops) const {
switch (testcase) {
case ac_hill:
if (!params.test_nops || params.test_nops >= nops)
return true;
__fallthrough;
default:
return false;
}
}
};

26
src/contrib/db/libmdbx/test/copy.cc Normal file
View file

@ -0,0 +1,26 @@
#include "test.h"
void testcase_copy::copy_db(const bool with_compaction) {
int err = osal_removefile(copy_pathname);
if (err != MDBX_SUCCESS && err != MDBX_ENOFILE)
failure_perror("mdbx_removefile()", err);
err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(),
with_compaction ? MDBX_CP_COMPACT : 0);
if (unlikely(err != MDBX_SUCCESS))
failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)"
: "mdbx_env_copy(MDBX_CP_ASIS)",
err);
}
bool testcase_copy::run() {
jitter_delay();
db_open();
assert(!txn_guard);
const bool order = flipcoin();
jitter_delay();
copy_db(order);
jitter_delay();
copy_db(!order);
return true;
}

24
src/contrib/db/libmdbx/test/darwin/LICENSE Normal file
View file

@ -0,0 +1,24 @@
Copyright (c) 2015, Aleksey Demakov
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

8
src/contrib/db/libmdbx/test/darwin/README.md Normal file
View file

@ -0,0 +1,8 @@
# DarwinPthreadBarrier
A pthread_barrier_t implementation for Mac OS/X
There is no pthread_barrier_t in Mac OS/X pthreads. This project fixes
this omission by providing a simple-minded barrier implementation based
on a pair of pthread_mutex_t and pthread_cond_t.

110
src/contrib/db/libmdbx/test/darwin/pthread_barrier.c Normal file
View file

@ -0,0 +1,110 @@
/*
* Copyright (c) 2015, Aleksey Demakov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "pthread_barrier.h"
#include <errno.h>
#ifdef __APPLE__
int pthread_barrierattr_init(pthread_barrierattr_t *attr) {
memset(attr, 0, sizeof(pthread_barrierattr_t));
int m = pthread_mutexattr_init(&attr->mattr);
int c = pthread_condattr_init(&attr->cattr);
return m ? m : c;
}
int pthread_barrierattr_destroy(pthread_barrierattr_t *attr) {
int c = pthread_condattr_destroy(&attr->cattr);
int m = pthread_mutexattr_destroy(&attr->mattr);
return m ? m : c;
}
int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
int *__restrict pshared) {
return pthread_condattr_getpshared(&attr->cattr, pshared);
}
int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) {
int m = pthread_mutexattr_setpshared(&attr->mattr, pshared);
int c = pthread_condattr_setpshared(&attr->cattr, pshared);
return m ? m : c;
}
int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
const pthread_barrierattr_t *__restrict attr,
unsigned count) {
if (count == 0)
return errno = EINVAL;
int rc = pthread_mutex_init(&barrier->mutex, attr ? &attr->mattr : 0);
if (rc)
return rc;
rc = pthread_cond_init(&barrier->cond, attr ? &attr->cattr : 0);
if (rc) {
int errno_save = errno;
pthread_mutex_destroy(&barrier->mutex);
errno = errno_save;
return rc;
}
barrier->limit = count;
barrier->count = 0;
barrier->phase = 0;
return 0;
}
int pthread_barrier_destroy(pthread_barrier_t *barrier) {
pthread_mutex_destroy(&barrier->mutex);
pthread_cond_destroy(&barrier->cond);
return 0;
}
int pthread_barrier_wait(pthread_barrier_t *barrier) {
int rc = pthread_mutex_lock(&barrier->mutex);
if (rc)
return rc;
barrier->count++;
if (barrier->count >= barrier->limit) {
barrier->phase++;
barrier->count = 0;
pthread_cond_broadcast(&barrier->cond);
pthread_mutex_unlock(&barrier->mutex);
return PTHREAD_BARRIER_SERIAL_THREAD;
} else {
unsigned phase = barrier->phase;
do
pthread_cond_wait(&barrier->cond, &barrier->mutex);
while (phase == barrier->phase);
pthread_mutex_unlock(&barrier->mutex);
return 0;
}
}
#endif /* __APPLE__ */

83
src/contrib/db/libmdbx/test/darwin/pthread_barrier.h Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright (c) 2015, Aleksey Demakov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef PTHREAD_BARRIER_H
#define PTHREAD_BARRIER_H
#include <pthread.h>
#ifdef __APPLE__
#ifdef __cplusplus
extern "C" {
#endif
#if !defined(PTHREAD_BARRIER_SERIAL_THREAD)
#define PTHREAD_BARRIER_SERIAL_THREAD (1)
#endif
#if !defined(PTHREAD_PROCESS_PRIVATE)
#define PTHREAD_PROCESS_PRIVATE (42)
#endif
#if !defined(PTHREAD_PROCESS_SHARED)
#define PTHREAD_PROCESS_SHARED (43)
#endif
typedef struct {
pthread_mutexattr_t mattr;
pthread_condattr_t cattr;
} pthread_barrierattr_t;
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
unsigned int limit;
unsigned int count;
unsigned int phase;
} pthread_barrier_t;
int pthread_barrierattr_init(pthread_barrierattr_t *attr);
int pthread_barrierattr_destroy(pthread_barrierattr_t *attr);
int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
int *__restrict pshared);
int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared);
int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
const pthread_barrierattr_t *__restrict attr,
unsigned int count);
int pthread_barrier_destroy(pthread_barrier_t *barrier);
int pthread_barrier_wait(pthread_barrier_t *barrier);
#ifdef __cplusplus
}
#endif
#endif /* __APPLE__ */
#endif /* PTHREAD_BARRIER_H */

35
src/contrib/db/libmdbx/test/dead.cc Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_deadread::run() {
db_open();
txn_begin(true);
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
return true;
}
//-----------------------------------------------------------------------------
bool testcase_deadwrite::run() {
db_open();
txn_begin(false);
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
return true;
}

409
src/contrib/db/libmdbx/test/hill.cc Normal file
View file

@ -0,0 +1,409 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_hill::run() {
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
return false;
}
speculum.clear();
speculum_commited.clear();
/* LY: тест "холмиком":
* - сначала наполняем таблицу циклическими CRUD-манипуляциями,
* которые в каждом цикле делают несколько операций, включая удаление,
* но в результате добавляют записи.
* - затем очищаем таблицу также CRUD-манипуляциями, но уже с другой
* пропорцией удалений.
*
* При этом очень многое зависит от порядка перебора ключей:
* - (псевдо)случайное распределение требуется лишь для полноты картины,
* но в целом не покрывает важных кейсов.
* - кроме (псевдо)случайного перебора требуется последовательное
* итерирование ключей интервалами различной ширины, с тем чтобы
* проверить различные варианты как разделения, так и слияния страниц
* внутри движка.
* - при не-уникальных ключах (MDBX_DUPSORT с подвариантами), для каждого
* повтора внутри движка формируется вложенное btree-дерево,
* соответственно требуется соблюдение аналогичных принципов
* итерирования для значений.
*/
/* TODO: работа в несколько потоков */
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
keygen::buffer a_key = keygen::alloc(config.params.keylen_max);
keygen::buffer a_data_0 = keygen::alloc(config.params.datalen_max);
keygen::buffer a_data_1 = keygen::alloc(config.params.datalen_max);
keygen::buffer b_key = keygen::alloc(config.params.keylen_max);
keygen::buffer b_data = keygen::alloc(config.params.datalen_max);
const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_NODUPDATA
: MDBX_NODUPDATA | MDBX_NOOVERWRITE;
const unsigned update_flags =
(config.params.table_flags & MDBX_DUPSORT)
? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE
: MDBX_NODUPDATA;
uint64_t serial_count = 0;
uint64_t commited_serial = serial_count;
unsigned txn_nops = 0;
bool rc = false;
while (should_continue()) {
const keygen::serial_t a_serial = serial_count;
if (unlikely(!keyvalue_maker.increment(serial_count, 1))) {
log_notice("uphill: unexpected key-space overflow");
break;
}
const keygen::serial_t b_serial = serial_count;
assert(b_serial > a_serial);
// создаем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_1, age_shift);
err = insert(a_key, a_data_1, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-a.1)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-a, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-a, after commit");
goto bailout;
}
}
// создаем вторую запись из пары
log_trace("uphill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = insert(b_key, b_data, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-b, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-b, after commit");
goto bailout;
}
}
// обновляем данные в первой записи
log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value);
err = replace(a_key, a_data_0, a_data_1, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_replace(update-a: 1->0)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after update-a, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after update-a, after commit");
goto bailout;
}
}
// удаляем вторую запись
log_trace("uphill: delete-b %" PRIu64, b_serial);
checkdata("uphill: delete-b", dbi, b_key->value, b_data->value);
err = remove(b_key, b_data);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after delete-b, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after delete-b, after commit");
goto bailout;
}
}
report(1);
if (!keyvalue_maker.increment(serial_count, 1)) {
// дошли до границы пространства ключей
serial_count = a_serial;
goto overflow;
}
}
while (serial_count > 1) {
if (unlikely(!keyvalue_maker.increment(serial_count, -2)))
failure("downhill: unexpected key-space underflow");
overflow:
const keygen::serial_t a_serial = serial_count;
const keygen::serial_t b_serial = a_serial + 1;
assert(b_serial > a_serial);
// обновляем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
generate_pair(a_serial, a_key, a_data_1, age_shift);
checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value);
err = replace(a_key, a_data_1, a_data_0, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at update-a due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(update-a: 0->1)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after update-a, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after update-a, after commit");
break;
}
}
// создаем вторую запись из пары
log_trace("downhill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = insert(b_key, b_data, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at insert-a due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after insert-b, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after insert-b, after commit");
break;
}
}
// удаляем первую запись
log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value);
err = remove(a_key, a_data_1);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-a due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(a)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-a, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-a, after commit");
break;
}
}
// удаляем вторую запись
log_trace("downhill: delete-b %" PRIu64, b_serial);
checkdata("downhill: delete-b", dbi, b_key->value, b_data->value);
err = remove(b_key, b_data);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-b due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-b, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-b, after commit");
goto bailout;
}
}
report(1);
}
rc = speculum_verify();
bailout:
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS))
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
}
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err));
return rc;
}
} else
db_table_close(dbi);
}
return rc;
}

59
src/contrib/db/libmdbx/test/jitter.cc Normal file
View file

@ -0,0 +1,59 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_jitter::run() {
while (should_continue()) {
jitter_delay();
db_open();
if (flipcoin()) {
jitter_delay();
txn_begin(true);
fetch_canary();
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
txn_begin(mode_readonly());
jitter_delay();
if (!mode_readonly()) {
fetch_canary();
update_canary(1);
/* TODO:
* - db_setsize()
* ...
*/
}
txn_end(flipcoin());
if (flipcoin()) {
jitter_delay();
txn_begin(true);
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
db_close();
/* just 'align' nops with other tests with batching */
const auto batching =
std::max(config.params.batch_read, config.params.batch_write);
report(std::max(1u, batching / 2));
}
return true;
}

275
src/contrib/db/libmdbx/test/keygen.cc Normal file
View file

@ -0,0 +1,275 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace keygen {
static inline __pure_function serial_t mask(unsigned bits) {
assert(bits > 0 && bits <= serial_maxwith);
return serial_allones >> (serial_maxwith - bits);
}
/* LY: https://en.wikipedia.org/wiki/Injective_function */
serial_t injective(const serial_t serial,
const unsigned bits /* at least serial_minwith (8) */,
const serial_t salt) {
assert(bits > serial_minwith && bits <= serial_maxwith);
/* LY: All these "magic" prime numbers were found
* and verified with a bit of brute force. */
static const uint64_t m[64 - serial_minwith + 1] = {
/* 8 - 24 */
113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937,
309277, 834497, 1499933, 4373441, 10184137,
/* 25 - 64 */
10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573,
6924051577, 19204053433, 45840188887, 53625693977, 73447827913,
141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197,
5871903036137, 11238507001417, 45264020802263, 105008404482889,
81921776907059, 199987980256399, 307207457507641, 946769023178273,
2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153,
23171543400565993, 53353226456762893, 155627817337932409,
227827205384840249, 816509268558278821, 576933057762605689,
2623957345935638441, 5048241705479929949, 4634245581946485653,
4613509448041658233, 4952535426879925961};
static const uint8_t s[64 - serial_minwith + 1] = {
/* 8 - 24 */
2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11,
/* 25 - 64 */
11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20,
10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20,
19};
const auto mult = m[bits - 8];
const auto shift = s[bits - 8];
serial_t result = serial * mult;
if (salt) {
const unsigned left = bits / 2;
const unsigned right = bits - left;
result = (result << left) | ((result & mask(bits)) >> right);
result = (result ^ salt) * mult;
}
result ^= result << shift;
result &= mask(bits);
log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64
" => %" PRIu64 "/%u",
serial, bits, mult, shift, salt, result, bits);
return result;
}
void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age) {
assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith);
assert(mapping.split <= mapping.width);
assert(mapping.mesh <= mapping.width);
assert(mapping.rotate <= mapping.width);
assert(mapping.offset <= mask(mapping.width));
assert(!(key_essentials.flags &
~(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)));
assert(!(value_essentials.flags & ~(MDBX_INTEGERDUP | MDBX_REVERSEDUP)));
log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial,
value_age);
if (mapping.mesh >= serial_minwith) {
serial =
(serial & ~mask(mapping.mesh)) | injective(serial, mapping.mesh, salt);
log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial);
}
if (mapping.rotate) {
const unsigned right = mapping.rotate;
const unsigned left = mapping.width - right;
serial = (serial << left) | ((serial & mask(mapping.width)) >> right);
log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64,
mapping.rotate, serial, serial);
}
if (mapping.offset) {
serial = (serial + mapping.offset) & mask(mapping.width);
log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset,
serial);
}
if (base) {
serial += base;
log_trace("keygen-pair: base@%" PRIu64 " => %" PRIu64, base, serial);
}
serial_t key_serial = serial;
serial_t value_serial = value_age << mapping.split;
if (mapping.split) {
if (key_essentials.flags & MDBX_DUPSORT) {
key_serial >>= mapping.split;
value_serial += serial & mask(mapping.split);
} else {
/* Без MDBX_DUPSORT требуется уникальность ключей, а для этого нельзя
* отбрасывать какие-либо биты serial после инъективного преобразования.
* Поэтому key_serial не трогаем, а в value_serial нелинейно вмешиваем
* запрошенное количество бит из serial */
value_serial +=
(serial ^ (serial >> mapping.split)) & mask(mapping.split);
}
value_serial |= value_age << mapping.split;
log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split,
key_serial, value_serial);
}
log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial,
value_serial);
mk(key_serial, key_essentials, *key);
mk(value_serial, value_essentials, *value);
if (log_enabled(logging::trace)) {
char dump_key[128], dump_value[128];
log_trace("keygen-pair: key %s, value %s",
mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&value->value, dump_value, sizeof(dump_value)));
}
}
void maker::setup(const config::actor_params_pod &actor, unsigned actor_id,
unsigned thread_number) {
key_essentials.flags =
actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT);
assert(actor.keylen_min <= UINT8_MAX);
key_essentials.minlen = (uint8_t)actor.keylen_min;
assert(actor.keylen_max <= UINT16_MAX);
key_essentials.maxlen = (uint16_t)actor.keylen_max;
value_essentials.flags =
actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP);
assert(actor.datalen_min <= UINT8_MAX);
value_essentials.minlen = (uint8_t)actor.datalen_min;
assert(actor.datalen_max <= UINT16_MAX);
value_essentials.maxlen = (uint16_t)actor.datalen_max;
assert(thread_number < 2);
(void)thread_number;
mapping = actor.keygen;
salt = (actor.keygen.seed + actor_id) * UINT64_C(14653293970879851569);
// FIXME: TODO
base = 0;
}
void maker::make_ordered() {
mapping.mesh = 0;
mapping.rotate = 0;
}
bool maker::is_unordered() const {
return (mapping.mesh >= serial_minwith || mapping.rotate) != 0;
}
bool maker::increment(serial_t &serial, int delta) const {
if (serial > mask(mapping.width)) {
log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial,
mask(mapping.width));
return false;
}
serial_t target = serial + (int64_t)delta;
if (target > mask(mapping.width) ||
((delta > 0) ? target < serial : target > serial)) {
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", overflow",
serial, delta, target);
return false;
}
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", continue", serial,
delta, target);
serial = target;
return true;
}
//-----------------------------------------------------------------------------
static size_t length(serial_t serial) {
size_t n = 0;
if (serial > UINT32_MAX) {
n = 4;
serial >>= 32;
}
if (serial > UINT16_MAX) {
n += 2;
serial >>= 16;
}
if (serial > UINT8_MAX) {
n += 1;
serial >>= 8;
}
return (serial > 0) ? n + 1 : n;
}
buffer alloc(size_t limit) {
result *ptr = (result *)malloc(sizeof(result) + limit);
if (unlikely(ptr == nullptr))
failure_perror("malloc(keyvalue_buffer)", errno);
ptr->value.iov_base = ptr->bytes;
ptr->value.iov_len = 0;
ptr->limit = limit;
return buffer(ptr);
}
void __hot maker::mk(const serial_t serial, const essentials &params,
result &out) {
assert(out.limit >= params.maxlen);
assert(params.maxlen >= params.minlen);
assert(params.maxlen >= length(serial));
out.value.iov_base = out.bytes;
out.value.iov_len =
(params.maxlen > params.minlen)
? params.minlen + serial % (params.maxlen - params.minlen)
: params.minlen;
if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) {
assert(params.maxlen == params.minlen);
assert(params.minlen == 4 || params.minlen == 8);
if (is_byteorder_le() || params.minlen == 8)
out.u64 = serial;
else
out.u32 = (uint32_t)serial;
} else if (params.flags & (MDBX_REVERSEKEY | MDBX_REVERSEDUP)) {
if (out.value.iov_len > 8) {
memset(out.bytes, '\0', out.value.iov_len - 8);
unaligned::store(out.bytes + out.value.iov_len - 8, htobe64(serial));
} else {
out.u64 = htobe64(serial);
if (out.value.iov_len < 8) {
out.value.iov_len = std::max(length(serial), out.value.iov_len);
out.value.iov_base = out.bytes + 8 - out.value.iov_len;
}
}
} else {
out.u64 = htole64(serial);
if (out.value.iov_len > 8)
memset(out.bytes + 8, '\0', out.value.iov_len - 8);
else
out.value.iov_len = std::max(length(serial), out.value.iov_len);
}
assert(out.value.iov_len >= params.minlen);
assert(out.value.iov_len <= params.maxlen);
assert(out.value.iov_len >= length(serial));
assert(out.value.iov_base >= out.bytes);
assert((uint8_t *)out.value.iov_base + out.value.iov_len <=
out.bytes + out.limit);
}
} /* namespace keygen */

130
src/contrib/db/libmdbx/test/keygen.h Normal file
View file

@ -0,0 +1,130 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "config.h"
#include "log.h"
#include "utils.h"
namespace keygen {
/* Под "генерацией ключей" здесь понимается генерация обоих значений для
* пар key-value, т.е. не только ключей, но и ассоциированных с ними данных.
*/
/* Генерацию ключей нельзя отнести к простым задачам, так как требования
* примерно следующие:
* - генерация разного количества уникальных ключей различной длины
* в задаваемом диапазоне;
* - возможность выбора как псевдо-случайного порядка ключей,
* так и по некоторым специфическим законам (ограниченными упорядоченными
* последовательностями, в шахматном порядке по граница диапазона и т.д.);
* - возможность генерации дубликатов с задаваемым законом распределения;
* - возможность генерации непересекающимися кластерами для параллельного
* использования в нескольких потоках;
* - использовать минимум ресурсов, как CPU, так и RAM, в том числе
* включая cache pollution и ram bandwidth.
*
* При этом заведомо известно, что для MDBX не имеет значения:
* - используемый алфавит (значения байтов);
* - частотное распределение по алфавиту;
* - абсолютное значение ключей или разность между отдельными значениями;
*
* Соответственно, в общих чертах, схема генерации следующая:
* - вводится плоская одномерная "координата" serial (uint64_t);
* - генерация специфических паттернов (последовательностей)
* реализуется посредством соответствующих преобразований "координат", при
* этом все подобные преобразования выполняются только над "координатой";
* - итоговая "координата" преобразуется в 8-байтное суррогатное значение
* ключа;
* - для получения ключей длиной МЕНЕЕ 8 байт суррогат может усекаться
* до ненулевых байт, в том числе до нулевой длины;
* - для получения ключей длиной БОЛЕЕ 8 байт суррогат дополняется
* нулями или псевдослучайной последовательностью;
*
* Механизм генерации паттернов:
* - реализованный механизм является компромиссом между скоростью/простотой
* и гибкостью, необходимой для получения последовательностей, которых
* будет достаточно для проверки сценариев разделения и слияния страниц
* с данными внутри mdbx;
* - псевдо-случайные паттерны реализуются посредством набора инъективных
* отображающих функций;
* - не-псевдо-случайные паттерны реализуются посредством параметризируемого
* трех-этапного преобразования:
* 1) смещение (сложение) по модулю;
* 2) циклический сдвиг;
* 3) добавление абсолютного смещения (базы);
*/
typedef uint64_t serial_t;
enum : serial_t {
serial_minwith = 8,
serial_maxwith = sizeof(serial_t) * 8,
serial_allones = ~(serial_t)0u
};
struct result {
MDBX_val value;
size_t limit;
union {
uint8_t bytes[sizeof(uint64_t)];
uint32_t u32;
uint64_t u64;
};
std::string as_string() const {
return std::string((const char *)value.iov_base, value.iov_len);
}
};
//-----------------------------------------------------------------------------
struct buffer_deleter /* : public std::unary_function<void, result *> */ {
void operator()(result *buffer) const { free(buffer); }
};
typedef std::unique_ptr<result, buffer_deleter> buffer;
buffer alloc(size_t limit);
class maker {
config::keygen_params_pod mapping;
serial_t base;
serial_t salt;
struct essentials {
uint8_t minlen;
uint8_t flags;
uint16_t maxlen;
} key_essentials, value_essentials;
static void mk(const serial_t serial, const essentials &params, result &out);
public:
maker() { memset(this, 0, sizeof(*this)); }
void pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age);
void setup(const config::actor_params_pod &actor, unsigned actor_id,
unsigned thread_number);
void make_ordered();
bool is_unordered() const;
bool increment(serial_t &serial, int delta) const;
};
} /* namespace keygen */

Some files were not shown because too many files have changed in this diff Show more