From d6e8ecb86f9852818565258b2453aeecab31ab1c Mon Sep 17 00:00:00 2001 From: sowle Date: Tue, 27 Aug 2019 22:18:55 +0300 Subject: [PATCH 1/9] callstack trace for win64 implemented and integrated into epee --- contrib/epee/include/misc_log_ex.h | 2 +- contrib/epee/include/misc_os_dependent.h | 30 ++- .../include/storages/levin_abstract_invoke2.h | 4 +- src/common/callstack_helper.cpp | 179 ++++++++++++++++++ src/common/callstack_helper.h | 24 +++ src/daemon/daemon.cpp | 3 + 6 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 src/common/callstack_helper.cpp create mode 100644 src/common/callstack_helper.h diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index c620daec..8f932e10 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -150,7 +150,7 @@ DISABLE_VS_WARNINGS(4100) {std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, true, log_name);}} #define LOG_ERROR2(log_name, x) { \ - std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::print_trace() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL); } + std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::get_callstack() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL); } #define LOG_FRAME2(log_name, x, y) epee::log_space::log_frame frame(x, y, log_name) diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 116826ce..c755d70c 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -113,7 +113,7 @@ namespace misc_utils #include #include #endif - inline std::string print_trace() + inline std::string print_trace_default() { std::stringstream ss; #if defined(__GNUC__) @@ -134,5 +134,33 @@ namespace misc_utils #endif return ss.str(); } + + typedef std::string (stack_retrieving_function_t)(); + + // + // To get stack trace call it with the defaults. + // + inline std::string get_callstack(stack_retrieving_function_t* p_stack_retrieving_function_to_be_added = nullptr, bool remove_func = false) + { + static stack_retrieving_function_t* p_srf = nullptr; + + if (remove_func) + { + p_srf = nullptr; + return ""; + } + + if (p_stack_retrieving_function_to_be_added != nullptr) + { + p_srf = p_stack_retrieving_function_to_be_added; + return ""; + } + + if (p_srf != nullptr) + return p_srf(); + + return print_trace_default(); + } + } } diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 72bb34b2..1b2c74a4 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -126,7 +126,7 @@ namespace epee t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) { - LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::print_trace()); + LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack()); LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code << "(" << epee::levin::get_err_descr(code) << ")context:" << print_connection_context(context)); TRY_ENTRY() cb(code, result_struct, context); @@ -150,7 +150,7 @@ namespace epee }, inv_timeout); if( res <=0 ) { - LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::print_trace()); + LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack()); LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res << "(" << epee::levin::get_err_descr(res) << ") conn_id=" << conn_id); return false; } diff --git a/src/common/callstack_helper.cpp b/src/common/callstack_helper.cpp new file mode 100644 index 00000000..eb1be7eb --- /dev/null +++ b/src/common/callstack_helper.cpp @@ -0,0 +1,179 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(WIN32) + +#define WIN32_LEAN_AND_MEAN 1 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#pragma comment(lib, "psapi.lib") +#pragma comment(lib, "dbghelp.lib") + +#pragma pack( push, before_imagehlp, 8 ) +#include +#pragma pack( pop, before_imagehlp ) + +#include "include_base_utils.h" +#include "callstack_helper.h" + +namespace +{ + struct module_data + { + std::string image_name; + std::string module_name; + void *base_address; + DWORD load_size; + }; + + + class get_mod_info + { + HANDLE process; + static const int buffer_length = 4096; + public: + get_mod_info(HANDLE h) : process(h) {} + + module_data operator()(HMODULE module) + { + module_data ret; + char temp[buffer_length]; + MODULEINFO mi; + + GetModuleInformation(process, module, &mi, sizeof(mi)); + ret.base_address = mi.lpBaseOfDll; + ret.load_size = mi.SizeOfImage; + + GetModuleFileNameEx(process, module, temp, sizeof(temp)); + ret.image_name = temp; + GetModuleBaseName(process, module, temp, sizeof(temp)); + ret.module_name = temp; + std::vector img(ret.image_name.begin(), ret.image_name.end()); + std::vector mod(ret.module_name.begin(), ret.module_name.end()); + SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address, ret.load_size); + return ret; + } + }; + + std::string get_symbol_undecorated_name(HANDLE process, DWORD64 address, std::stringstream& ss) + { + SYMBOL_INFO* sym; + static const int max_name_len = 1024; + std::vector sym_buffer(sizeof(SYMBOL_INFO) + max_name_len, '\0'); + sym = (SYMBOL_INFO *)sym_buffer.data(); + sym->SizeOfStruct = sizeof(SYMBOL_INFO); + sym->MaxNameLen = max_name_len; + + DWORD64 displacement; + if (!SymFromAddr(process, address, &displacement, sym)) + return std::string("SymFromAddr failed1: ") + epee::string_tools::num_to_string_fast(GetLastError()); + + if (*sym->Name == '\0') + return std::string("SymFromAddr failed2: ") + epee::string_tools::num_to_string_fast(GetLastError()); + + /* + ss << " SizeOfStruct : " << sym->SizeOfStruct << ENDL; + ss << " TypeIndex : " << sym->TypeIndex << ENDL; // Type Index of symbol + ss << " Index : " << sym->Index << ENDL; + ss << " Size : " << sym->Size << ENDL; + ss << " ModBase : " << sym->ModBase << ENDL; // Base Address of module comtaining this symbol + ss << " Flags : " << sym->Flags << ENDL; + ss << " Value : " << sym->Value << ENDL; // Value of symbol, ValuePresent should be 1 + ss << " Address : " << sym->Address << ENDL; // Address of symbol including base address of module + ss << " Register : " << sym->Register << ENDL; // register holding value or pointer to value + ss << " Scope : " << sym->Scope << ENDL; // scope of the symbol + ss << " Tag : " << sym->Tag << ENDL; // pdb classification + ss << " NameLen : " << sym->NameLen << ENDL; // Actual length of name + ss << " MaxNameLen : " << sym->MaxNameLen << ENDL; + ss << " Name[1] : " << &sym->Name << ENDL; // Name of symbol + */ + + std::string und_name(max_name_len, '\0'); + DWORD chars_written = UnDecorateSymbolName(sym->Name, &und_name.front(), max_name_len, UNDNAME_COMPLETE); + und_name.resize(chars_written); + return und_name; + } + +} // namespace + +namespace tools +{ + + std::string get_callstack_win_x64() + { + HANDLE h_process = GetCurrentProcess(); + HANDLE h_thread = GetCurrentThread(); + + PCSTR user_search_path = NULL; // may be path to a pdb? + if (!SymInitialize(h_process, user_search_path, false)) + return ""; + + DWORD sym_options = SymGetOptions(); + sym_options |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME; + SymSetOptions(sym_options); + + DWORD cb_needed; + std::vector module_handles(1); + EnumProcessModules(h_process, &module_handles[0], module_handles.size() * sizeof(HMODULE), &cb_needed); + module_handles.resize(cb_needed / sizeof(HMODULE)); + EnumProcessModules(h_process, &module_handles[0], module_handles.size() * sizeof(HMODULE), &cb_needed); + + std::vector modules; + std::transform(module_handles.begin(), module_handles.end(), std::back_inserter(modules), get_mod_info(h_process)); + void *base = modules[0].base_address; + + CONTEXT context; + memset(&context, 0, sizeof context); + RtlCaptureContext( &context ); + + STACKFRAME64 frame; + memset(&frame, 0, sizeof frame); + frame.AddrPC.Offset = context.Rip; + frame.AddrPC.Mode = AddrModeFlat; + frame.AddrStack.Offset = context.Rsp; + frame.AddrStack.Mode = AddrModeFlat; + frame.AddrFrame.Offset = context.Rbp; + frame.AddrFrame.Mode = AddrModeFlat; + + IMAGEHLP_LINE64 line = { 0 }; + line.SizeOfStruct = sizeof line; + IMAGE_NT_HEADERS *image_nt_header = ImageNtHeader(base); + + std::stringstream ss; + ss << ENDL; + // ss << "main module loaded at 0x" << std::hex << std::setw(16) << std::setfill('0') << base << std::dec << " from " << modules[0].image_name << ENDL; + for (size_t n = 0; n < 250; ++n) + { + if (!StackWalk64(image_nt_header->FileHeader.Machine, h_process, h_thread, &frame, &context, NULL, SymFunctionTableAccess64, SymGetModuleBase64, NULL)) + break; + if (frame.AddrReturn.Offset == 0) + break; + + std::string fnName = get_symbol_undecorated_name(h_process, frame.AddrPC.Offset, ss); + ss << "0x" << std::setw(16) << std::setfill('0') << std::hex << frame.AddrPC.Offset << " " << std::dec << fnName; + DWORD offset_from_line = 0; + if (SymGetLineFromAddr64(h_process, frame.AddrPC.Offset, &offset_from_line, &line)) + ss << "+" << offset_from_line << " " << line.FileName << "(" << line.LineNumber << ")"; + + for (auto el : modules) + { + if ((DWORD64)el.base_address <= frame.AddrPC.Offset && frame.AddrPC.Offset < (DWORD64)el.base_address + (DWORD64)el.load_size) + { + ss << " : " << el.module_name << " @ 0x" << std::setw(0) << std::hex << (DWORD64)el.base_address << ENDL; + break; + } + } + + } + SymCleanup(h_process); + + return ss.str(); + } + +} // namespace tools + +#endif // #if defined(WIN32) diff --git a/src/common/callstack_helper.h b/src/common/callstack_helper.h new file mode 100644 index 00000000..1f3ef8e5 --- /dev/null +++ b/src/common/callstack_helper.h @@ -0,0 +1,24 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#pragma once +#include + +namespace tools +{ +#if defined(WIN32) + extern std::string get_callstack_win_x64(); +#endif + + inline std::string get_callstack() + { +#if defined(__GNUC__) + return epee::misc_utils::print_trace(); +#elif defined(WIN32) + return get_callstack_win_x64(); +#else + return ""; +#endif + } + +} // namespace tools diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 65b58cb1..e939e06a 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -26,6 +26,7 @@ using namespace epee; #include "common/miniupnp_helper.h" #include "version.h" #include "currency_core/core_tools.h" +#include "common/callstack_helper.h" #include @@ -108,6 +109,8 @@ int main(int argc, char* argv[]) std::fflush(nullptr); // all open output streams are flushed }); + epee::misc_utils::get_callstack(tools::get_callstack); + po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options", 130, 83); From ffa1d9ea55b0769b91da882ceaa12d9871ed9c57 Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 04:36:35 +0300 Subject: [PATCH 2/9] windown callstack enabled for Zano app as well as for daemon --- src/daemon/daemon.cpp | 1 + src/gui/qt-daemon/application/daemon_backend.cpp | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index e939e06a..e2e40876 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -109,6 +109,7 @@ int main(int argc, char* argv[]) std::fflush(nullptr); // all open output streams are flushed }); + // setup custom callstack retrieving function epee::misc_utils::get_callstack(tools::get_callstack); po::options_description desc_cmd_only("Command line options"); diff --git a/src/gui/qt-daemon/application/daemon_backend.cpp b/src/gui/qt-daemon/application/daemon_backend.cpp index b36568be..3c39d8e2 100644 --- a/src/gui/qt-daemon/application/daemon_backend.cpp +++ b/src/gui/qt-daemon/application/daemon_backend.cpp @@ -9,7 +9,7 @@ #include "core_fast_rpc_proxy.h" #include "string_coding.h" #include "currency_core/core_tools.h" -//#include +#include "common/callstack_helper.h" #define GET_WALLET_OPT_BY_ID(wallet_id, name) \ CRITICAL_REGION_LOCAL(m_wallets_lock); \ @@ -85,6 +85,8 @@ bool daemon_backend::init(int argc, char* argv[], view::i_view* pview_handler) std::fflush(nullptr); // all open output streams are flushed }); + // setup custom callstack retrieving function + epee::misc_utils::get_callstack(tools::get_callstack); //#if !defined(NDEBUG) // log_space::log_singletone::add_logger(LOGGER_DEBUGGER, nullptr, nullptr); From 2ccc082ff9d4e37e5927e4ba2e5ed817d18e7394 Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 04:37:24 +0300 Subject: [PATCH 3/9] project version is now added to crashdump filename --- src/common/util.cpp | 65 ++++++++++++++++++++++++++++++++++++++++++++- src/common/util.h | 62 +----------------------------------------- 2 files changed, 65 insertions(+), 62 deletions(-) diff --git a/src/common/util.cpp b/src/common/util.cpp index c4c179b7..c9597f21 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -10,6 +10,7 @@ using namespace epee; #include "util.h" #include "currency_core/currency_config.h" +#include "version.h" #ifdef WIN32 #include @@ -362,7 +363,69 @@ namespace tools return pszOS; } } -#else + + void signal_handler::GenerateCrashDump(EXCEPTION_POINTERS *pep /* = NULL*/) + { + SYSTEMTIME sysTime = { 0 }; + GetSystemTime(&sysTime); + // get the computer name + char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 }; + DWORD compNameLen = ARRAYSIZE(compName); + GetComputerNameA(compName, &compNameLen); + // build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP + char path[MAX_PATH*10] = { 0 }; + std::string folder = epee::log_space::log_singletone::get_default_log_folder(); + sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_" PROJECT_VERSION_LONG "_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp", + folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay, + sysTime.wHour, sysTime.wMinute, sysTime.wSecond); + + HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, + 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + + if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE)) + { + // Create the minidump + MINIDUMP_EXCEPTION_INFORMATION mdei; + + mdei.ThreadId = GetCurrentThreadId(); + mdei.ExceptionPointers = pep; + mdei.ClientPointers = FALSE; + + MINIDUMP_CALLBACK_INFORMATION mci; + + mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback; + mci.CallbackParam = 0; + + MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory | + MiniDumpWithDataSegs | + MiniDumpWithHandleData | + MiniDumpWithFullMemoryInfo | + MiniDumpWithThreadInfo | + MiniDumpWithUnloadedModules); + + BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), + hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci); + + if (!rv) + { + LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path); + } + else + { + LOG_PRINT_L0("Minidump file created on path: " << path); + } + // Close the file + CloseHandle(hFile); + } + else + { + LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path); + } + } + + + +#else // ifdef WIN32 std::string get_nix_version_display_string() { utsname un; diff --git a/src/common/util.h b/src/common/util.h index e6989acf..ce01088c 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -231,67 +231,7 @@ namespace tools } - static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL) - - { - SYSTEMTIME sysTime = { 0 }; - GetSystemTime(&sysTime); - // get the computer name - char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 }; - DWORD compNameLen = ARRAYSIZE(compName); - GetComputerNameA(compName, &compNameLen); - // build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP - char path[MAX_PATH*10] = { 0 }; - std::string folder = epee::log_space::log_singletone::get_default_log_folder(); - sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp", - folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay, - sysTime.wHour, sysTime.wMinute, sysTime.wSecond); - - HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, - 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - - if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE)) - { - // Create the minidump - MINIDUMP_EXCEPTION_INFORMATION mdei; - - mdei.ThreadId = GetCurrentThreadId(); - mdei.ExceptionPointers = pep; - mdei.ClientPointers = FALSE; - - MINIDUMP_CALLBACK_INFORMATION mci; - - mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback; - mci.CallbackParam = 0; - - MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory | - MiniDumpWithDataSegs | - MiniDumpWithHandleData | - MiniDumpWithFullMemoryInfo | - MiniDumpWithThreadInfo | - MiniDumpWithUnloadedModules); - - BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), - hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci); - - if (!rv) - { - LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path); - } - else - { - LOG_PRINT_L0("Minidump file created on path: " << path); - } - // Close the file - CloseHandle(hFile); - } - else - { - LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path); - } - } - - + static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL); static LONG WINAPI win_unhandled_exception_handler(_In_ struct _EXCEPTION_POINTERS *ep) { From fb66630676935e93840af8d4c34b41cd5cd2cdbc Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 04:40:55 +0300 Subject: [PATCH 4/9] === build number: 46 -> 47 === --- src/version.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h.in b/src/version.h.in index 9e7da9bb..a0e92f25 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -2,6 +2,6 @@ #define BUILD_COMMIT_ID "@VERSION@" #define PROJECT_VERSION "1.0" -#define PROJECT_VERSION_BUILD_NO 46 +#define PROJECT_VERSION_BUILD_NO 47 #define PROJECT_VERSION_BUILD_NO_STR STRINGIFY_EXPAND(PROJECT_VERSION_BUILD_NO) #define PROJECT_VERSION_LONG PROJECT_VERSION "." PROJECT_VERSION_BUILD_NO_STR "[" BUILD_COMMIT_ID "]" From 972acf2632821ea8bd0079bb7aac0240df6f3a3f Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 12:21:05 +0300 Subject: [PATCH 5/9] lmdb back end reverted --- src/common/db_backend_lmdb.cpp | 69 ++++------------------------------ src/common/db_backend_lmdb.h | 15 ++------ 2 files changed, 10 insertions(+), 74 deletions(-) diff --git a/src/common/db_backend_lmdb.cpp b/src/common/db_backend_lmdb.cpp index ceb46796..fdb289ff 100644 --- a/src/common/db_backend_lmdb.cpp +++ b/src/common/db_backend_lmdb.cpp @@ -10,10 +10,6 @@ #include "util.h" #define BUF_SIZE 1024 -#define DB_RESIZE_MIN_FREE_SIZE (100 * 1024 * 1024) // DB map size will grow if that much space left on DB -#define DB_RESIZE_MIN_MAX_SIZE (50 * 1024 * 1024) // Minimum DB map size (starting size) -#define DB_RESIZE_INCREMENT_SIZE (100 * 1024 * 1024) // Grow step size -#define DB_RESIZE_COMMITS_TO_CHECK 50 #define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); #define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); @@ -27,13 +23,10 @@ namespace tools { namespace db { - lmdb_db_backend::lmdb_db_backend() - : m_penv(AUTO_VAL_INIT(m_penv)) - , m_commits_count(0) + lmdb_db_backend::lmdb_db_backend() : m_penv(AUTO_VAL_INIT(m_penv)) { } - lmdb_db_backend::~lmdb_db_backend() { NESTED_TRY_ENTRY(); @@ -43,16 +36,18 @@ namespace tools NESTED_CATCH_ENTRY(__func__); } - bool lmdb_db_backend::open(const std::string& path_, uint64_t /* flags -- unused atm */) + bool lmdb_db_backend::open(const std::string& path_, uint64_t cache_sz) { int res = 0; res = mdb_env_create(&m_penv); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_create"); - MDB_dbi max_dbs = 15; - res = mdb_env_set_maxdbs(m_penv, max_dbs); + res = mdb_env_set_maxdbs(m_penv, 15); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_maxdbs"); + res = mdb_env_set_mapsize(m_penv, cache_sz); + CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize"); + m_path = path_; #ifdef WIN32 m_path = epee::string_encoding::convert_ansii_to_utf8(m_path); @@ -60,14 +55,8 @@ namespace tools CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path); - // TODO: convert flags to lmdb_flags (implement fast lmdb modes) - unsigned int lmdb_flags = MDB_NORDAHEAD /*| MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC*/; - mdb_mode_t lmdb_mode = 0644; - - res = mdb_env_open(m_penv, m_path.c_str(), lmdb_flags, lmdb_mode); + res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD , 0644); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_open, m_path=" << m_path); - - resize_if_needed(); return true; } @@ -116,13 +105,6 @@ namespace tools { LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3); CRITICAL_SECTION_LOCK(m_write_exclusive_lock); - - if (m_commits_count.fetch_add(1, std::memory_order_relaxed) % DB_RESIZE_COMMITS_TO_CHECK == DB_RESIZE_COMMITS_TO_CHECK - 1) - { - if (!resize_if_needed()) - m_commits_count.store(DB_RESIZE_COMMITS_TO_CHECK - 1, std::memory_order_relaxed); // if failed, try again on next commit - } - } PROFILE_FUNC("lmdb_db_backend::begin_transaction"); { @@ -347,7 +329,6 @@ namespace tools CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put"); return true; } - bool lmdb_db_backend::enumerate(container_handle h, i_db_callback* pcb) { CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate"); @@ -402,42 +383,6 @@ namespace tools } return true; } - - bool lmdb_db_backend::resize_if_needed() - { - LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED in resize_if_needed()", LOG_LEVEL_3); - CRITICAL_REGION_LOCAL(m_write_exclusive_lock); - - if (have_tx()) - { - LOG_PRINT_RED("[DB " << m_path << "] : resize_if_needed(): Have txs on stack, unable to resize!", LOG_LEVEL_0); - return false; - } - - MDB_stat st = AUTO_VAL_INIT(st); - mdb_env_stat(m_penv, &st); - MDB_envinfo ei = AUTO_VAL_INIT(ei); - mdb_env_info(m_penv, &ei); - - uint64_t dirty_size = ei.me_last_pgno * st.ms_psize; - int64_t size_diff = ei.me_mapsize - dirty_size; - if (size_diff >= DB_RESIZE_MIN_FREE_SIZE && ei.me_mapsize >= DB_RESIZE_MIN_MAX_SIZE) - return true; // resize is not needed - - double gigabyte = 1024 * 1024 * 1024; - const uint64_t increment_size_pg_aligned = DB_RESIZE_INCREMENT_SIZE - (DB_RESIZE_INCREMENT_SIZE % st.ms_psize); - - // need to resize DB - uint64_t new_size = ei.me_mapsize - (ei.me_mapsize % increment_size_pg_aligned) + increment_size_pg_aligned; - - int res = mdb_env_set_mapsize(m_penv, new_size); - CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize"); - - LOG_PRINT_CYAN("[DB " << m_path << "] has grown: " << std::fixed << std::setprecision(2) << ei.me_mapsize / gigabyte << " GiB -> " << std::fixed << std::setprecision(2) << new_size / gigabyte << " GiB", LOG_LEVEL_0); - - return true; - } - } } diff --git a/src/common/db_backend_lmdb.h b/src/common/db_backend_lmdb.h index 84c441d3..95c9d8a9 100644 --- a/src/common/db_backend_lmdb.h +++ b/src/common/db_backend_lmdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,6 @@ #pragma once #include - #include "include_base_utils.h" #include "db_backend_base.h" @@ -17,9 +16,6 @@ namespace tools { namespace db { - - - class lmdb_db_backend : public i_db_backend { @@ -38,11 +34,7 @@ namespace tools boost::recursive_mutex m_cs; boost::recursive_mutex m_write_exclusive_lock; std::map m_txs; // size_t -> count of nested read_only transactions - std::atomic m_commits_count; - bool pop_tx_entry(tx_entry& txe); - - public: lmdb_db_backend(); ~lmdb_db_backend(); @@ -52,7 +44,7 @@ namespace tools bool begin_transaction(bool read_only = false); bool commit_transaction(); void abort_transaction(); - bool open(const std::string& path, uint64_t flags = 0); + bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE); bool open_container(const std::string& name, container_handle& h); bool erase(container_handle h, const char* k, size_t s); bool get(container_handle h, const char* k, size_t s, std::string& res_buff); @@ -64,8 +56,7 @@ namespace tools //------------------------------------------------------------------------------------- bool have_tx(); MDB_txn* get_current_tx(); - bool resize_if_needed(); }; } -} \ No newline at end of file +} From c0d9f0f5f016abea6bd6139881d530dc22b7918f Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 12:21:33 +0300 Subject: [PATCH 6/9] === build number: 47 -> 48 === --- src/version.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h.in b/src/version.h.in index a0e92f25..351ac76a 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -2,6 +2,6 @@ #define BUILD_COMMIT_ID "@VERSION@" #define PROJECT_VERSION "1.0" -#define PROJECT_VERSION_BUILD_NO 47 +#define PROJECT_VERSION_BUILD_NO 48 #define PROJECT_VERSION_BUILD_NO_STR STRINGIFY_EXPAND(PROJECT_VERSION_BUILD_NO) #define PROJECT_VERSION_LONG PROJECT_VERSION "." PROJECT_VERSION_BUILD_NO_STR "[" BUILD_COMMIT_ID "]" From 6d173d70774dbdd0abcf91d30b687d4b49480732 Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 12:28:05 +0300 Subject: [PATCH 7/9] gcc compilation fixed --- src/common/callstack_helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/callstack_helper.h b/src/common/callstack_helper.h index 1f3ef8e5..3853d148 100644 --- a/src/common/callstack_helper.h +++ b/src/common/callstack_helper.h @@ -13,7 +13,7 @@ namespace tools inline std::string get_callstack() { #if defined(__GNUC__) - return epee::misc_utils::print_trace(); + return epee::misc_utils::print_trace_default(); #elif defined(WIN32) return get_callstack_win_x64(); #else From e07e4b0b9b237118587631208c7b291b60dc216f Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 18:44:59 +0300 Subject: [PATCH 8/9] correctly handle null ptr that may be returned from progpow in get_block_longhash() --- contrib/ethereum/libethash/ethash/ethash.hpp | 2 +- contrib/ethereum/libethash/managed.cpp | 4 ++-- src/currency_core/basic_pow_helpers.cpp | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/ethereum/libethash/ethash/ethash.hpp b/contrib/ethereum/libethash/ethash/ethash.hpp index 03f29afd..8034090b 100644 --- a/contrib/ethereum/libethash/ethash/ethash.hpp +++ b/contrib/ethereum/libethash/ethash/ethash.hpp @@ -156,5 +156,5 @@ int find_epoch_number(const hash256& seed) noexcept; const epoch_context& get_global_epoch_context(int epoch_number); /// Get global shared epoch context with full dataset initialized. -const epoch_context_full& get_global_epoch_context_full(int epoch_number); +std::shared_ptr get_global_epoch_context_full(int epoch_number); } // namespace ethash diff --git a/contrib/ethereum/libethash/managed.cpp b/contrib/ethereum/libethash/managed.cpp index 900da7e7..82ff69b2 100644 --- a/contrib/ethereum/libethash/managed.cpp +++ b/contrib/ethereum/libethash/managed.cpp @@ -89,12 +89,12 @@ const epoch_context& get_global_epoch_context(int epoch_number) return *thread_local_context; } -const epoch_context_full& get_global_epoch_context_full(int epoch_number) +std::shared_ptr get_global_epoch_context_full(int epoch_number) { // Check if local context matches epoch number. if (!thread_local_context_full || thread_local_context_full->epoch_number != epoch_number) update_local_context_full(epoch_number); - return *thread_local_context_full; + return thread_local_context_full; } } // namespace ethash diff --git a/src/currency_core/basic_pow_helpers.cpp b/src/currency_core/basic_pow_helpers.cpp index 7c2275cc..b97a1087 100644 --- a/src/currency_core/basic_pow_helpers.cpp +++ b/src/currency_core/basic_pow_helpers.cpp @@ -50,8 +50,9 @@ namespace currency crypto::hash get_block_longhash(uint64_t height, const crypto::hash& block_header_hash, uint64_t nonce) { int epoch = ethash_height_to_epoch(height); - const auto& context = progpow::get_global_epoch_context_full(static_cast(epoch)); - auto res_eth = progpow::hash(context, static_cast(height), *(ethash::hash256*)&block_header_hash, nonce); + std::shared_ptr p_context = progpow::get_global_epoch_context_full(static_cast(epoch)); + CHECK_AND_ASSERT_THROW_MES(p_context, "progpow::get_global_epoch_context_full returned null"); + auto res_eth = progpow::hash(*p_context, static_cast(height), *(ethash::hash256*)&block_header_hash, nonce); crypto::hash result = currency::null_hash; memcpy(&result.data, &res_eth.final_hash, sizeof(res_eth.final_hash)); return result; From a8173575bb100d23903bbae5fd3a58ba374e2f07 Mon Sep 17 00:00:00 2001 From: sowle Date: Thu, 29 Aug 2019 18:45:59 +0300 Subject: [PATCH 9/9] === build number: 48 -> 49 === --- src/version.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h.in b/src/version.h.in index 351ac76a..307ec14d 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -2,6 +2,6 @@ #define BUILD_COMMIT_ID "@VERSION@" #define PROJECT_VERSION "1.0" -#define PROJECT_VERSION_BUILD_NO 48 +#define PROJECT_VERSION_BUILD_NO 49 #define PROJECT_VERSION_BUILD_NO_STR STRINGIFY_EXPAND(PROJECT_VERSION_BUILD_NO) #define PROJECT_VERSION_LONG PROJECT_VERSION "." PROJECT_VERSION_BUILD_NO_STR "[" BUILD_COMMIT_ID "]"