an attempt to fix a possible deadlock at higher level triggered by lmdb resizing (randomly blocking begin_transaction when preparing)

contrib/epee/include/misc_log_ex.h

@@ -150,7 +150,7 @@ DISABLE_VS_WARNINGS(4100)
   {std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, true, log_name);}}
 
 #define LOG_ERROR2(log_name, x) { \
-  std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::print_trace() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL); }
+  std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::get_callstack() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL); }
 
 #define LOG_FRAME2(log_name, x, y) epee::log_space::log_frame frame(x, y, log_name)
 

contrib/epee/include/misc_os_dependent.h

@@ -113,7 +113,7 @@ namespace misc_utils
 #include <execinfo.h>
 #include <boost/core/demangle.hpp>
 #endif
-  inline std::string print_trace()
+  inline std::string print_trace_default()
   {
     std::stringstream ss;
 #if defined(__GNUC__)
@@ -134,5 +134,33 @@ namespace misc_utils
 #endif
     return ss.str();
   }
+
+  typedef std::string (stack_retrieving_function_t)();
+
+  //
+  // To get stack trace call it with the defaults.
+  // 
+  inline std::string get_callstack(stack_retrieving_function_t* p_stack_retrieving_function_to_be_added = nullptr, bool remove_func = false)
+  {
+    static stack_retrieving_function_t* p_srf = nullptr;
+    
+    if (remove_func)
+    {
+      p_srf = nullptr;
+      return "";
+    }
+    
+    if (p_stack_retrieving_function_to_be_added != nullptr)
+    {
+      p_srf = p_stack_retrieving_function_to_be_added;
+      return "";
+    }
+
+    if (p_srf != nullptr)
+      return p_srf();
+
+    return print_trace_default();
+  }
+
 }
 }

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -126,7 +126,7 @@ namespace epee
         t_result result_struct = AUTO_VAL_INIT(result_struct);
         if( code <=0 )
         {
-          LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::print_trace());
+          LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack());
           LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code << "(" << epee::levin::get_err_descr(code) << ")context:" << print_connection_context(context));
           TRY_ENTRY()
           cb(code, result_struct, context);
@@ -150,7 +150,7 @@ namespace epee
       }, inv_timeout);
       if( res <=0 )
       {
-        LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::print_trace());
+        LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack());
         LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res << "(" << epee::levin::get_err_descr(res) << ") conn_id=" << conn_id);
         return false;
       }

src/common/callstack_helper.cpp

@@ -0,0 +1,179 @@
+// Copyright (c) 2019 Zano Project
+// Distributed under the MIT/X11 software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#if defined(WIN32)
+
+#define WIN32_LEAN_AND_MEAN 1
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <Psapi.h>
+#pragma comment(lib, "psapi.lib")
+#pragma comment(lib, "dbghelp.lib")
+
+#pragma pack( push, before_imagehlp, 8 )
+#include <imagehlp.h>
+#pragma pack( pop, before_imagehlp )
+
+#include "include_base_utils.h"
+#include "callstack_helper.h"
+
+namespace
+{
+  struct module_data
+  {
+    std::string image_name;
+    std::string module_name;
+    void *base_address;
+    DWORD load_size;
+  };
+
+
+  class get_mod_info
+  {
+    HANDLE process;
+    static const int buffer_length = 4096;
+  public:
+    get_mod_info(HANDLE h) : process(h) {}
+
+    module_data operator()(HMODULE module)
+    {
+      module_data ret;
+      char temp[buffer_length];
+      MODULEINFO mi;
+
+      GetModuleInformation(process, module, &mi, sizeof(mi));
+      ret.base_address = mi.lpBaseOfDll;
+      ret.load_size = mi.SizeOfImage;
+
+      GetModuleFileNameEx(process, module, temp, sizeof(temp));
+      ret.image_name = temp;
+      GetModuleBaseName(process, module, temp, sizeof(temp));
+      ret.module_name = temp;
+      std::vector<char> img(ret.image_name.begin(), ret.image_name.end());
+      std::vector<char> mod(ret.module_name.begin(), ret.module_name.end());
+      SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address, ret.load_size);
+      return ret;
+    }
+  };
+
+  std::string get_symbol_undecorated_name(HANDLE process, DWORD64 address, std::stringstream& ss)
+  {
+    SYMBOL_INFO* sym;
+    static const int max_name_len = 1024;
+    std::vector<char> sym_buffer(sizeof(SYMBOL_INFO) + max_name_len, '\0');
+    sym = (SYMBOL_INFO *)sym_buffer.data();
+    sym->SizeOfStruct = sizeof(SYMBOL_INFO);
+    sym->MaxNameLen = max_name_len;
+
+    DWORD64 displacement;
+    if (!SymFromAddr(process, address, &displacement, sym))
+      return std::string("SymFromAddr failed1: ") + epee::string_tools::num_to_string_fast(GetLastError());
+
+    if (*sym->Name == '\0')
+      return std::string("SymFromAddr failed2: ") + epee::string_tools::num_to_string_fast(GetLastError());
+
+    /*
+    ss << "  SizeOfStruct : " << sym->SizeOfStruct << ENDL;
+    ss << "  TypeIndex    : " << sym->TypeIndex << ENDL;        // Type Index of symbol
+    ss << "  Index        : " << sym->Index << ENDL;
+    ss << "  Size         : " << sym->Size << ENDL;
+    ss << "  ModBase      : " << sym->ModBase << ENDL;          // Base Address of module comtaining this symbol
+    ss << "  Flags        : " << sym->Flags << ENDL;
+    ss << "  Value        : " << sym->Value << ENDL;            // Value of symbol, ValuePresent should be 1
+    ss << "  Address      : " << sym->Address << ENDL;          // Address of symbol including base address of module
+    ss << "  Register     : " << sym->Register << ENDL;         // register holding value or pointer to value
+    ss << "  Scope        : " << sym->Scope << ENDL;            // scope of the symbol
+    ss << "  Tag          : " << sym->Tag << ENDL;              // pdb classification
+    ss << "  NameLen      : " << sym->NameLen << ENDL;          // Actual length of name
+    ss << "  MaxNameLen   : " << sym->MaxNameLen << ENDL;
+    ss << "  Name[1]      : " << &sym->Name << ENDL;            // Name of symbol
+    */
+
+    std::string und_name(max_name_len, '\0');
+    DWORD chars_written = UnDecorateSymbolName(sym->Name, &und_name.front(), max_name_len, UNDNAME_COMPLETE);
+    und_name.resize(chars_written);
+    return und_name;
+  }
+
+} // namespace
+
+namespace tools
+{
+
+  std::string get_callstack_win_x64()
+  {
+    HANDLE h_process = GetCurrentProcess();
+    HANDLE h_thread = GetCurrentThread();
+
+    PCSTR user_search_path = NULL; // may be path to a pdb?
+    if (!SymInitialize(h_process, user_search_path, false))
+      return "<win callstack error 1>";
+
+    DWORD sym_options = SymGetOptions();
+    sym_options |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME;
+    SymSetOptions(sym_options);
+
+    DWORD cb_needed;
+    std::vector<HMODULE> module_handles(1);
+    EnumProcessModules(h_process, &module_handles[0], module_handles.size() * sizeof(HMODULE), &cb_needed);
+    module_handles.resize(cb_needed / sizeof(HMODULE));
+    EnumProcessModules(h_process, &module_handles[0], module_handles.size() * sizeof(HMODULE), &cb_needed);
+
+    std::vector<module_data> modules;
+    std::transform(module_handles.begin(), module_handles.end(), std::back_inserter(modules), get_mod_info(h_process));
+    void *base = modules[0].base_address;
+
+    CONTEXT context;
+    memset(&context, 0, sizeof context);
+    RtlCaptureContext( &context );
+
+    STACKFRAME64 frame;
+    memset(&frame, 0, sizeof frame);
+    frame.AddrPC.Offset = context.Rip;
+    frame.AddrPC.Mode = AddrModeFlat;
+    frame.AddrStack.Offset = context.Rsp;
+    frame.AddrStack.Mode = AddrModeFlat;
+    frame.AddrFrame.Offset = context.Rbp;
+    frame.AddrFrame.Mode = AddrModeFlat;
+  
+    IMAGEHLP_LINE64 line = { 0 };
+    line.SizeOfStruct = sizeof line;
+    IMAGE_NT_HEADERS *image_nt_header = ImageNtHeader(base);
+
+    std::stringstream ss;
+    ss << ENDL;
+    // ss << "main module loaded at 0x" << std::hex << std::setw(16) << std::setfill('0') << base << std::dec << " from " << modules[0].image_name << ENDL;
+    for (size_t n = 0; n < 250; ++n)
+    {
+      if (!StackWalk64(image_nt_header->FileHeader.Machine, h_process, h_thread, &frame, &context, NULL, SymFunctionTableAccess64, SymGetModuleBase64, NULL))
+        break;
+      if (frame.AddrReturn.Offset == 0)
+        break;
+
+      std::string fnName = get_symbol_undecorated_name(h_process, frame.AddrPC.Offset, ss);
+      ss << "0x" << std::setw(16) << std::setfill('0') << std::hex << frame.AddrPC.Offset << " " << std::dec << fnName;
+      DWORD offset_from_line = 0;
+      if (SymGetLineFromAddr64(h_process, frame.AddrPC.Offset, &offset_from_line, &line))
+        ss << "+" << offset_from_line << " " << line.FileName << "(" << line.LineNumber << ")";
+
+      for (auto el : modules)
+      {
+        if ((DWORD64)el.base_address <= frame.AddrPC.Offset && frame.AddrPC.Offset < (DWORD64)el.base_address + (DWORD64)el.load_size)
+        {
+          ss << " : " << el.module_name << " @ 0x" << std::setw(0) << std::hex << (DWORD64)el.base_address << ENDL;
+          break;
+        }
+      }
+
+    }
+    SymCleanup(h_process);
+
+    return ss.str();
+  }
+
+} // namespace tools
+
+#endif // #if defined(WIN32)

src/common/callstack_helper.h

@@ -0,0 +1,24 @@
+// Copyright (c) 2019 Zano Project
+// Distributed under the MIT/X11 software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#pragma once
+#include <epee/include/misc_os_dependent.h>
+
+namespace tools
+{
+#if defined(WIN32)
+  extern std::string get_callstack_win_x64();
+#endif
+
+  inline std::string get_callstack()
+  {
+#if defined(__GNUC__)
+    return epee::misc_utils::print_trace();
+#elif defined(WIN32)
+    return get_callstack_win_x64();
+#else
+    return "";
+#endif
+  }
+
+} // namespace tools

src/common/db_backend_lmdb.cpp

@@ -10,18 +10,38 @@
 #include "util.h"
 
 #define BUF_SIZE 1024
-#define DB_RESIZE_MIN_FREE_SIZE    (100 * 1024 * 1024) // DB map size will grow if that much space left on DB
-#define DB_RESIZE_MIN_MAX_SIZE     (50 * 1024 * 1024)  // Minimum DB map size (starting size)
-#define DB_RESIZE_INCREMENT_SIZE   (100 * 1024 * 1024) // Grow step size
-#define DB_RESIZE_COMMITS_TO_CHECK 50
+#define DB_RESIZE_MIN_FREE_SIZE    (8 * 1024 * 1024)  // DB resize will triggeres if that much space left on DB map
+#define DB_RESIZE_MIN_MAX_SIZE     (1 * 1024 * 1024)  // DB resize will triggered if DB map size is less than this
+#define DB_RESIZE_INCREMENT_SIZE   (5 * 1024 * 1024)  // Grow step size
+#define DB_RESIZE_COMMITS_TO_CHECK 1
 
-#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
-#define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
-#define ASSERT_MES_AND_THROW_LMDB(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
+#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess << ENDL << "LMDB " << get_brief_lmdb_stat(m_penv));
+#define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess << ENDL << "LMDB " << get_brief_lmdb_stat(m_penv));
+#define ASSERT_MES_AND_THROW_LMDB(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess << ENDL << "LMDB " << get_brief_lmdb_stat(m_penv));
 
 #undef LOG_DEFAULT_CHANNEL 
 #define LOG_DEFAULT_CHANNEL "lmdb"
-// 'lmdb' channel is disabled by default
+ENABLE_CHANNEL_BY_DEFAULT("lmdb");
+
+namespace
+{
+  std::string get_brief_lmdb_stat(MDB_env* p_env)
+  {
+    MDB_stat st = AUTO_VAL_INIT(st);
+    mdb_env_stat(p_env, &st);
+    MDB_envinfo ei = AUTO_VAL_INIT(ei);
+    mdb_env_info(p_env, &ei);
+    
+    static const double megabyte = 1024 * 1024;
+    
+    std::stringstream ss;
+    ss << "mapsz: " << std::fixed << std::setprecision(2) << ei.me_mapsize / megabyte
+       << " MB, dirty: " << ei.me_last_pgno * st.ms_psize / megabyte
+       << " MB, free: " << (ei.me_mapsize - ei.me_last_pgno * st.ms_psize) / megabyte
+       << " MB (" << (100.0 * (ei.me_mapsize - ei.me_last_pgno * st.ms_psize) / ei.me_mapsize) << "%)";
+    return ss.str();
+  }
+}
 
 namespace tools
 {
@@ -30,6 +50,9 @@ namespace tools
     lmdb_db_backend::lmdb_db_backend()
       : m_penv(AUTO_VAL_INIT(m_penv))
       , m_commits_count(0)
+      , m_resize_thread_active(false)
+      , m_resize_wait_condition(false)
+      , m_resize_wait_txs_count(0)
     {
 
     }
@@ -38,6 +61,8 @@ namespace tools
     {
       NESTED_TRY_ENTRY();
 
+      wait_for_resize_if_needed();
+
       close();
 
       NESTED_CATCH_ENTRY(__func__);
@@ -112,17 +137,15 @@ namespace tools
 
     bool lmdb_db_backend::begin_transaction(bool read_only)
     {
+      wait_for_resize_if_needed();
+
       if (!read_only)
       {
         LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3);
         CRITICAL_SECTION_LOCK(m_write_exclusive_lock);
 
         if (m_commits_count.fetch_add(1, std::memory_order_relaxed) % DB_RESIZE_COMMITS_TO_CHECK == DB_RESIZE_COMMITS_TO_CHECK - 1)
-        {
-          if (!resize_if_needed())
-            m_commits_count.store(DB_RESIZE_COMMITS_TO_CHECK - 1, std::memory_order_relaxed); // if failed, try again on next commit
-        }
-
+          resize_if_needed();
       }
       PROFILE_FUNC("lmdb_db_backend::begin_transaction");
       {
@@ -233,6 +256,8 @@ namespace tools
         } 
       }
       LOG_PRINT_L4("[DB] Transaction committed");
+
+      notify_resize_thread_on_tx_end();
       return true;
     }
     
@@ -251,10 +276,10 @@ namespace tools
             LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED(ABORTED)", LOG_LEVEL_3);
           }
         }
-          
-
       }
       LOG_PRINT_L4("[DB] Transaction aborted");
+
+      notify_resize_thread_on_tx_end();
     }
 
     bool lmdb_db_backend::erase(container_handle h, const char* k, size_t ks)
@@ -344,7 +369,7 @@ namespace tools
       data.mv_size = vs;
 
       res = mdb_put(get_current_tx(), static_cast<MDB_dbi>(h), &key, &data, 0);
-      CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put");
+      CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put, ks=" << ks << ", vs=" << vs);
       return true;
     }
 
@@ -403,39 +428,143 @@ namespace tools
       return true;
     }
 
-    bool lmdb_db_backend::resize_if_needed()
+    void lmdb_db_backend::wait_for_resize_if_needed()
     {
-      LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED in resize_if_needed()", LOG_LEVEL_3);
-      CRITICAL_REGION_LOCAL(m_write_exclusive_lock);
-
-      if (have_tx())
+      // wait for m_resize_wait_condition == false
       {
-        LOG_PRINT_RED("[DB " << m_path << "] : resize_if_needed(): Have txs on stack, unable to resize!", LOG_LEVEL_0);
-        return false;
+        std::unique_lock<decltype(m_resize_wait_mutex)> lock(m_resize_wait_mutex);
+        m_resize_wait_cv.wait(lock, [this](){ return !m_resize_wait_condition; });
       }
 
+      // increment current tx counter
+      {
+        std::lock_guard<std::mutex> lock(m_resize_wait_txs_count_mutex);
+        ++m_resize_wait_txs_count;
+      }
+    }
+
+    void lmdb_db_backend::notify_resize_thread_on_tx_end()
+    {
+      // decrement current tx counter
+      {
+        std::lock_guard<std::mutex> lock(m_resize_wait_txs_count_mutex);
+        --m_resize_wait_txs_count;
+      }
+
+      // notify resize thread on counter changed so it can go ahead when it's zero
+      m_resize_wait_txs_cv.notify_one();
+    }
+
+
+    bool lmdb_db_backend::resize_condition(const MDB_stat& st, const MDB_envinfo& ei)
+    {
+      uint64_t dirty_size = ei.me_last_pgno * st.ms_psize;
+      int64_t size_left = ei.me_mapsize - dirty_size;
+      return size_left < DB_RESIZE_MIN_FREE_SIZE || ei.me_mapsize < DB_RESIZE_MIN_MAX_SIZE;
+    }
+
+    void lmdb_db_backend::resize_thread()
+    {
+      epee::log_space::log_singletone::set_thread_log_prefix("[lmdb resize]");
+      if (m_resize_thread_active)
+        return; // another thread already started
+      m_resize_thread_active = true;
+      auto slh = epee::misc_utils::create_scope_leave_handler([&](){
+        m_resize_thread_active = false;
+      });
+      LOG_PRINT_CYAN("[DB " << m_path << "] resize thread started", LOG_LEVEL_0);
+
+      // asquare an exclusive access to LMDB
+
+      // make sure all threads will be freed on m_resize_wait_cv on exit
+      auto slh2 = epee::misc_utils::create_scope_leave_handler([&](){
+        {
+          std::lock_guard<decltype(m_resize_wait_mutex)> lock(m_resize_wait_mutex);
+          m_resize_wait_condition = false;
+        }
+        m_resize_wait_cv.notify_all();
+      });
+
+      size_t iter_n = 0;
+      while(true)
+      {
+        ++iter_n;
+        // prevent all new txs from starting
+        {
+          std::lock_guard<decltype(m_resize_wait_mutex)> lock(m_resize_wait_mutex);
+          m_resize_wait_condition = true;
+        }
+
+        // wait for all ongoing lmdb txs are finished (until m_resize_wait_txs_count_mutex is zero)
+        {
+          std::unique_lock<std::mutex> lock(m_resize_wait_txs_count_mutex);
+          if (m_resize_wait_txs_cv.wait_for(lock, std::chrono::seconds(2), [this](){ return m_resize_wait_txs_count == 0; }))
+            break; // m_resize_wait_txs_count becomes zero within specified period, access acquired
+        }
+
+        LOG_PRINT_CYAN("[DB " << m_path << "] iteration " << iter_n << ": failed to get exclusive access within the time, seems to be deadlock, releasing....", LOG_LEVEL_0);
+
+        // okay, we failed to get all txs fineshed, seems to be deadlock
+        // release all blocked txs
+        {
+          std::lock_guard<decltype(m_resize_wait_mutex)> lock(m_resize_wait_mutex);
+          m_resize_wait_condition = false;
+        }
+        m_resize_wait_cv.notify_all();
+
+        // sleep some time and try again
+        std::this_thread::sleep_for(std::chrono::seconds(2));
+      }
+
+      LOG_PRINT_CYAN("[DB " << m_path << "] exclusive access acquired", LOG_LEVEL_0);
+
+      // check resize condition again and calculate new size
+
       MDB_stat st = AUTO_VAL_INIT(st);
       mdb_env_stat(m_penv, &st);
       MDB_envinfo ei = AUTO_VAL_INIT(ei);
       mdb_env_info(m_penv, &ei);
 
-      uint64_t dirty_size = ei.me_last_pgno * st.ms_psize;
-      int64_t size_diff = ei.me_mapsize - dirty_size;
-      if (size_diff >= DB_RESIZE_MIN_FREE_SIZE && ei.me_mapsize >= DB_RESIZE_MIN_MAX_SIZE)
-        return true; // resize is not needed
+      if (!resize_condition(st, ei))
+      {
+        LOG_PRINT_RED("resize is not needed: " << ei.me_mapsize << ", " << ei.me_mapsize - ei.me_last_pgno * st.ms_psize << ", " << ei.me_last_pgno << ", " << st.ms_psize, LOG_LEVEL_0);
+        return;
+      }
 
-      double gigabyte = 1024 * 1024 * 1024;
+      static const double gigabyte = 1024 * 1024 * 1024;
       const uint64_t increment_size_pg_aligned = DB_RESIZE_INCREMENT_SIZE - (DB_RESIZE_INCREMENT_SIZE % st.ms_psize);
 
-      // need to resize DB
+      // calculate new size
       uint64_t new_size = ei.me_mapsize - (ei.me_mapsize % increment_size_pg_aligned) + increment_size_pg_aligned;
 
+      // perform a resize
       int res = mdb_env_set_mapsize(m_penv, new_size);
-      CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize");
+      CHECK_AND_ASSERT_MESS_LMDB_DB(res, ((void)(0)), "mdb_env_set_mapsize failed");
 
       LOG_PRINT_CYAN("[DB " << m_path << "] has grown: " << std::fixed << std::setprecision(2) << ei.me_mapsize / gigabyte << " GiB  ->  " << std::fixed << std::setprecision(2) << new_size / gigabyte << " GiB", LOG_LEVEL_0);
 
-      return true;
+      // slh2 dtor should free all threads waiting on m_resize_wait_cv
+      // slh dtor should set m_resize_thread_active to false
+    }
+
+    void lmdb_db_backend::resize_if_needed()
+    {
+      LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED in resize_if_needed()", LOG_LEVEL_3);
+      CRITICAL_REGION_LOCAL(m_write_exclusive_lock);
+
+      // m_write_exclusive_lock should be enough to safely call these:
+      MDB_stat st = AUTO_VAL_INIT(st);
+      mdb_env_stat(m_penv, &st);
+      MDB_envinfo ei = AUTO_VAL_INIT(ei);
+      mdb_env_info(m_penv, &ei);
+
+      if (!resize_condition(st, ei))
+        return; // resize is not needed
+
+      std::thread rt(&lmdb_db_backend::resize_thread, this);
+      rt.detach();
+
+      return;
     }
 
   }

src/common/db_backend_lmdb.h

@@ -42,6 +42,21 @@ namespace tools
       
       bool pop_tx_entry(tx_entry& txe);
 
+      void resize_thread();
+      bool resize_condition(const MDB_stat& st, const MDB_envinfo& ei);
+      void wait_for_resize_if_needed();
+      void notify_resize_thread_on_tx_end();
+
+      std::atomic<bool> m_resize_thread_active;
+      std::mutex m_resize_wait_mutex;
+      bool       m_resize_wait_condition;
+      std::condition_variable m_resize_wait_cv;
+
+      std::mutex m_resize_wait_txs_count_mutex;
+      std::condition_variable m_resize_wait_txs_cv;
+      size_t m_resize_wait_txs_count;
+
+
 
     public:
       lmdb_db_backend();
@@ -64,7 +79,7 @@ namespace tools
       //-------------------------------------------------------------------------------------
       bool have_tx();
       MDB_txn* get_current_tx();
-      bool resize_if_needed();
+      void resize_if_needed();
 
     };
   }

src/daemon/daemon.cpp

@@ -26,6 +26,7 @@ using namespace epee;
 #include "common/miniupnp_helper.h"
 #include "version.h"
 #include "currency_core/core_tools.h"
+#include "common/callstack_helper.h"
 
 #include <cstdlib>
 
@@ -108,6 +109,8 @@ int main(int argc, char* argv[])
     std::fflush(nullptr); // all open output streams are flushed
   });
 
+  epee::misc_utils::get_callstack(tools::get_callstack);
+
   po::options_description desc_cmd_only("Command line options");
   po::options_description desc_cmd_sett("Command line options and settings options", 130, 83);