From c25641b0e0a5dba222214e9855186f1a928f0e3b Mon Sep 17 00:00:00 2001 From: sowle Date: Sat, 16 Aug 2025 05:10:01 +0300 Subject: [PATCH 01/13] get_last_alt_x_block_cumulative_precise_difficulty() clarified --- src/currency_core/blockchain_storage.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index a6fb1142..f06f3a93 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -6852,9 +6852,8 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise return res; } //------------------------------------------------------------------ -wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos, wide_difficulty_type& cumulative_diff_precise_adj) const +wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_difficulty(const alt_chain_type& alt_chain, uint64_t main_chain_start_block_height, bool pos, wide_difficulty_type& cumulative_diff_precise_adj) const { - uint64_t main_chain_first_block = block_height; for (auto it = alt_chain.rbegin(); it != alt_chain.rend(); it++) { if (is_pos_block((*it)->second.bl) == pos) @@ -6862,14 +6861,14 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise cumulative_diff_precise_adj = (*it)->second.cumulative_diff_precise_adjusted; return (*it)->second.cumulative_diff_precise; } - main_chain_first_block = (*it)->second.height - 1; + main_chain_start_block_height = (*it)->second.height - 1; } CRITICAL_REGION_LOCAL(m_read_lock); - CHECK_AND_ASSERT_MES(main_chain_first_block < m_db_blocks.size(), false, "Intrnal error: main_chain_first_block(" << main_chain_first_block << ") < m_blocks.size() (" << m_db_blocks.size() << ")"); + CHECK_AND_ASSERT_MES(main_chain_start_block_height < m_db_blocks.size(), false, "Internal error: main_chain_start_block_height (" << main_chain_start_block_height << ") < m_blocks.size() (" << m_db_blocks.size() << ")"); - for (uint64_t i = main_chain_first_block; i != 0; i--) + for (uint64_t i = main_chain_start_block_height; i != 0; i--) { if (is_pos_block(m_db_blocks[i]->bl) == pos) { From 935feb66f519fa3552fd318ef869a8bea578f7e5 Mon Sep 17 00:00:00 2001 From: sowle Date: Sat, 16 Aug 2025 05:12:34 +0300 Subject: [PATCH 02/13] wallet: fixed/improved: set_tids_to_be_only_used_in_the_next_transfer(), get_transfer_info_by_index(), get_asset_info() --- src/wallet/wallet2.cpp | 8 +++++--- src/wallet/wallet2.h | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 42f7ac0c..b0716dd3 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -223,8 +223,10 @@ bool wallet2::get_transfer_info_by_key_image(const crypto::key_image& ki, transf //---------------------------------------------------------------------------------------------------- bool wallet2::get_transfer_info_by_index(size_t i, transfer_details& td) { - //WLT_CHECK_AND_ASSERT_MES(i < m_transfers.size(), false, "wrong out in transaction: internal index, m_transfers.size()=" << m_transfers.size()); - td = m_transfers.at(i); + auto it = m_transfers.find(i); + if (it == m_transfers.end()) + return false; + td = it->second; return true; } //---------------------------------------------------------------------------------------------------- @@ -3865,7 +3867,7 @@ bool wallet2::get_asset_info(const crypto::public_key& asset_id, currency::asset asset_flags |= aif_custom; } - if (ask_daemon_for_unknown) + if (asset_flags == aif_none && ask_daemon_for_unknown) { if (daemon_get_asset_info(asset_id, asset_info)) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index ef2c3a39..f1cfd471 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -588,7 +588,6 @@ namespace tools void set_tids_to_be_only_used_in_the_next_transfer(const std::vector& tids) { - WLT_THROW_IF_FALSE_WALLET_CMN_ERR_EX(std::all_of(tids.cbegin(), tids.cend(), [&](size_t i){ return i < m_transfers.size(); }), "some transfers IDs are out of range"); m_found_free_amounts.clear(); add_transfers_to_transfers_cache(tids); } From 5ff0399568853c8d3624d29e6691bce856003978 Mon Sep 17 00:00:00 2001 From: sowle Date: Sat, 16 Aug 2025 05:13:15 +0300 Subject: [PATCH 03/13] simplewallet: transfer_so fixed --- src/simplewallet/simplewallet.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 26c1a698..7320b703 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1840,7 +1840,14 @@ bool simple_wallet::transfer_so(const std::vector &args) std::stringstream ss; for (auto i : outs_idxs) + { ss << i << " "; + tools::transfer_details td{}; + if (!m_wallet->get_transfer_info_by_index(i, td) || !td.is_spendable()) + { + fail_msg_writer() << "output #" << i << ": wrong index or not spendable"; + } + } success_msg_writer() << "outputs' indicies allowed to spent: " << (outs_idxs.empty() ? std::string("all") : ss.str()); // 2nd arg: fee From 2884df9edc8eee93ef23b5378e652411b9956041 Mon Sep 17 00:00:00 2001 From: sowle Date: Sat, 16 Aug 2025 05:13:45 +0300 Subject: [PATCH 04/13] wallet: minor improvements --- src/simplewallet/simplewallet.cpp | 2 +- src/wallet/wallet2.cpp | 34 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 7320b703..b0539f24 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -2116,7 +2116,7 @@ bool simple_wallet::list_outputs(const std::vector &args) arg_spent_flags = true, include_spent = false; else if (!arg_spent_flags && (arg == "s" || arg == "spent" || arg == "unavailable")) arg_spent_flags = true, include_unspent = false; - else if (!arg_unknown_assets && (arg == "unknown")) + else if (!arg_unknown_assets && (arg == "unknown" || arg == "unk")) arg_unknown_assets = true, show_only_unknown = true; else if (!arg_ticker_filer && (arg.find("ticker=") == 0 || arg.find("t=") == 0)) arg_ticker_filer = true, filter_asset_ticker = boost::erase_all_copy(boost::erase_all_copy(arg, "ticker="), "t="); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b0716dd3..7fd80544 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -4014,7 +4014,7 @@ bool wallet2::generate_utxo_defragmentation_transaction_if_needed(currency::tran //---------------------------------------------------------------------------------------------------- std::string wallet2::get_transfers_str(bool include_spent /*= true*/, bool include_unspent /*= true*/, bool show_only_unknown /*= false*/, const std::string& filter_asset_ticker /*= std::string{}*/) const { - static const char* header = " index amount ticker g_index flags block tx out# asset id"; + static const char* header = " index amount ticker g_index flags block tx out# asset id"; std::stringstream ss; ss << header << ENDL; size_t count = 0; @@ -4045,7 +4045,7 @@ std::string wallet2::get_transfers_str(bool include_spent /*= true*/, bool inclu ss << std::right << (is_locked ? "*" : " ") << std::setw(5) << i << " " << std::setw(21) << print_asset_money(td.m_amount, adb.decimal_point) << " " << - std::setw(6) << std::left << (native_coin ? std::string(" ") : adb.ticker) << " " << std::right << + std::setw(14) << std::left << (native_coin ? std::string(" ") : adb.ticker) << " " << std::right << std::setw(7) << td.m_global_output_index << " " << std::setw(2) << std::setfill('0') << td.m_flags << std::setfill(' ') << ":" << std::setw(8) << transfer_flags_to_str(td.m_flags) << " " << @@ -4064,21 +4064,21 @@ std::string wallet2::get_transfers_str(bool include_spent /*= true*/, bool inclu ss << "printed " << count << " outputs of " << m_transfers.size() << " total" << ENDL; if (unknown_assets_outs_count == 1) - ss << "(" << unknown_assets_outs_count << " output with unrecognized asset id is not shown, use 'list_outputs unknown' to see it)" << ENDL; + ss << "(" << unknown_assets_outs_count << " output with unrecognized asset id is not shown, use 'list_outputs unknown' or 'lo unk' to see it)" << ENDL; else if (unknown_assets_outs_count > 1) - ss << "(" << unknown_assets_outs_count << " outputs with unrecognized asset ids are not shown, use 'list_outputs unknown' to see them)" << ENDL; + ss << "(" << unknown_assets_outs_count << " outputs with unrecognized asset ids are not shown, use 'list_outputs unknown' or 'lo unk' to see them)" << ENDL; return ss.str(); } //---------------------------------------------------------------------------------------------------- std::string wallet2::get_balance_str() const { - // balance unlocked / [balance total] ticker asset id - // 0.21 / 98.51 DP2 a6974d5874e97e5f4ed5ad0a62f0975edbccb1bb55502fc75c7fe808f12f44d3 - // 190.123456789012 / 199.123456789012 ZANO d6329b5b1f7c0805b5c345f4957554002a2f557845f64d7645dae0e051a6498a NATIVE - // 98.0 BGTVUW af2b12f3033337f9aea1845a6bc3fc966ed4d13227a3ace7706fca7dbcdaa7e2 - // 1000.034 DP3 d4aba1020f26927571771e04b585b4ffb211f52708d5e4c465bbdfa4a12e6271 + // balance unlocked / [balance total] ticker asset id + // 0.21 / 98.51 DP2 a6974d5874e97e5f4ed5ad0a62f0975edbccb1bb55502fc75c7fe808f12f44d3 + // 190.123456789012 / 199.123456789012 ZANO d6329b5b1f7c0805b5c345f4957554002a2f557845f64d7645dae0e051a6498a NATIVE + // 98.0 BGTVUW af2b12f3033337f9aea1845a6bc3fc966ed4d13227a3ace7706fca7dbcdaa7e2 + // 1000.034 DP3 d4aba1020f26927571771e04b585b4ffb211f52708d5e4c465bbdfa4a12e6271 - static const char* header = " balance unlocked / [balance total] ticker asset id"; + static const char* header = " balance unlocked / [balance total] ticker asset id"; std::stringstream ss; ss << header << ENDL; @@ -4100,7 +4100,7 @@ std::string wallet2::get_balance_str() const ss << std::string(21 + 3, ' '); else ss << " / " << std::setw(21) << print_fixed_decimal_point_with_trailing_spaces(b.total, b.asset_info.decimal_point); - ss << " " << std::setw(8) << std::left << b.asset_info.ticker << " " << b.asset_info.asset_id; + ss << " " << std::setw(14) << std::left << b.asset_info.ticker << " " << b.asset_info.asset_id; if (b.asset_info.asset_id == native_coin_asset_id) ss << " NATIVE"; ss << ENDL; @@ -4111,15 +4111,15 @@ std::string wallet2::get_balance_str() const //---------------------------------------------------------------------------------------------------- std::string wallet2::get_balance_str_raw() const { - // balance unlocked / [balance total] ticker asset id DP flags - // 0.21 / 98.51 ZANO d6329b5b1f7c0805b5c345f4957554002a2f557845f64d7645dae0e051a6498a 12 NATIVE - // 2.0 MYFB 13615ffdfbdc09275a1dfc0fbdaf6a9b07849b835ffdfed0b9e1478ea8924774 1 custom - // 1000.0 BurnCT 14608811180d4bbad96a6b91405e329e4f2a10519e6dcea644f83b9f8ccb5863 12 unknown asset + // balance unlocked / [balance total] ticker asset id DP flags + // 0.21 / 98.51 ZANO d6329b5b1f7c0805b5c345f4957554002a2f557845f64d7645dae0e051a6498a 12 NATIVE + // 2.0 MYFB 13615ffdfbdc09275a1dfc0fbdaf6a9b07849b835ffdfed0b9e1478ea8924774 1 custom + // 1000.0 BurnCT 14608811180d4bbad96a6b91405e329e4f2a10519e6dcea644f83b9f8ccb5863 12 unknown asset //WHITELIST: // a7e8e5b31c24f2d6a07e141701237b136d704c9a89f9a5d1ca4a8290df0b9edc WETH // ... - static const char* header = " balance unlocked / [balance total] ticker asset id DP flags"; + static const char* header = " balance unlocked / [balance total] ticker asset id DP flags"; std::stringstream ss; ss << header << ENDL; @@ -4140,7 +4140,7 @@ std::string wallet2::get_balance_str_raw() const else ss << " / " << std::setw(21) << print_fixed_decimal_point_with_trailing_spaces(entry.second.total, asset_info.decimal_point); - ss << " " << std::setw(8) << std::left << asset_info.ticker; + ss << " " << std::setw(14) << std::left << asset_info.ticker; ss << " " << entry.first << " "; if (has_info) From c3b7ac9691d15f3f1c41174f348a20145b143375 Mon Sep 17 00:00:00 2001 From: sowle Date: Mon, 18 Aug 2025 14:32:07 +0300 Subject: [PATCH 05/13] ui update (PR 159) --- src/gui/qt-daemon/layout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gui/qt-daemon/layout b/src/gui/qt-daemon/layout index bcd40213..aaac631c 160000 --- a/src/gui/qt-daemon/layout +++ b/src/gui/qt-daemon/layout @@ -1 +1 @@ -Subproject commit bcd4021364bbcd29a4208225de33a359ee35a447 +Subproject commit aaac631c18c890d89a761ef171d6dc2493ab5559 From a91b097f4e3ddbeede7e38e6446339b9f523d92a Mon Sep 17 00:00:00 2001 From: zano build machine Date: Mon, 18 Aug 2025 14:44:19 +0300 Subject: [PATCH 06/13] === build number: 428 -> 429 === --- src/version.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h.in b/src/version.h.in index 3112b961..205e245a 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -8,6 +8,6 @@ #define PROJECT_REVISION "8" #define PROJECT_VERSION PROJECT_MAJOR_VERSION "." PROJECT_MINOR_VERSION "." PROJECT_REVISION -#define PROJECT_VERSION_BUILD_NO 428 +#define PROJECT_VERSION_BUILD_NO 429 #define PROJECT_VERSION_BUILD_NO_STR STRINGIFY_EXPAND(PROJECT_VERSION_BUILD_NO) #define PROJECT_VERSION_LONG PROJECT_VERSION "." PROJECT_VERSION_BUILD_NO_STR "[" BUILD_COMMIT_ID "]" From db96e6638d4fee15eee6057983d722a1fed4b376 Mon Sep 17 00:00:00 2001 From: Dmitry Matsiukhov <46869283+dimmarvel@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:42:46 +0300 Subject: [PATCH 07/13] bulk writer (#561) * bulk writer * delete * refactor --- tests/db_tests/CMakeLists.txt | 3 +- tests/db_tests/db_size_test.cpp | 324 ++++++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 tests/db_tests/db_size_test.cpp diff --git a/tests/db_tests/CMakeLists.txt b/tests/db_tests/CMakeLists.txt index 50d0e9df..156270e7 100644 --- a/tests/db_tests/CMakeLists.txt +++ b/tests/db_tests/CMakeLists.txt @@ -1,3 +1,4 @@ add_executable(db_tests db_tests.cpp) +add_executable(db_size_test db_size_test.cpp) target_link_libraries(db_tests crypto common lmdb zlibstatic ${Boost_LIBRARIES}) - +target_link_libraries(db_size_test crypto common lmdb zlibstatic ${Boost_LIBRARIES}) diff --git a/tests/db_tests/db_size_test.cpp b/tests/db_tests/db_size_test.cpp new file mode 100644 index 00000000..5298851b --- /dev/null +++ b/tests/db_tests/db_size_test.cpp @@ -0,0 +1,324 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "epee/include/include_base_utils.h" +#include "include_base_utils.h" +#include "common/db_backend_lmdb.h" +#include "common/db_abstract_accessor.h" +#include "readwrite_lock.h" + +// #undef LOG_DEFAULT_CHANNEL +// #define LOG_DEFAULT_CHANNEL "db-bulk-writer" + +using namespace tools::db; + +enum class Mode { kv, array }; + +struct first_item_cb : public i_db_callback +{ + std::string k, v; + bool got = false; + bool on_enum_item(uint64_t /*i*/, const void* pkey, uint64_t ks, const void* pval, uint64_t vs) override + { + k.assign(static_cast(pkey), static_cast(ks)); + v.assign(static_cast(pval), static_cast(vs)); + got = true; + return false; // stop + } +}; + +static void gen_key_bytes(std::mt19937_64& gen, uint64_t counter, std::string& out, size_t size) +{ + out.resize(size); + for (size_t i = 0; i < size; i += 8) + { + uint64_t r = gen() ^ (0x9E3779B97F4A7C15ull * (counter + i)); + std::memcpy(&out[i], &r, std::min(8, size - i)); + } +} + +class bulk_writer +{ +public: + bulk_writer(basic_db_accessor& db, i_db_backend& _be, const std::string& table, container_handle h, Mode mode) + : _db(db), _be_(_be), _table(table), _h(h), _mode(mode) {} + + bool init_sample() + { + if (_mode == Mode::kv) + { + if (_be_.size(_h) == 0) + { + LOG_ERROR("[" << _table << "] empty"); + return false; + } + first_item_cb cb; + if (!_be_.enumerate(_h, &cb) || !cb.got) + { + LOG_ERROR("[" << _table << "] enumerate failed"); + return false; + } + _sample_key = cb.k; + _sample_val = cb.v; + _key_size = _sample_key.size(); + _bytes_per_put = _key_size + _sample_val.size(); + _entries = _be_.size(_h); + return true; + } + else + { // array + _entries = _be_.size(_h); + if (_entries == 0) + { + LOG_ERROR("[" << _table << "] empty"); + return false; + } + uint64_t last_key = _entries - 1; + if (!_be_.get(_h, reinterpret_cast(&last_key), sizeof(last_key), _sample_val)) + { + LOG_ERROR("[" << _table << "] get(last) failed"); + return false; + } + _key_size = sizeof(uint64_t); + _bytes_per_put = _key_size + _sample_val.size(); + return true; + } + } + + void print_initial_estimate() const + { + const long double MB_DEC = 1'000'000.0L; + long double approx_initial_bytes = static_cast(_entries) * _bytes_per_put; + LOG_PRINT_L0("[" << _table << "] entries=" << _entries + << ", sample_value=" << _sample_val.size() << " B" + << ", approx bytes/put=" << _bytes_per_put << " B"); + LOG_PRINT_L0("[" << _table << "] Approx initial size: " + << (approx_initial_bytes / MB_DEC) << " MB"); + } + + bool write(uint64_t target_mb, uint64_t step_mb) + { + using clock = std::chrono::steady_clock; + + const uint64_t MB_DEC = 1'000'000; + const uint64_t target_bytes = target_mb * MB_DEC; + const uint64_t step_bytes = step_mb * MB_DEC; + + if (!_db.begin_transaction(false)) + { + LOG_ERROR("begin_transaction failed"); + return false; + } + bool tx_open = true; + bool ok = true; + + std::random_device rd; + std::mt19937_64 gen(rd()); + + uint64_t written_total = 0; + uint64_t next_report = step_bytes; + uint64_t chunks_done = 0; + uint64_t counter = 0; + uint64_t array_key = _entries; + + const auto all_start = clock::now(); + auto chunk_start = all_start; + + while (written_total < target_bytes) + { + if (_mode == Mode::kv) + { + std::string new_key; + gen_key_bytes(gen, counter++, new_key, _key_size); + if (!_be_.set(_h, new_key.data(), new_key.size(), _sample_val.data(), _sample_val.size())) + { + LOG_ERROR("[" << _table << "] set(new kv) failed"); + ok = false; break; + } + } + else + { // array + if (!_be_.set(_h, reinterpret_cast(&array_key), sizeof(array_key), + _sample_val.data(), _sample_val.size())) + { + LOG_ERROR("[" << _table << "] set(new array) failed"); + ok = false; break; + } + ++array_key; + } + + written_total += _bytes_per_put; + + if (written_total >= next_report) + { + const auto t_fill_done = clock::now(); + + if (tx_open) + { + _db.commit_transaction(); + tx_open = false; + } + const auto t_after_commit = clock::now(); + + const double fill_sec = std::chrono::duration(t_fill_done - chunk_start).count(); + const double commit_sec = std::chrono::duration(t_after_commit - t_fill_done).count(); + const double chunk_sec = fill_sec + commit_sec; + const double total_sec = std::chrono::duration(t_after_commit - all_start).count(); + + ++chunks_done; + + const long double approx_now_mb = + (static_cast(_entries) * _bytes_per_put + + static_cast(chunks_done) * step_mb * MB_DEC) / 1'000'000.0L; + + const double chunk_mb = static_cast(step_mb); + const double fill_mb_s = chunk_mb / (fill_sec > 0 ? fill_sec : 1e-9); + const double commit_mb_s = chunk_mb / (commit_sec > 0 ? commit_sec : 1e-9); + const double chunk_mb_s = chunk_mb / (chunk_sec > 0 ? chunk_sec : 1e-9); + const double total_written = static_cast(chunks_done * step_mb); + const double total_mb_s = total_written / (total_sec > 0 ? total_sec : 1e-9); + + LOG_PRINT_L0( + "Writed " << step_mb << "MB in " << _table + << " | fill: " << fill_sec << " s (" << fill_mb_s << " MB/s)" + << ", commit: " << commit_sec << " s (" << commit_mb_s << " MB/s)" + << ", total: " << chunk_sec << " s (" << chunk_mb_s << " MB/s). " + << "Total written: " << total_written << " MB, total time: " + << total_sec << " s (" << total_mb_s << " MB/s avg). " + << "Est. table size now: " << approx_now_mb << " MB." + ); + + if (written_total < target_bytes) + { + if (!_db.begin_transaction(false)) + { + LOG_ERROR("begin_transaction failed (after commit)"); + ok = false; break; + } + tx_open = true; + chunk_start = clock::now(); + } + + next_report += step_bytes; + } + } + + if (tx_open) + { + if (ok) _db.commit_transaction(); + else _db.abort_transaction(); + } + return ok; + } + +private: + basic_db_accessor& _db; + i_db_backend& _be_; + std::string _table; + container_handle _h; + Mode _mode; + uint64_t _entries = 0; + size_t _key_size = 0; + uint64_t _bytes_per_put = 0; + std::string _sample_key; + std::string _sample_val; +}; + +static Mode parse_mode(const char* s) +{ + if (!s) + return Mode::kv; + std::string v(s); + std::transform(v.begin(), v.end(), v.begin(), ::tolower); + if (v == "array") + return Mode::array; + return Mode::kv; +} + +int main(int argc, char** argv) +{ + epee::string_tools::set_module_name_and_folder(argv[0]); + epee::log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); + epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, nullptr, nullptr, LOG_LEVEL_0); + epee::log_space::log_singletone::set_thread_log_prefix("[db-bulk-writer] "); + + if (argc < 3) + { + LOG_PRINT_L0( + std::string("Usage:\n ") + argv[0] + + " [--mode=kv|array] [--target-mb=N] [--step-mb=M]\n\n" + "Examples:\n " + argv[0] + + " ~/.Zano/blockchain_lmdb_v3 transactions --mode=kv --target-mb=1000 --step-mb=100\n " + + argv[0] + + " ~/.Zano/blockchain_lmdb_v3 blocks --mode=array --target-mb=200 --step-mb=50"); + return 0; + } + + + std::string env_path = argv[1]; + std::string table_name = argv[2]; + Mode mode = Mode::kv; + uint64_t target_mb = 1000; + uint64_t step_mb = 100; + + for (int i = 3; i < argc; ++i) + { + std::string a = argv[i]; + auto eat = [&](const char* pfx)->const char* + { + size_t L = std::strlen(pfx); + if (a.size() > L && a.compare(0,L,pfx) == 0) + return a.c_str()+L; + return nullptr; + }; + if (const char* v = eat("--mode=")) + mode = parse_mode(v); + else if (const char* v = eat("--target-mb=")) + target_mb = std::strtoull(v, nullptr, 10); + else if (const char* v = eat("--step-mb=")) + step_mb = std::strtoull(v, nullptr, 10); + } + + epee::shared_recursive_mutex rwlock; + std::shared_ptr be(new lmdb_db_backend()); + basic_db_accessor db(be, rwlock); + + uint64_t max_db_size = uint64_t(1UL * 512UL) * 1024UL * 1024UL * 1024UL; + if (!db.open(env_path, max_db_size)) + { + LOG_ERROR("Failed to open LMDB env at " << env_path); + return EXIT_FAILURE; + } + + container_handle h = 0; + if (!be->open_container(table_name, h)) + { + LOG_ERROR("open_container(" << table_name << ") failed"); + return EXIT_FAILURE; + } + + bulk_writer writer(db, *be, table_name, h, mode); + if (!writer.init_sample()) + return EXIT_FAILURE; + writer.print_initial_estimate(); + + bool ok = writer.write(target_mb, step_mb); + db.close(); + + if (ok) + { + LOG_PRINT_L0("DONE"); + return EXIT_SUCCESS; + } + else + { + LOG_ERROR("FAILED"); + return EXIT_FAILURE; + } +} From a9b981f30b223a2158bef05fc1677e8aa4357694 Mon Sep 17 00:00:00 2001 From: sowle Date: Fri, 22 Aug 2025 13:59:51 +0300 Subject: [PATCH 08/13] ui update (PR 160) --- src/gui/qt-daemon/layout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gui/qt-daemon/layout b/src/gui/qt-daemon/layout index aaac631c..1b8e3da0 160000 --- a/src/gui/qt-daemon/layout +++ b/src/gui/qt-daemon/layout @@ -1 +1 @@ -Subproject commit aaac631c18c890d89a761ef171d6dc2493ab5559 +Subproject commit 1b8e3da0d15c76f285f97c8fe29f6e7f6cab2fe6 From d92c4bfdd53f15bfd2010dba3328f82c630409aa Mon Sep 17 00:00:00 2001 From: zano build machine Date: Fri, 22 Aug 2025 14:39:40 +0300 Subject: [PATCH 09/13] === build number: 429 -> 430 === --- src/version.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h.in b/src/version.h.in index 205e245a..3de66e42 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -8,6 +8,6 @@ #define PROJECT_REVISION "8" #define PROJECT_VERSION PROJECT_MAJOR_VERSION "." PROJECT_MINOR_VERSION "." PROJECT_REVISION -#define PROJECT_VERSION_BUILD_NO 429 +#define PROJECT_VERSION_BUILD_NO 430 #define PROJECT_VERSION_BUILD_NO_STR STRINGIFY_EXPAND(PROJECT_VERSION_BUILD_NO) #define PROJECT_VERSION_LONG PROJECT_VERSION "." PROJECT_VERSION_BUILD_NO_STR "[" BUILD_COMMIT_ID "]" From 6a4fc0d34ff9734c0b3ece6a5912143db5eb407b Mon Sep 17 00:00:00 2001 From: Dmitry Matsiukhov <46869283+dimmarvel@users.noreply.github.com> Date: Mon, 25 Aug 2025 10:19:38 -0700 Subject: [PATCH 10/13] Add cert verification (#564) * Add cert verification * Refactor, stackoverflow windows solution * Finalize * Add more verifications --- contrib/epee/include/net/net_helper.h | 94 ++++++++++++++++++++++----- tests/performance_tests/main.cpp | 11 ++-- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index bb6bbcc2..f59f94a1 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -48,7 +48,6 @@ #include "misc_helpers.h" //#include "profile_tools.h" #include "../string_tools.h" - #ifndef MAKE_IP #define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) #endif @@ -58,6 +57,37 @@ namespace epee { namespace net_utils { + +#ifdef _WIN32 + // https://stackoverflow.com/questions/40307541 + #include + static void add_windows_root_certs(boost::asio::ssl::context& ctx) noexcept + { + HCERTSTORE hStore = CertOpenSystemStore(0, "ROOT"); + if (hStore == NULL) { + return; + } + + X509_STORE *store = X509_STORE_new(); + PCCERT_CONTEXT pContext = NULL; + while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) != NULL) { + // convert from DER to internal format + X509 *x509 = d2i_X509(NULL, + (const unsigned char **)&pContext->pbCertEncoded, + pContext->cbCertEncoded); + if(x509 != NULL) { + X509_STORE_add_cert(store, x509); + X509_free(x509); + } + } + + CertFreeCertificateContext(pContext); + CertCloseStore(hStore, 0); + + // attach X509_STORE to boost ssl context + SSL_CTX_set_cert_store(ctx.native_handle(), store); + } +#endif template struct socket_backend; @@ -70,11 +100,12 @@ namespace epee { // Create a context that uses the default paths for // finding CA certificates. +#ifdef _WIN32 + add_windows_root_certs(m_ssl_context); +#else m_ssl_context.set_default_verify_paths(); - /*m_socket.set_verify_mode(boost::asio::ssl::verify_peer); - m_socket.set_verify_callback( - boost::bind(&socket_backend::verify_certificate, this, _1, _2));*/ - +#endif + m_ssl_context.set_verify_mode(boost::asio::ssl::verify_peer); } /* @@ -101,7 +132,21 @@ namespace epee void set_domain(const std::string& domain_name) { - SSL_set_tlsext_host_name(m_socket.native_handle(), domain_name.c_str()); + SSL* ssl = m_socket.native_handle(); + + SSL_set_tlsext_host_name(ssl, domain_name.c_str()); +#if BOOST_VERSION >= 107300 + m_socket.set_verify_callback(boost::asio::ssl::host_name_verification(domain_name)); +#else + m_socket.set_verify_callback(boost::asio::ssl::rfc2818_verification(domain_name)); +#endif + + X509_VERIFY_PARAM* param = SSL_get0_param(ssl); + X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + if (X509_VERIFY_PARAM_set1_host(param, domain_name.c_str(), 0) != 1) + { + LOG_PRINT_L0("Failed to set expected hostname: " << domain_name); + } } boost::asio::ip::tcp::socket& get_socket() @@ -114,11 +159,27 @@ namespace epee return m_socket; } - void on_after_connect() + bool on_after_connect() { LOG_PRINT_L2("SSL Handshake...."); - m_socket.handshake(boost::asio::ssl::stream_base::client); + m_socket.set_verify_mode(boost::asio::ssl::verify_peer); + + boost::system::error_code ec; + m_socket.handshake(boost::asio::ssl::stream_base::client, ec); + + if (ec) + { + long vr = SSL_get_verify_result(m_socket.native_handle()); + LOG_PRINT_L0("TLS Handshake failed: " << ec.message() << " (verify: " << X509_verify_cert_error_string(vr) << ")"); + ERR_clear_error(); + boost::system::error_code ignored; + m_socket.lowest_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored); + m_socket.lowest_layer().close(ignored); + return false; + } + LOG_PRINT_L2("SSL Handshake OK"); + return true; } private: @@ -147,9 +208,9 @@ namespace epee return m_socket; } - void on_after_connect() + bool on_after_connect() { - + return true; } void reset() @@ -182,7 +243,7 @@ namespace epee return m_pbackend->get_stream(); } - void on_after_connect() + bool on_after_connect() { return m_pbackend->on_after_connect(); } @@ -321,13 +382,16 @@ namespace epee { m_io_service.run_one(); } - if (!ec && m_sct_back.get_socket().is_open()) { - m_sct_back.on_after_connect(); - m_connected = true; + if (!m_sct_back.on_after_connect()) + { + return false; + } + + m_connected = true; m_deadline.expires_at(boost::posix_time::pos_infin); - LOG_PRINT_L1("Connected OK: " << addr << ":" << port); + LOG_PRINT_L1("TLS connected OK: " << addr << ":" << port); return true; } else diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index 521ec083..023c6025 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -74,7 +74,8 @@ void test_plain_wallet() { //std::string res = plain_wallet::init("195.201.107.230", "33340", "C:\\Users\\roky\\home\\", 0); //std::string res = plain_wallet::init("", "", "C:\\Users\\roky\\home\\", 0); - std::string res = plain_wallet::init("https://node.zano.org", "443", "C:\\Users\\roky\\home\\", LOG_LEVEL_2); + // std::string res = plain_wallet::init("https://195.201.107.230", "443", "C:\\git_repos\\zano\\build_msvc2022_64\\src\\Debug\\", LOG_LEVEL_2); + std::string res = plain_wallet::init("https://node.zano.org", "443", "C:\\git_repos\\zano\\build_msvc2022_64\\src\\Debug\\", LOG_LEVEL_2); //std::string res = plain_wallet::init("127.0.0.1", "12111", "C:\\Users\\roky\\home22\\", 0); plain_wallet::configure_object conf = AUTO_VAL_INIT(conf); @@ -228,16 +229,16 @@ void multithread_test_of_get_coinbase_hash_cached() int main(int argc, char** argv) { epee::string_tools::set_module_name_and_folder(argv[0]); - epee::log_space::get_set_log_detalisation_level(true, LOG_LEVEL_2); - epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL, LOG_LEVEL_2); + epee::log_space::get_set_log_detalisation_level(true, LOG_LEVEL_3); + epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL, LOG_LEVEL_3); //epee::log_space::log_singletone::add_logger(LOGGER_FILE, // epee::log_space::log_singletone::get_default_log_file().c_str(), // epee::log_space::log_singletone::get_default_log_folder().c_str()); - multithread_test_of_get_coinbase_hash_cached(); + //multithread_test_of_get_coinbase_hash_cached(); //test_tx_json_serialization(); //test_base64_serialization(); - //test_plain_wallet(); + test_plain_wallet(); //parse_weird_tx(); //thread_pool_tests(); From 8e237a054ce5a573beb166bc39e44f85f215abcb Mon Sep 17 00:00:00 2001 From: sowle Date: Tue, 26 Aug 2025 21:51:49 +0300 Subject: [PATCH 11/13] tests: PCH fixed for db_size_test --- tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fb38a979..bfa36b71 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -67,4 +67,5 @@ if(MSVC AND USE_PCH) set_property(SOURCE "core_tests/chaingen_pch.cpp" APPEND_STRING PROPERTY COMPILE_FLAGS " /Ycchaingen.h /Zm1000") set_property(TARGET coretests functional_tests hash-target-tests performance_tests unit_tests APPEND_STRING PROPERTY LINK_FLAGS "$(MSBuildProjectDirectory)/../src/$(ConfigurationName)/stdafx.obj") set_property(TARGET db_tests APPEND_STRING PROPERTY LINK_FLAGS "$(MSBuildProjectDirectory)/../../src/$(ConfigurationName)/stdafx.obj") + set_property(TARGET db_size_test APPEND_STRING PROPERTY LINK_FLAGS "$(MSBuildProjectDirectory)/../../src/$(ConfigurationName)/stdafx.obj") endif() From fa214892377914f69f57aa1f11c1a66f24116ebf Mon Sep 17 00:00:00 2001 From: sowle Date: Wed, 27 Aug 2025 15:32:45 +0300 Subject: [PATCH 12/13] wallet rpc: tx_comment enabled back --- src/wallet/wallet_rpc_server.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 45cbcdea..5c375b10 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -529,10 +529,9 @@ namespace tools if (!req.comment.empty() && payment_id.empty()) { - // tx_comment is temporary disabled -- sowle - //currency::tx_comment comment = AUTO_VAL_INIT(comment); - //comment.comment = req.comment; - //extra.push_back(comment); + currency::tx_comment{}; + comment.comment = req.comment; + extra.push_back(comment); } if (req.push_payer && !wrap) From 0de1633f835a2c1d78b350d16518b3e87ec5d62a Mon Sep 17 00:00:00 2001 From: sowle Date: Wed, 27 Aug 2025 15:36:45 +0300 Subject: [PATCH 13/13] === version bump: 2.1.8.430 -> 2.1.9.431 === --- src/version.h.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/version.h.in b/src/version.h.in index 3de66e42..daa6f55a 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -5,9 +5,9 @@ #define PROJECT_MAJOR_VERSION "2" #define PROJECT_MINOR_VERSION "1" -#define PROJECT_REVISION "8" +#define PROJECT_REVISION "9" #define PROJECT_VERSION PROJECT_MAJOR_VERSION "." PROJECT_MINOR_VERSION "." PROJECT_REVISION -#define PROJECT_VERSION_BUILD_NO 430 +#define PROJECT_VERSION_BUILD_NO 431 #define PROJECT_VERSION_BUILD_NO_STR STRINGIFY_EXPAND(PROJECT_VERSION_BUILD_NO) #define PROJECT_VERSION_LONG PROJECT_VERSION "." PROJECT_VERSION_BUILD_NO_STR "[" BUILD_COMMIT_ID "]"