diff --git a/CMakeLists.txt b/CMakeLists.txt index f710955e..4a38a381 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android") add_definitions(-DMOBILE_WALLET_BUILD) if(CMAKE_SYSTEM_NAME STREQUAL "iOS" ) add_definitions(-DIOS_BUILD) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fembed-bitcode") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fembed-bitcode") endif() if(CMAKE_SYSTEM_NAME STREQUAL "Android") add_definitions(-DANDROID_BUILD) @@ -187,14 +189,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "iOS") set(Boost_LIBRARIES "libboost.a") set(Boost_VERSION "ofxiOSBoost 1.60.0") elseif(CMAKE_SYSTEM_NAME STREQUAL "Android") - #set(Boost_INCLUDE_DIRS "/Users/roky/projects/Zano/mobile_repo/Boost-for-Android-Prebuilt/boost_1_68_0-clang/include") - #set(Boost_LIBRARY_DIRS "/Users/roky/projects/Zano/mobile_repo/Boost-for-Android-Prebuilt/boost_1_68_0-clang/${CMAKE_ANDROID_ARCH_ABI}/lib/") set(Boost_INCLUDE_DIRS "/Users/roky/projects/Zano/mobile_repo/Boost-for-Android-Prebuilt/1.69.0/include") set(Boost_LIBRARY_DIRS "/Users/roky/projects/Zano/mobile_repo/Boost-for-Android-Prebuilt/1.69.0/libs/llvm/${CMAKE_ANDROID_ARCH_ABI}/") - #link_directories("${Boost_LIBRARY_DIRS}") set(Boost_LIBRARIES "${Boost_LIBRARY_DIRS}libboost_system.a;${Boost_LIBRARY_DIRS}libboost_filesystem.a;${Boost_LIBRARY_DIRS}libboost_thread.a;${Boost_LIBRARY_DIRS}libboost_timer.a;${Boost_LIBRARY_DIRS}libboost_date_time.a;${Boost_LIBRARY_DIRS}libboost_chrono.a;${Boost_LIBRARY_DIRS}libboost_regex.a;${Boost_LIBRARY_DIRS}libboost_serialization.a;${Boost_LIBRARY_DIRS}libboost_atomic.a;${Boost_LIBRARY_DIRS}libboost_program_options.a") - # set(Boost_LIBRARIES "libboost_system_w.a libboost_filesystem.a libboost_thread.a libboost_timer.a libboost_date_time.a libboost_chrono.a libboost_regex.a libboost_serialization.a libboost_atomic.a libboost_program_options.a libboost_locale.a") - set(Boost_VERSION "PurpleI2PBoost 1.68.0") + set(Boost_VERSION "PurpleI2PBoost 1.69.0") else() find_package(Boost 1.55 REQUIRED COMPONENTS system filesystem thread timer date_time chrono regex serialization atomic program_options locale) endif() diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index b79db5b5..3e850493 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -51,7 +51,7 @@ #include #endif -#include "include_base_utils.h" +//#include "include_base_utils.h" #include "string_coding.h" namespace epee diff --git a/contrib/epee/include/gzip_encoding.h b/contrib/epee/include/gzip_encoding.h index 2be51e77..ecaff2df 100644 --- a/contrib/epee/include/gzip_encoding.h +++ b/contrib/epee/include/gzip_encoding.h @@ -29,9 +29,9 @@ #ifndef _GZIP_ENCODING_H_ #define _GZIP_ENCODING_H_ +#include "boost/core/ignore_unused.hpp" #include "net/http_client_base.h" #include "zlib/zlib.h" -//#include "http.h" namespace epee @@ -41,186 +41,325 @@ namespace net_utils - class content_encoding_gzip: public i_sub_handler - { - public: - /*! \brief - * Function content_encoding_gzip : Constructor - * - */ - inline - content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false):m_powner_filter(powner_filter), - m_is_stream_ended(false), - m_is_deflate_mode(is_deflate_mode), - m_is_first_update_in(true) - { - memset(&m_zstream_in, 0, sizeof(m_zstream_in)); - memset(&m_zstream_out, 0, sizeof(m_zstream_out)); - int ret = 0; - if(is_deflate_mode) - { - ret = inflateInit(&m_zstream_in); - ret = deflateInit(&m_zstream_out, Z_DEFAULT_COMPRESSION); - }else - { - ret = inflateInit2(&m_zstream_in, 0x1F); - ret = deflateInit2(&m_zstream_out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY); - } - } - /*! \brief - * Function content_encoding_gzip : Destructor - * - */ - inline - ~content_encoding_gzip() - { - inflateEnd(& m_zstream_in ); - deflateEnd(& m_zstream_out ); - } - /*! \brief - * Function update_in : Entry point for income data - * - */ - inline - virtual bool update_in( std::string& piece_of_transfer) - { + class content_encoding_gzip: public i_sub_handler + { + public: + /*! \brief + * Function content_encoding_gzip : Constructor + * + */ + inline + content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false, int compression_level = Z_DEFAULT_COMPRESSION) :m_powner_filter(powner_filter), + m_is_stream_ended(false), + m_is_deflate_mode(is_deflate_mode), + m_is_first_update_in(true) + { + memset(&m_zstream_in, 0, sizeof(m_zstream_in)); + memset(&m_zstream_out, 0, sizeof(m_zstream_out)); + int ret = 0; + boost::ignore_unused(ret); + if(is_deflate_mode) + { + ret = inflateInit(&m_zstream_in); + ret = deflateInit(&m_zstream_out, compression_level); + }else + { + ret = inflateInit2(&m_zstream_in, 0x1F); + ret = deflateInit2(&m_zstream_out, compression_level, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY); + } + } + /*! \brief + * Function content_encoding_gzip : Destructor + * + */ + inline + ~content_encoding_gzip() + { + inflateEnd(& m_zstream_in ); + deflateEnd(& m_zstream_out ); + } + /*! \brief + * Function update_in : Entry point for income data + * + */ + inline + virtual bool update_in( std::string& piece_of_transfer) + { - bool is_first_time_here = m_is_first_update_in; - m_is_first_update_in = false; + bool is_first_time_here = m_is_first_update_in; + m_is_first_update_in = false; - if(m_pre_decode.size()) - m_pre_decode += piece_of_transfer; - else - m_pre_decode.swap(piece_of_transfer); - piece_of_transfer.clear(); + if(m_pre_decode.size()) + m_pre_decode += piece_of_transfer; + else + m_pre_decode.swap(piece_of_transfer); + piece_of_transfer.clear(); - std::string decode_summary_buff; + std::string decode_summary_buff; - size_t ungzip_size = m_pre_decode.size() * 0x30; - std::string current_decode_buff(ungzip_size, 'X'); + size_t ungzip_size = m_pre_decode.size() * 0x30; + std::string current_decode_buff(ungzip_size, 'X'); - //Here the cycle is introduced where we unpack the buffer, the cycle is required - //because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error - bool continue_unpacking = true; - bool first_step = true; - while(m_pre_decode.size() && continue_unpacking) - { + //Here the cycle is introduced where we unpack the buffer, the cycle is required + //because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error + bool continue_unpacking = true; + bool first_step = true; + while(m_pre_decode.size() && continue_unpacking) + { - //fill buffers - m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); - m_zstream_in.avail_in = (uInt)m_pre_decode.size(); - m_zstream_in.next_out = (Bytef*)current_decode_buff.data(); - m_zstream_in.avail_out = (uInt)ungzip_size; + //fill buffers + m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); + m_zstream_in.avail_in = (uInt)m_pre_decode.size(); + m_zstream_in.next_out = (Bytef*)current_decode_buff.data(); + m_zstream_in.avail_out = (uInt)ungzip_size; - int flag = Z_SYNC_FLUSH; - int ret = inflate(&m_zstream_in, flag); - CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. err = " << ret); + int flag = Z_NO_FLUSH; + int ret = inflate(&m_zstream_in, flag); + CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. ret = " << ret << ", msg: " << (m_zstream_in.msg ? m_zstream_in.msg : "")); - if(Z_STREAM_END == ret) - m_is_stream_ended = true; - else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step) - { - // some servers (notably Apache with mod_deflate) don't generate zlib headers - // insert a dummy header and try again - static char dummy_head[2] = - { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, - }; - inflateReset(&m_zstream_in); - m_zstream_in.next_in = (Bytef*) dummy_head; - m_zstream_in.avail_in = sizeof(dummy_head); + if(Z_STREAM_END == ret) + m_is_stream_ended = true; + else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step) + { + // some servers (notably Apache with mod_deflate) don't generate zlib headers + // insert a dummy header and try again + static char dummy_head[2] = + { + 0x8 + 0x7 * 0x10, + (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, + }; + inflateReset(&m_zstream_in); + m_zstream_in.next_in = (Bytef*) dummy_head; + m_zstream_in.avail_in = sizeof(dummy_head); - ret = inflate(&m_zstream_in, Z_NO_FLUSH); - if (ret != Z_OK) - { - LOCAL_ASSERT(0); - m_pre_decode.swap(piece_of_transfer); - return false; - } - m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); - m_zstream_in.avail_in = (uInt)m_pre_decode.size(); + ret = inflate(&m_zstream_in, Z_NO_FLUSH); + if (ret != Z_OK) + { + LOCAL_ASSERT(0); + m_pre_decode.swap(piece_of_transfer); + return false; + } + m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); + m_zstream_in.avail_in = (uInt)m_pre_decode.size(); - ret = inflate(&m_zstream_in, Z_NO_FLUSH); - if (ret != Z_OK) - { - LOCAL_ASSERT(0); - m_pre_decode.swap(piece_of_transfer); - return false; - } - } + ret = inflate(&m_zstream_in, Z_NO_FLUSH); + if (ret != Z_OK) + { + LOCAL_ASSERT(0); + m_pre_decode.swap(piece_of_transfer); + return false; + } + } + else + { + CHECK_AND_ASSERT_MES(Z_DATA_ERROR != ret, false, "content_encoding_gzip::update_in() Failed to inflate. err = Z_DATA_ERROR"); + } - //leave only unpacked part in the output buffer to start with it the next time - m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in); - //if decoder gave nothing to return, then everything is ahead, now simply break - if(ungzip_size == m_zstream_in.avail_out) - break; + //leave only unpacked part in the output buffer to start with it the next time + m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in); + //if decoder gave nothing to return, then everything is ahead, now simply break + if(ungzip_size == m_zstream_in.avail_out) + break; - //decode_buff currently stores data parts that were unpacked, fix this size - current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out); - if(decode_summary_buff.size()) - decode_summary_buff += current_decode_buff; - else - current_decode_buff.swap(decode_summary_buff); + //decode_buff currently stores data parts that were unpacked, fix this size + current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out); + if(decode_summary_buff.size()) + decode_summary_buff += current_decode_buff; + else + current_decode_buff.swap(decode_summary_buff); - current_decode_buff.resize(ungzip_size); - first_step = false; - } + current_decode_buff.resize(ungzip_size); + first_step = false; + } - //Process these data if required - bool res = true; + //Process these data if required + return m_powner_filter->handle_target_data(decode_summary_buff); + } + /*! \brief + * Function stop : Entry point for stop signal and flushing cached data buffer. + * + */ + inline + virtual void stop(std::string& OUT collect_remains) + { + } + protected: + private: + /*! \brief + * Pointer to parent HTTP-parser + */ + i_target_handler* m_powner_filter; + /*! \brief + * ZLIB object for income stream + */ + z_stream m_zstream_in; + /*! \brief + * ZLIB object for outcome stream + */ + z_stream m_zstream_out; + /*! \brief + * Data that could not be unpacked immediately, left to wait for the next packet of data + */ + std::string m_pre_decode; + /*! \brief + * The data are accumulated for a package in the buffer to send the web client + */ + std::string m_pre_encode; + /*! \brief + * Signals that stream looks like ended + */ + bool m_is_stream_ended; + /*! \brief + * If this flag is set, income data is in HTTP-deflate mode + */ + bool m_is_deflate_mode; + /*! \brief + * Marks that it is a first data packet + */ + bool m_is_first_update_in; + }; // class content_encoding_gzip - res = m_powner_filter->handle_target_data(decode_summary_buff); + struct abstract_callback_base + { + virtual bool do_call(const std::string& piece_of_transfer) = 0; + }; - return true; + template + struct abstract_callback : public abstract_callback_base + { + callback_t m_cb; - } - /*! \brief - * Function stop : Entry point for stop signal and flushing cached data buffer. - * - */ - inline - virtual void stop(std::string& OUT collect_remains) - { - } - protected: - private: - /*! \brief - * Pointer to parent HTTP-parser - */ - i_target_handler* m_powner_filter; - /*! \brief - * ZLIB object for income stream - */ - z_stream m_zstream_in; - /*! \brief - * ZLIB object for outcome stream - */ - z_stream m_zstream_out; - /*! \brief - * Data that could not be unpacked immediately, left to wait for the next packet of data - */ - std::string m_pre_decode; - /*! \brief - * The data are accumulated for a package in the buffer to send the web client - */ - std::string m_pre_encode; - /*! \brief - * Signals that stream looks like ended - */ - bool m_is_stream_ended; - /*! \brief - * If this flag is set, income data is in HTTP-deflate mode - */ - bool m_is_deflate_mode; - /*! \brief - * Marks that it is a first data packet - */ - bool m_is_first_update_in; - }; -} -} + abstract_callback(callback_t cb) : m_cb(cb){} + virtual bool do_call(const std::string& piece_of_transfer) + { + return m_cb(piece_of_transfer); + } + }; + + class gzip_decoder_lambda : public content_encoding_gzip, + public i_target_handler + { + std::shared_ptr m_pcb; + + virtual bool handle_target_data(std::string& piece_of_transfer) + { + bool r = m_pcb->do_call(piece_of_transfer); + piece_of_transfer.clear(); + return r; + } + public: + gzip_decoder_lambda() : content_encoding_gzip(this, true, Z_BEST_COMPRESSION) + {} + + template + bool update_in(std::string& piece_of_transfer, callback_t cb) + { + m_pcb.reset(new abstract_callback(cb)); + return content_encoding_gzip::update_in(piece_of_transfer); + } + template + bool stop(callback_t cb) + {return true;} + }; // class gzip_decoder_lambda + + class gzip_encoder_lyambda + { + bool m_initialized; + z_stream m_zstream; + public: + gzip_encoder_lyambda(int compression_level = Z_DEFAULT_COMPRESSION) :m_initialized(false), m_zstream(AUTO_VAL_INIT(m_zstream)) + { + int ret = deflateInit(&m_zstream, compression_level); + if (ret == Z_OK) + m_initialized = true; + } + + ~gzip_encoder_lyambda() + { + deflateEnd(&m_zstream); + } + + template + bool update_in(const std::string& target, callback_t cb) + { + if (!m_initialized) + { + return false; + } + + if (!target.size()) + { + return true; + } + + std::string result_packed_buff; + //theoretically it supposed to be smaller + result_packed_buff.resize(target.size(), 'X'); + while (true) + { + m_zstream.next_in = (Bytef*)target.data(); + m_zstream.avail_in = (uInt)target.size(); + m_zstream.next_out = (Bytef*)result_packed_buff.data(); + m_zstream.avail_out = (uInt)result_packed_buff.size(); + + int ret = deflate(&m_zstream, Z_NO_FLUSH); + CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate. err = " << ret); + if (m_zstream.avail_out == 0) + { + //twice bigger buffer + result_packed_buff.resize(result_packed_buff.size()*2); + continue; + } + CHECK_AND_ASSERT_MES(result_packed_buff.size() >= m_zstream.avail_out, false, "result_packed_buff.size()=" << result_packed_buff.size() << " >= m_zstream.avail_out=" << m_zstream.avail_out); + result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out); + break; + } + + return cb(result_packed_buff); + } + + template + bool stop(callback_t cb) + { + if (!m_initialized) + { + return false; + } + + std::string result_packed_buff; + //theoretically it supposed to be smaller + result_packed_buff.resize(1000000, 'X'); + while (true) + { + m_zstream.next_in = nullptr; + m_zstream.avail_in = 0; + m_zstream.next_out = (Bytef*)result_packed_buff.data(); + m_zstream.avail_out = (uInt)result_packed_buff.size(); + + int ret = deflate(&m_zstream, Z_FINISH); + CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate at finish. err = " << ret); + if (ret != Z_STREAM_END) + { + //twice bigger buffer + result_packed_buff.resize(result_packed_buff.size() * 2); + continue; + } + + CHECK_AND_ASSERT_MES(result_packed_buff.size() >= m_zstream.avail_out, false, "result_packed_buff.size()=" << result_packed_buff.size() << " >= m_zstream.avail_out=" << m_zstream.avail_out); + result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out); + + m_initialized = false; + break; + } + + return cb(result_packed_buff); + } + + }; // class gzip_encoder_lyambda + +} // namespace net_utils +} // namespace epee diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 6cb4f406..782ef241 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -69,7 +69,7 @@ DISABLE_VS_WARNINGS(4100) #include "syncobj.h" #include "sync_locked_object.h" #include "string_coding.h" - +#include "file_io_utils.h" #define LOG_LEVEL_SILENT -1 #define LOG_LEVEL_0 0 @@ -290,6 +290,8 @@ namespace log_space virtual bool set_max_logfile_size(uint64_t max_size){return true;}; virtual bool set_log_rotate_cmd(const std::string& cmd){return true;}; + virtual bool truncate_log_files() { return true; } + virtual std::string copy_logs_to_buffer() { return ""; } }; /************************************************************************/ @@ -629,7 +631,7 @@ namespace log_space class file_output_stream : public ibase_log_stream { public: - typedef std::map named_log_streams; + typedef std::map > named_log_streams; file_output_stream( const std::string& default_log_file_name, const std::string& log_path ) { @@ -643,12 +645,12 @@ namespace log_space { for(named_log_streams::iterator it = m_log_file_names.begin(); it!=m_log_file_names.end(); it++) { - if ( it->second->is_open() ) + if ( it->second.first->is_open() ) { - it->second->flush(); - it->second->close(); + it->second.first->flush(); + it->second.first->close(); } - delete it->second; + delete it->second.first; } } private: @@ -666,12 +668,14 @@ namespace log_space //log_space::rotate_log_file((m_default_log_path + "\\" + pstream_name).c_str()); boost::system::error_code ec; boost::filesystem::create_directories(m_default_log_path_w, ec); - boost::filesystem::ofstream* pstream = (m_log_file_names[pstream_name] = new boost::filesystem::ofstream); + boost::filesystem::ofstream* pstream = new boost::filesystem::ofstream; + std::wstring target_path = m_default_log_path_w + L"/" + epee::string_encoding::utf8_to_wstring(pstream_name); pstream->open( target_path.c_str(), std::ios_base::out | std::ios::app /*ios_base::trunc */); if(pstream->fail()) return NULL; + m_log_file_names[pstream_name] = std::pair(pstream, target_path); return pstream; } @@ -687,6 +691,51 @@ namespace log_space return true; } + bool truncate_log_files() + { + for (named_log_streams::iterator it = m_log_file_names.begin(); it != m_log_file_names.end(); it++) + { + std::wstring target_path = it->second.second; + //close and delete current stream + if (it->second.first->is_open()) + { + it->second.first->flush(); + it->second.first->close(); + } + delete it->second.first; + it->second.first = nullptr; + //reopen it with truncate + boost::filesystem::ofstream* pstream = new boost::filesystem::ofstream; + pstream->open(target_path.c_str(), std::ios_base::out | std::ios::trunc ); + if (pstream->fail()) + { + throw std::runtime_error("Unexpected error: failed to re-open log stream on truncate"); + } + it->second.first = pstream; + } + return true; + } + + std::string copy_logs_to_buffer() + { + std::stringstream res; + + for (named_log_streams::iterator it = m_log_file_names.begin(); it != m_log_file_names.end(); it++) + { + std::wstring target_path = it->second.second; + res << "[" << epee::string_encoding::convert_to_ansii(target_path) << "]" << ENDL; + std::string res_buf; + if (!epee::file_io_utils::load_file_to_string(target_path, res_buf)) + { + res << "ERROR"; + } + else + { + res << res_buf; + } + } + return res.str(); + } virtual bool out_buffer( const char* buffer, int buffer_len, int log_level, int color, const char* plog_name = NULL ) @@ -698,7 +747,7 @@ namespace log_space if(it == m_log_file_names.end()) m_target_file_stream = add_new_stream_and_open(plog_name); else - m_target_file_stream = it->second; + m_target_file_stream = it->second.first; } if(!m_target_file_stream || !m_target_file_stream->is_open()) return false;//TODO: add assert here @@ -796,6 +845,22 @@ namespace log_space return true; } + bool truncate_log_files() + { + for (streams_container::iterator it = m_log_streams.begin(); it != m_log_streams.end(); it++) + it->first->truncate_log_files(); + return true; + } + + std::string copy_logs_to_buffer() + { + std::string res; + for (streams_container::iterator it = m_log_streams.begin(); it != m_log_streams.end(); it++) + res += it->first->copy_logs_to_buffer(); + return res; + } + + bool do_log_message(const std::string& rlog_mes, int log_level, int color, const char* plog_name = NULL) { std::string str_mess = rlog_mes; @@ -967,6 +1032,21 @@ namespace log_space return true; } + std::string copy_logs_to_buffer() + { + FAST_CRITICAL_REGION_BEGIN(m_critical_sec); + return m_log_target.copy_logs_to_buffer(); + FAST_CRITICAL_REGION_END(); + } + + + bool truncate_log_files() + { + FAST_CRITICAL_REGION_BEGIN(m_critical_sec); + return m_log_target.truncate_log_files(); + FAST_CRITICAL_REGION_END(); + } + bool take_away_journal(std::list& journal) { FAST_CRITICAL_REGION_BEGIN(m_critical_sec); @@ -1254,6 +1334,23 @@ namespace log_space return plogger->set_log_rotate_cmd(cmd); } + + static std::string copy_logs_to_buffer() + { + logger* plogger = get_or_create_instance(); + if (!plogger) return ""; + return plogger->copy_logs_to_buffer(); + } + + + static bool truncate_log_files() + { + logger* plogger = get_or_create_instance(); + if (!plogger) return false; + return plogger->truncate_log_files(); + } + + static bool add_logger( int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit = LOG_LEVEL_4) { diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 28e6a30b..001d7ceb 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -38,9 +38,9 @@ #include "net_helper.h" #include "http_client_base.h" -#ifdef HTTP_ENABLE_GZIP +//#ifdef HTTP_ENABLE_GZIP #include "gzip_encoding.h" -#endif +//#endif #include "string_tools.h" #include "reg_exp_definer.h" @@ -230,11 +230,11 @@ using namespace std; blocked_mode_client m_net_client; std::string m_host_buff; std::string m_port; - unsigned int m_timeout; + //unsigned int m_timeout; + unsigned int m_connection_timeout; + unsigned int m_recv_timeout; std::string m_header_cache; http_response_info m_response_info; - size_t m_len_in_summary; - size_t m_len_in_remain; //std::string* m_ptarget_buffer; boost::shared_ptr m_pcontent_encoding_handler; reciev_machine_state m_state; @@ -242,6 +242,10 @@ using namespace std; std::string m_chunked_cache; critical_section m_lock; + protected: + uint64_t m_len_in_summary; + uint64_t m_len_in_remain; + public: void set_host_name(const std::string& name) { @@ -259,14 +263,27 @@ using namespace std; { return connect(host, std::to_string(port), timeout); } - bool connect(const std::string& host, const std::string& port, unsigned int timeout) + + bool set_timeouts(unsigned int connection_timeout, unsigned int recv_timeout) + { + m_connection_timeout = connection_timeout; + m_recv_timeout = recv_timeout; + return true; + } + + bool connect(const std::string& host, std::string port) { CRITICAL_REGION_LOCAL(m_lock); m_host_buff = host; m_port = port; - m_timeout = timeout; - return m_net_client.connect(host, port, timeout, timeout); + return m_net_client.connect(host, port, m_connection_timeout, m_recv_timeout); + } + + bool connect(const std::string& host, const std::string& port, unsigned int timeout) + { + m_connection_timeout = m_recv_timeout = timeout; + return connect(host, port); } //--------------------------------------------------------------------------- bool disconnect() @@ -303,7 +320,7 @@ using namespace std; if(!is_connected()) { LOG_PRINT("Reconnecting...", LOG_LEVEL_3); - if(!connect(m_host_buff, m_port, m_timeout)) + if(!connect(m_host_buff, m_port)) { LOG_PRINT("Failed to connect to " << m_host_buff << ":" << m_port, LOG_LEVEL_3); return false; @@ -447,7 +464,14 @@ using namespace std; } CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()"); m_len_in_remain -= recv_buff.size(); - m_pcontent_encoding_handler->update_in(recv_buff); + bool r = m_pcontent_encoding_handler->update_in(recv_buff); + //CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_pcontent_encoding_handler->update_in returned false"); + if (!r) + { + m_state = reciev_machine_state_error; + disconnect(); + return false; + } if(m_len_in_remain == 0) m_state = reciev_machine_state_done; @@ -483,7 +507,7 @@ using namespace std; } //--------------------------------------------------------------------------- inline - bool get_len_from_chunk_head(const std::string &chunk_head, size_t& result_size) + bool get_len_from_chunk_head(const std::string &chunk_head, uint64_t& result_size) { std::stringstream str_stream; str_stream << std::hex; @@ -494,7 +518,7 @@ using namespace std; } //--------------------------------------------------------------------------- inline - bool get_chunk_head(std::string& buff, size_t& chunk_size, bool& is_matched) + bool get_chunk_head(std::string& buff, uint64_t& chunk_size, bool& is_matched) { is_matched = false; size_t offset = 0; @@ -850,13 +874,13 @@ using namespace std; return true; } }; + // class http_simple_client /************************************************************************/ /* */ /************************************************************************/ - //inline template bool invoke_request(const std::string& url, t_transport& tr, unsigned int timeout, const http_response_info** ppresponse_info, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list()) { @@ -880,6 +904,128 @@ using namespace std; return tr.invoke(u_c.uri, method, body, ppresponse_info, additional_params); } - } -} -} + struct idle_handler_base + { + virtual bool do_call(const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) = 0; + }; + + template + struct idle_handler : public idle_handler_base + { + callback_t m_cb; + + idle_handler(callback_t cb) : m_cb(cb) {} + virtual bool do_call(const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) + { + return m_cb(piece_of_data, total_bytes, received_bytes); + } + }; + + class interruptible_http_client : public http_simple_client + { + std::shared_ptr m_pcb; + + virtual bool handle_target_data(std::string& piece_of_transfer) + { + bool r = m_pcb->do_call(piece_of_transfer, m_len_in_summary, m_len_in_summary - m_len_in_remain); + piece_of_transfer.clear(); + return r; + } + + public: + template + bool invoke_cb(callback_t cb, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list()) + { + m_pcb.reset(new idle_handler(cb)); + const http_response_info* p_hri = nullptr; + bool r = invoke_request(url, *this, timeout, &p_hri, method, body, additional_params); + if (p_hri && !(p_hri->m_response_code >= 200 && p_hri->m_response_code < 300)) + { + LOG_PRINT_L0("HTTP request to " << url << " failed with code: " << p_hri->m_response_code); + return false; + } + return r; + } + + template + bool download(callback_t cb, const std::string& path_for_file, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list()) + { + std::ofstream fs; + fs.open(path_for_file, std::ios::binary | std::ios::out | std::ios::trunc); + if (!fs.is_open()) + { + LOG_ERROR("Fsiled to open " << path_for_file); + return false; + } + auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) + { + fs.write(piece_of_data.data(), piece_of_data.size()); + return cb(total_bytes, received_bytes); + }; + bool r = this->invoke_cb(local_cb, url, timeout, method, body, additional_params); + fs.close(); + return r; + } + + // + template + bool download_and_unzip(callback_t cb, const std::string& path_for_file, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), uint64_t fails_count = 1000, const fields_list& additional_params = fields_list()) + { + std::ofstream fs; + fs.open(path_for_file, std::ios::binary | std::ios::out | std::ios::trunc); + if (!fs.is_open()) + { + LOG_ERROR("Fsiled to open " << path_for_file); + return false; + } + std::string buff; + gzip_decoder_lambda zip_decoder; + uint64_t state_total_bytes = 0; + uint64_t state_received_bytes_base = 0; + uint64_t state_received_bytes_current = 0; + bool stopped = false; + auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) + { + //remember total_bytes only for first attempt, where fetched full lenght of the file + if (!state_total_bytes) + state_total_bytes = total_bytes; + + buff += piece_of_data; + return zip_decoder.update_in(buff, [&](const std::string& unpacked_buff) + { + state_received_bytes_current = received_bytes; + fs.write(unpacked_buff.data(), unpacked_buff.size()); + stopped = !cb(unpacked_buff, state_total_bytes, state_received_bytes_base + received_bytes); + return !stopped; + }); + }; + uint64_t current_err_count = 0; + bool r = false; + + while (!r && current_err_count < fails_count) + { + LOG_PRINT_L0("Attempt to invoke http: " << url << " (offset:" << state_received_bytes_base << ")"); + fields_list additional_params_local = additional_params; + additional_params_local.push_back(std::make_pair("Range", std::string("bytes=") + std::to_string(state_received_bytes_base) + "-")); + r = this->invoke_cb(local_cb, url, timeout, method, body, additional_params_local); + if (!r) + { + if (stopped) + break; + current_err_count++; + state_received_bytes_base += state_received_bytes_current; + state_received_bytes_current = 0; + boost::this_thread::sleep_for(boost::chrono::milliseconds(500)); + } + } + + fs.close(); + return r; + } + }; + + + } // namespace http + +} // namespace net_utils +} // namespace epee diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index e3407412..645057eb 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -12,7 +12,7 @@ namespace command_line { const arg_descriptor arg_help = {"help", "Produce help message"}; const arg_descriptor arg_version = {"version", "Output version information"}; - const arg_descriptor arg_data_dir = {"data-dir", "Specify data directory"}; + const arg_descriptor arg_data_dir = {"data-dir", "Specify data directory", ""}; const arg_descriptor arg_config_file = { "config-file", "Specify configuration file", std::string(CURRENCY_NAME_SHORT ".conf") }; const arg_descriptor arg_os_version = { "os-version", "" }; @@ -31,4 +31,10 @@ namespace command_line const arg_descriptor arg_disable_stop_on_low_free_space = { "disable-stop-on-low-free-space", "Do not stop the daemon if free space at data dir is critically low", false, true }; const arg_descriptor arg_enable_offers_service = { "enable-offers-service", "Enables marketplace feature", false, false}; const arg_descriptor arg_db_engine = { "db-engine", "Specify database engine for storage. May be \"lmdb\"(default) or \"mdbx\"", ARG_DB_ENGINE_LMDB, false }; + + const arg_descriptor arg_no_predownload = { "no-predownload", "Do not pre-download blockchain database", }; + const arg_descriptor arg_force_predownload = { "force-predownload", "Pre-download blockchain database regardless of it's status", }; + const arg_descriptor arg_validate_predownload = { "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database", }; + const arg_descriptor arg_predownload_link = { "predownload-link", "Override url for blockchain database pre-downloading", "", true }; + } diff --git a/src/common/command_line.h b/src/common/command_line.h index eb849101..9d6bec47 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -23,6 +23,25 @@ namespace command_line struct arg_descriptor { typedef T value_type; + arg_descriptor(const char* _name, const char* _description): + name(_name), + description(_description), + not_use_default(true), + default_value(T()) + {} + arg_descriptor(const char* _name, const char* _description, const T& default_val) : + name(_name), + description(_description), + not_use_default(false), + default_value(default_val) + {} + arg_descriptor(const char* _name, const char* _description, const T& default_val, bool not_use_default) : + name(_name), + description(_description), + default_value(default_val), + not_use_default(not_use_default) + {} + const char* name; const char* description; @@ -192,4 +211,8 @@ namespace command_line extern const arg_descriptor arg_disable_stop_on_low_free_space; extern const arg_descriptor arg_enable_offers_service; extern const arg_descriptor arg_db_engine; + extern const arg_descriptor arg_no_predownload; + extern const arg_descriptor arg_force_predownload; + extern const arg_descriptor arg_validate_predownload; + extern const arg_descriptor arg_predownload_link; } diff --git a/src/common/config_encrypt_helper.h b/src/common/config_encrypt_helper.h new file mode 100644 index 00000000..c2208c79 --- /dev/null +++ b/src/common/config_encrypt_helper.h @@ -0,0 +1,66 @@ +// Copyright (c) 2014-2020 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include "wallet/view_iface.h" + + +namespace tools +{ + +#pragma pack(push, 1) + struct app_data_file_binary_header + { + uint64_t m_signature; + uint64_t m_cb_body; + }; +#pragma pack (pop) + + inline + std::string load_encrypted_file(const std::string& path, const std::string& key, std::string& body, uint64_t signature) + { + std::string app_data_buff; + bool r = epee::file_io_utils::load_file_to_string(path, app_data_buff); + if (!r) + { + return API_RETURN_CODE_NOT_FOUND; + } + + if (app_data_buff.size() < sizeof(app_data_file_binary_header)) + { + LOG_ERROR("app_data_buff.size()(" << app_data_buff.size() << ") < sizeof(app_data_file_binary_header) (" << sizeof(app_data_file_binary_header) << ") check failed while loading from " << path); + return API_RETURN_CODE_INVALID_FILE; + } + + crypto::chacha_crypt(app_data_buff, key); + + const app_data_file_binary_header* phdr = reinterpret_cast(app_data_buff.data()); + if (phdr->m_signature != signature) + { + return API_RETURN_CODE_WRONG_PASSWORD; + } + body = app_data_buff.substr(sizeof(app_data_file_binary_header)).c_str(); + return API_RETURN_CODE_OK; + } + inline + std::string store_encrypted_file(const std::string& path, const std::string& key, const std::string& body, uint64_t signature) + { + std::string buff(sizeof(app_data_file_binary_header), 0); + app_data_file_binary_header* phdr = (app_data_file_binary_header*)buff.data(); + phdr->m_signature = signature; + phdr->m_cb_body = 0; // for future use + + buff.append(body); + crypto::chacha_crypt(buff, key); + + bool r = epee::file_io_utils::save_string_to_file(path, buff); + if (r) + return API_RETURN_CODE_OK; + else + return API_RETURN_CODE_FAIL; + } + +} \ No newline at end of file diff --git a/src/common/db_backend_mdbx.cpp b/src/common/db_backend_mdbx.cpp index 55bf350c..e5ed964e 100644 --- a/src/common/db_backend_mdbx.cpp +++ b/src/common/db_backend_mdbx.cpp @@ -322,13 +322,13 @@ namespace tools PROFILE_FUNC("mdbx_db_backend::set"); int res = 0; MDBX_val key = AUTO_VAL_INIT(key); - MDBX_val data = AUTO_VAL_INIT(data); + MDBX_val data[2] = {}; // mdbx_put may access data[1] if some flags are set, this may trigger static code analizers, so here we allocate two elements to avoid it key.iov_base = (void*)k; key.iov_len = ks; - data.iov_base = (void*)v; - data.iov_len = vs; + data[0].iov_base = (void*)v; + data[0].iov_len = vs; - res = mdbx_put(get_current_tx(), static_cast(h), &key, &data, 0); + res = mdbx_put(get_current_tx(), static_cast(h), &key, data, 0); CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_put"); return true; } diff --git a/src/common/db_backend_selector.cpp b/src/common/db_backend_selector.cpp new file mode 100644 index 00000000..9c7917ca --- /dev/null +++ b/src/common/db_backend_selector.cpp @@ -0,0 +1,132 @@ +// Copyright (c) 2014-2020 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "db_backend_selector.h" +#include "currency_core/currency_config.h" +#include "command_line.h" +#include "db_backend_lmdb.h" +#include "db_backend_mdbx.h" + +#define LMDB_MAIN_FILE_NAME "data.mdb" +#define MDBX_MAIN_FILE_NAME "mdbx.dat" + +namespace tools +{ +namespace db +{ + + db_backend_selector::db_backend_selector() + : m_engine_type(db_none) + { + } + + void db_backend_selector::init_options(boost::program_options::options_description& desc) + { + command_line::add_arg(desc, command_line::arg_db_engine); + } + + bool db_backend_selector::init(const boost::program_options::variables_map& vm) + { + try + { + m_config_folder = command_line::get_arg(vm, command_line::arg_data_dir); + + if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_LMDB) + { + m_engine_type = db_lmdb; + } + else if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_MDBX) + { +#ifdef ENABLED_ENGINE_MDBX + m_engine_type = db_mdbx; +#else + LOG_PRINT_L0(" DB ENGINE: " << ARG_DB_ENGINE_MDBX << " is not suported by this build(see DISABLE_MDBX cmake option), STOPPING"); +#endif + } + else + { + LOG_PRINT_RED_L0("UNKNOWN DB ENGINE: " << command_line::get_arg(vm, command_line::arg_db_engine) << ", STOPPING"); + } + } + catch (std::exception& e) + { + LOG_ERROR("internal error: db_backend_selector::init failed on command-line parsing, exception: " << e.what()); + return false; + } + + if (m_engine_type == db_none) + return false; + + return true; + } + + std::string db_backend_selector::get_db_folder_path() const + { + //CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited"); + return m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; + } + + std::string db_backend_selector::get_temp_db_folder_path() const + { + //CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited"); + return get_temp_config_folder() + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; + } + + std::string db_backend_selector::get_pool_db_folder_path() const + { + return m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX; + } + + std::string db_backend_selector::get_db_main_file_name() const + { + switch (m_engine_type) + { + case db_lmdb: + return LMDB_MAIN_FILE_NAME; + case db_mdbx: + return MDBX_MAIN_FILE_NAME; + default: + return ""; + } + } + + std::string db_backend_selector::get_engine_name() const + { + switch (m_engine_type) + { + case db_lmdb: + return "lmdb"; + case db_mdbx: + return "mdbx"; + default: + return "unknown"; + } + } + + std::shared_ptr db_backend_selector::create_backend() + { + switch (m_engine_type) + { + case db_lmdb: + return std::shared_ptr(new tools::db::lmdb_db_backend); + + case db_mdbx: + return std::shared_ptr(new tools::db::mdbx_db_backend); + + default: + LOG_ERROR("db_backend_selector was no inited"); + return nullptr; + } + } + + std::string db_backend_selector::get_temp_config_folder() const + { + return m_config_folder + "_TEMP"; + } + + + + +} // namespace db +} // namespace tools diff --git a/src/common/db_backend_selector.h b/src/common/db_backend_selector.h index cc90af76..b9d811b8 100644 --- a/src/common/db_backend_selector.h +++ b/src/common/db_backend_selector.h @@ -1,48 +1,43 @@ -// Copyright (c) 2014-2019 Zano Project -// Copyright (c) 2014-2018 The Louisdor Project +// Copyright (c) 2014-2020 Zano Project // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include "db_backend_lmdb.h" -#include "db_backend_mdbx.h" -#include "common/command_line.h" -#include "common/db_abstract_accessor.h" +#include +#include "misc_language.h" +#include "db_backend_base.h" namespace tools { namespace db { - inline - bool select_db_engine_from_arg(const boost::program_options::variables_map& vm, tools::db::basic_db_accessor& rdb) + enum db_engine_type { db_none = 0, db_lmdb, db_mdbx }; + + class db_backend_selector { - try - { - if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_LMDB) - { - rdb.reset_backend(std::shared_ptr(new tools::db::lmdb_db_backend)); - return true; - } - else if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_MDBX) - { - #ifdef ENABLED_ENGINE_MDBX - rdb.reset_backend(std::shared_ptr(new tools::db::mdbx_db_backend)); - return true; - #else - LOG_PRINT_L0(" DB ENGINE: " << ARG_DB_ENGINE_MDBX << " is not suported by this build(see DISABLE_MDBX cmake option), STOPPING"); - return false; - #endif - } - } - catch (...) - { - LOG_ERROR("internal error: arg_db_engine command-line option could not be read (exception caught)"); - return false; - } + public: + db_backend_selector(); - LOG_PRINT_RED_L0(" UNKNOWN DB ENGINE: " << command_line::get_arg(vm, command_line::arg_db_engine) << ", STOPPING"); - return false; - } - } -} + static void init_options(boost::program_options::options_description& desc); + bool init(const boost::program_options::variables_map& vm); + + std::string get_db_folder_path() const; + std::string get_db_main_file_name() const; + db_engine_type get_engine_type() const { return m_engine_type; } + std::string get_engine_name() const; + std::string get_config_folder() const { return m_config_folder; } + std::string get_temp_config_folder() const; + std::string get_temp_db_folder_path() const; + + std::string get_pool_db_folder_path() const; + + std::shared_ptr create_backend(); + + private: + db_engine_type m_engine_type; + std::string m_config_folder; + }; + + } // namespace db +} // namespace tools diff --git a/src/common/pre_download.h b/src/common/pre_download.h new file mode 100644 index 00000000..d4598207 --- /dev/null +++ b/src/common/pre_download.h @@ -0,0 +1,225 @@ +// Copyright (c) 2020 Zano Project +// Copyright (c) 2012-2018 The Boolberry developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#pragma once + +#include +#include "net/http_client.h" +#include "db_backend_selector.h" +#include "crypto/crypto.h" +#include "currency_core/currency_core.h" + +namespace tools +{ + struct pre_download_entry + { + const char* url; + const char* hash; + uint64_t packed_size; + uint64_t unpacked_size; + }; + +#ifndef TESTNET + static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_lmdb_94_425000.pak", "e6ac69dcf8e7a7017d032cb4326d661c541a3b6a328e6299e6d61c3acde5d49f", 684683820, 1021865984 }; + static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_mdbx_94_425000.pak", "e1f50efba1149a349eb626037dda30052c0233091693a00a10dd5363d5de1206", 535268266, 1073725440 }; +#else + static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_testnet_lmdb_96_99000.pak", "9e8522b287ac7637ca770970542e94702f9fbaa267633cfcaeee4383dfe15bd0", 83851119, 131493888 }; + static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_testnet_mdbx_96_99000.pak", "de33646711f2276e5b22db5741d7b2bf6a8e4c4231d393b730f9a4fce1d7ec03", 63257747, 268431360 }; +#endif + + static constexpr uint64_t pre_download_min_size_difference = 512 * 1024 * 1024; // minimum difference in size between local DB and the downloadable one to start downloading + + template + bool process_predownload(const boost::program_options::variables_map& vm, callback_t cb_should_stop) + { + tools::db::db_backend_selector dbbs; + bool r = dbbs.init(vm); + CHECK_AND_ASSERT_MES(r, false, "db_backend_selector failed to initialize"); + + std::string config_folder = dbbs.get_config_folder(); + std::string working_folder = dbbs.get_db_folder_path(); + std::string db_main_file_path = working_folder + "/" + dbbs.get_db_main_file_name(); + + pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? c_pre_download_lmdb : c_pre_download_mdbx; + + // override pre-download link if necessary + std::string url = pre_download.url; + if (command_line::has_arg(vm, command_line::arg_predownload_link)) + url = command_line::get_arg(vm, command_line::arg_predownload_link); + + boost::system::error_code ec; + uint64_t sz = boost::filesystem::file_size(db_main_file_path, ec); + if (!(ec || (pre_download.unpacked_size > sz && pre_download.unpacked_size - sz > pre_download_min_size_difference) || command_line::has_arg(vm, command_line::arg_force_predownload)) ) + { + LOG_PRINT_MAGENTA("Pre-downloading not needed (db file size = " << sz << ")", LOG_LEVEL_0); + return true; + } + + // okay, let's download + + std::string downloading_file_path = db_main_file_path + ".download"; + + LOG_PRINT_MAGENTA("Trying to download blockchain database from " << url << " ...", LOG_LEVEL_0); + epee::net_utils::http::interruptible_http_client cl; + + crypto::stream_cn_hash hash_stream; + auto last_update = std::chrono::system_clock::now(); + + auto cb = [&hash_stream, &last_update, &cb_should_stop](const std::string& buff, uint64_t total_bytes, uint64_t received_bytes) + { + if (cb_should_stop(total_bytes, received_bytes)) + { + LOG_PRINT_MAGENTA(ENDL << "Interrupting download", LOG_LEVEL_0); + return false; + } + + hash_stream.update(buff.data(), buff.size()); + + auto dif = std::chrono::system_clock::now() - last_update; + if (dif >= std::chrono::milliseconds(300)) + { + boost::io::ios_flags_saver ifs(std::cout); + std::cout << "Received " << received_bytes / 1048576 << " of " << total_bytes / 1048576 << " MiB ( " << std::fixed << std::setprecision(1) << 100.0 * received_bytes / total_bytes << " %)\r"; + last_update = std::chrono::system_clock::now(); + } + + return true; + }; + + tools::create_directories_if_necessary(working_folder); + r = cl.download_and_unzip(cb, downloading_file_path, url, 1000 /* timout */, "GET", std::string(), 3 /* fails count */); + if (!r) + { + LOG_PRINT_RED("Download failed", LOG_LEVEL_0); + return false; + } + + crypto::hash data_hash = hash_stream.calculate_hash(); + if (epee::string_tools::pod_to_hex(data_hash) != pre_download.hash) + { + LOG_ERROR("hash missmatch in downloaded file, got: " << epee::string_tools::pod_to_hex(data_hash) << ", expected: " << pre_download.hash); + return false; + } + + LOG_PRINT_GREEN("Download succeeded, hash " << pre_download.hash << " is correct" , LOG_LEVEL_0); + + if (!command_line::has_arg(vm, command_line::arg_validate_predownload)) + { + boost::filesystem::remove(db_main_file_path, ec); + if (ec) + { + LOG_ERROR("Failed to remove " << db_main_file_path); + return false; + } + LOG_PRINT_L1("Removed " << db_main_file_path); + + boost::filesystem::rename(downloading_file_path, db_main_file_path, ec); + if (ec) + { + LOG_ERROR("Failed to rename " << downloading_file_path << " -> " << db_main_file_path); + return false; + } + LOG_PRINT_L1("Renamed " << downloading_file_path << " -> " << db_main_file_path); + + LOG_PRINT_GREEN("Blockchain successfully replaced with the new pre-downloaded data file", LOG_LEVEL_0); + return true; + } + + // + // paranoid mode + // move downloaded blockchain into a temporary folder + // + LOG_PRINT_MAGENTA(ENDL << "Downloaded blockchain database is about to be validated and added to the local database block-by-block" << ENDL, LOG_LEVEL_0); + std::string path_to_temp_datafolder = dbbs.get_temp_config_folder(); + std::string path_to_temp_blockchain = dbbs.get_temp_db_folder_path(); + std::string path_to_temp_blockchain_file = path_to_temp_blockchain + "/" + dbbs.get_db_main_file_name(); + + tools::create_directories_if_necessary(path_to_temp_blockchain); + boost::filesystem::rename(downloading_file_path, path_to_temp_blockchain_file, ec); + if (ec) + { + LOG_ERROR("Rename failed: " << downloading_file_path << " -> " << path_to_temp_blockchain_file); + return false; + } + + // remove old blockchain database from disk + boost::filesystem::remove_all(working_folder, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << working_folder); + return false; + } + + std::string pool_db_path = dbbs.get_pool_db_folder_path(); + boost::filesystem::remove_all(pool_db_path, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << pool_db_path); + return false; + } + + // source core + currency::core source_core(nullptr); + boost::program_options::variables_map source_core_vm; + source_core_vm.insert(std::make_pair("data-dir", boost::program_options::variable_value(path_to_temp_datafolder, false))); + source_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false))); + //source_core_vm.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false))); + + r = source_core.init(source_core_vm); + CHECK_AND_ASSERT_MES(r, false, "Failed to init source core"); + + // target core + currency::core target_core(nullptr); + boost::program_options::variables_map target_core_vm(vm); + target_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false))); + //vm_with_fast_sync.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false))); + + r = target_core.init(target_core_vm); + CHECK_AND_ASSERT_MES(r, false, "Failed to init target core"); + + CHECK_AND_ASSERT_MES(target_core.get_top_block_height() == 0, false, "Target blockchain initialized not empty"); + uint64_t total_blocks = source_core.get_current_blockchain_size(); + + LOG_PRINT_GREEN("Manually processing blocks from 1 to " << total_blocks << "...", LOG_LEVEL_0); + + for (uint64_t i = 1; i != total_blocks; i++) + { + std::list blocks; + std::list txs; + bool r = source_core.get_blocks(i, 1, blocks, txs); + CHECK_AND_ASSERT_MES(r && blocks.size()==1, false, "Filed to get block " << i << " from core"); + currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash); + for (auto& tx : txs) + { + r = target_core.handle_incoming_tx(tx, tvc, true /* kept_by_block */); + CHECK_AND_ASSERT_MES(r && tvc.m_added_to_pool == true, false, "Failed to add a tx from block " << i << " from core"); + } + currency::block_verification_context bvc = AUTO_VAL_INIT(bvc); + r = target_core.handle_incoming_block(*blocks.begin(), bvc); + CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain == true, false, "Failed to add block " << i << " to core"); + if (!(i % 100)) + std::cout << "Block " << i << "(" << (i * 100) / total_blocks << "%) \r"; + + if (cb_should_stop(total_blocks, i)) + { + LOG_PRINT_MAGENTA(ENDL << "Interrupting updating db...", LOG_LEVEL_0); + return false; + } + } + + LOG_PRINT_GREEN("Processing finished, " << total_blocks << " successfully added.", LOG_LEVEL_0); + target_core.deinit(); + source_core.deinit(); + + boost::filesystem::remove_all(path_to_temp_datafolder, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << path_to_temp_datafolder); + } + + return true; + } +} + diff --git a/src/connectivity_tool/conn_tool.cpp b/src/connectivity_tool/conn_tool.cpp index 666f2f8a..604ced91 100644 --- a/src/connectivity_tool/conn_tool.cpp +++ b/src/connectivity_tool/conn_tool.cpp @@ -8,6 +8,7 @@ #include "include_base_utils.h" #include "version.h" +#include "epee/include/gzip_encoding.h" using namespace epee; #include @@ -23,7 +24,6 @@ using namespace epee; #include "storages/http_abstract_invoke.h" #include "net/http_client.h" #include "currency_core/genesis_acc.h" - #include namespace po = boost::program_options; @@ -59,6 +59,9 @@ namespace const command_line::arg_descriptor arg_download_peer_log = { "download-peer-log", "Download log from remote peer [,]", "", true }; const command_line::arg_descriptor arg_do_consloe_log = { "do-console-log", "Tool generates debug console output(debug purposes)", "", true }; const command_line::arg_descriptor arg_generate_integrated_address = { "generate-integrated-address", "Tool create integrated address from simple address and payment_id", "", true }; + const command_line::arg_descriptor arg_pack_file = {"pack-file", "perform gzip-packing and calculate hash for a given file", "", true }; + const command_line::arg_descriptor arg_unpack_file = {"unpack-file", "Perform gzip-unpacking and calculate hash for a given file", "", true }; + const command_line::arg_descriptor arg_target_file = {"target-file", "Specify target file for pack-file and unpack-file commands", "", true }; } typedef COMMAND_REQUEST_STAT_INFO_T::stat_info> COMMAND_REQUEST_STAT_INFO; @@ -910,7 +913,6 @@ bool invoke_debug_command(po::variables_map& vm, const crypto::secret_key& sk, l return net_utils::invoke_remote_command2(command_t::ID, req, rsp, transport); } - //--------------------------------------------------------------------------------------------------------------- bool handle_set_peer_log_level(po::variables_map& vm) { @@ -1101,6 +1103,121 @@ bool handle_generate_integrated_address(po::variables_map& vm) return true; } //--------------------------------------------------------------------------------------------------------------- +template +bool process_archive(archive_processor_t& arch_processor, bool is_packing, std::ifstream& source, std::ofstream& target) +{ + source.seekg(0, std::ios::end); + uint64_t sz = source.tellg(); + uint64_t remaining = sz; + uint64_t written_bytes = 0; + + crypto::stream_cn_hash hash_stream; + + source.seekg(0, std::ios::beg); + +#define PACK_READ_BLOCKS_SIZE 1048576 // 1MB blocks + + std::string buff; + + auto writer_cb = [&](const std::string& piece_of_transfer) + { + target.write(piece_of_transfer.data(), piece_of_transfer.size()); + written_bytes += piece_of_transfer.size(); + if (!is_packing) + hash_stream.update(piece_of_transfer.data(), piece_of_transfer.size()); + return true; + }; + + while (remaining) + { + uint64_t read_sz = remaining >= PACK_READ_BLOCKS_SIZE ? PACK_READ_BLOCKS_SIZE : remaining; + buff.resize(read_sz); + source.read(const_cast(buff.data()), buff.size()); + if (!source) + { + std::cout << "Error on read from source" << ENDL; + return true; + } + + if (is_packing) + hash_stream.update(buff.data(), buff.size()); + + bool r = arch_processor.update_in(buff, writer_cb); + CHECK_AND_ASSERT_MES(r, false, "arch_processor.update_in failed"); + + + remaining -= read_sz; + std::cout << "Progress: " << ((sz - remaining) * 100) / sz << "%\r"; + } + + //flush gzip decoder + arch_processor.stop(writer_cb); + + source.close(); + target.close(); + + crypto::hash data_hash = hash_stream.calculate_hash(); + + std::cout << "\r\nFile " << (is_packing ? "packed" : "unpacked") << " from size " << sz << " to " << written_bytes << + "\r\nhash of the data is " << epee::string_tools::pod_to_hex(data_hash) << "\r\n"; + + return true; +} + +bool handle_pack_file(po::variables_map& vm) +{ + bool do_pack = false; + std::string path_source; + std::string path_target; + if (command_line::has_arg(vm, arg_pack_file)) + { + path_source = command_line::get_arg(vm, arg_pack_file); + do_pack = true; + } + else if (command_line::has_arg(vm, arg_unpack_file)) + { + path_source = command_line::get_arg(vm, arg_unpack_file); + do_pack = false; + } + else + { + return false; + } + + if (!command_line::has_arg(vm, arg_target_file)) + std::cout << "Error: Parameter target_file is not set." << ENDL; + path_target = command_line::get_arg(vm, arg_target_file); + + std::ifstream source; + source.open(path_source, std::ios::binary | std::ios::in ); + if (!source.is_open()) + { + std::cout << "Error: Unable to open " << path_source << ENDL; + return false; + } + + std::ofstream target; + target.open(path_target, std::ios::binary | std::ios::out | std::ios::trunc); + if (!target.is_open()) + { + std::cout << "Error: Unable to open " << path_target << ENDL; + return false; + } + + if (do_pack) + { + epee::net_utils::gzip_encoder_lyambda gzip_encoder(Z_BEST_COMPRESSION); + return process_archive(gzip_encoder, true, source, target); + } + else + { + epee::net_utils::gzip_decoder_lambda gzip_decoder; + return process_archive(gzip_decoder, false, source, target); + } +} + +//--------------------------------------------------------------------------------------------------------------- + int main(int argc, char* argv[]) { try @@ -1142,8 +1259,9 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_download_peer_log); command_line::add_arg(desc_params, arg_do_consloe_log); command_line::add_arg(desc_params, arg_generate_integrated_address); - - + command_line::add_arg(desc_params, arg_pack_file); + command_line::add_arg(desc_params, arg_unpack_file); + command_line::add_arg(desc_params, arg_target_file); po::options_description desc_all; desc_all.add(desc_general).add(desc_params); @@ -1216,6 +1334,10 @@ int main(int argc, char* argv[]) { return handle_generate_integrated_address(vm) ? EXIT_SUCCESS : EXIT_FAILURE; } + else if (command_line::has_arg(vm, arg_pack_file) || command_line::has_arg(vm, arg_unpack_file)) + { + return handle_pack_file(vm) ? EXIT_SUCCESS : EXIT_FAILURE; + } else { std::cerr << "Not enough arguments." << ENDL; diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 9026ed36..99262c91 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "common/pod-class.h" #include "generic-ops.h" @@ -226,6 +227,69 @@ namespace crypto { return check_ring_signature(prefix_hash, image, pubs.data(), pubs.size(), sig); } + class stream_cn_hash + { + public: + static constexpr size_t DATA_BLOCK_SIZE = 1024 * 1024; + + stream_cn_hash() + : m_p_hash(reinterpret_cast(&m_buffer)) + , m_p_data(m_buffer + HASH_SIZE) + , m_ready(false) + , m_data_used(0) + { + memset(m_buffer, 0, HASH_SIZE + DATA_BLOCK_SIZE); + m_ready = true; + } + + bool update(const void* data, size_t size) + { + if (!m_ready) + return false; + + const uint8_t* p_source_data = reinterpret_cast(data); + + while(size > 0) + { + // fill the buffer up + size_t bytes_to_copy = std::min(size, DATA_BLOCK_SIZE - m_data_used); + memcpy(m_p_data + m_data_used, p_source_data, bytes_to_copy); + m_data_used += bytes_to_copy; + p_source_data += bytes_to_copy; + size -= bytes_to_copy; + + if (m_data_used == DATA_BLOCK_SIZE) + { + // calc imtermediate hash of the whole buffer and put the result into the beginning of the buffer + *m_p_hash = cn_fast_hash(m_buffer, HASH_SIZE + m_data_used); + // clear data buffer for new bytes + memset(m_p_data, 0, DATA_BLOCK_SIZE); + m_data_used = 0; + } + + // repeat if there are source bytes left + } + + return true; + } + + hash calculate_hash() + { + if (m_data_used == 0) + return *m_p_hash; + + m_ready = false; + return cn_fast_hash(m_buffer, HASH_SIZE + m_data_used); + } + + private: + uint8_t m_buffer[HASH_SIZE + DATA_BLOCK_SIZE]; + hash* const m_p_hash; + uint8_t* const m_p_data; + size_t m_data_used; + bool m_ready; + }; + } diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 481ea09e..54b45be9 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -152,7 +152,6 @@ void blockchain_storage::init_options(boost::program_options::options_descriptio { command_line::add_arg(desc, arg_db_cache_l1); command_line::add_arg(desc, arg_db_cache_l2); - command_line::add_arg(desc, command_line::arg_db_engine); } //------------------------------------------------------------------ uint64_t blockchain_storage::get_block_h_older_then(uint64_t timestamp) const @@ -206,11 +205,16 @@ bool blockchain_storage::validate_instance(const std::string& path) bool blockchain_storage::init(const std::string& config_folder, const boost::program_options::variables_map& vm) { // CRITICAL_REGION_LOCAL(m_read_lock); - if (!select_db_engine_from_arg(vm, m_db)) + + tools::db::db_backend_selector dbbs; + dbbs.init(vm); + auto p_backend = dbbs.create_backend(); + if (!p_backend) { - LOG_PRINT_RED_L0("Failed to select db engine"); + LOG_PRINT_RED_L0("Failed to create db engine"); return false; } + m_db.reset_backend(p_backend); LOG_PRINT_L0("DB ENGINE USED BY CORE: " << m_db.get_backend()->name()); if (!validate_instance(config_folder)) @@ -235,8 +239,8 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0); boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path)); } - ; - const std::string db_folder_path = m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + m_db.get_backend()->name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; + + const std::string db_folder_path = dbbs.get_db_folder_path(); LOG_PRINT_L0("Loading blockchain from " << db_folder_path); bool db_opened_okay = false; diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index ceb88406..004df00e 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -184,6 +184,34 @@ namespace currency return true; } //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ /* = null_hash */) + { + TIME_MEASURE_START_MS(wait_lock_time); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); + TIME_MEASURE_FINISH_MS(wait_lock_time); + + crypto::hash tx_hash = tx_hash_; + if (tx_hash == null_hash) + tx_hash = get_transaction_hash(tx); + + TIME_MEASURE_START_MS(add_new_tx_time); + bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block); + TIME_MEASURE_FINISH_MS(add_new_tx_time); + + if(tvc.m_verification_failed) + {LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);} + else if(tvc.m_verification_impossible) + {LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);} + + if (tvc.m_added_to_pool) + { + LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool"); + } + LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX1]: timing " << wait_lock_time + << "/" << add_new_tx_time); + return r; + } + //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block) { CHECK_AND_ASSERT_MES(!kept_by_block, false, "Transaction associated with block came throw handle_incoming_tx!(not allowed anymore)"); @@ -212,7 +240,6 @@ namespace currency } TIME_MEASURE_FINISH_MS(parse_tx_time); - TIME_MEASURE_START_MS(check_tx_semantic_time); if(!validate_tx_semantic(tx, tx_blob.size())) { @@ -222,23 +249,10 @@ namespace currency } TIME_MEASURE_FINISH_MS(check_tx_semantic_time); - TIME_MEASURE_START_MS(add_new_tx_time); - bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block); - TIME_MEASURE_FINISH_MS(add_new_tx_time); - - if(tvc.m_verification_failed) - {LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);} - else if(tvc.m_verification_impossible) - {LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);} - - if (tvc.m_added_to_pool) - { - LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool"); - } - LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX]: timing " << wait_lock_time + bool r = handle_incoming_tx(tx, tvc, kept_by_block, tx_hash); + LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX2]: timing " << wait_lock_time << "/" << parse_tx_time - << "/" << check_tx_semantic_time - << "/" << add_new_tx_time); + << "/" << check_tx_semantic_time); return r; } //----------------------------------------------------------------------------------------------- diff --git a/src/currency_core/currency_core.h b/src/currency_core/currency_core.h index faeec262..68d9bead 100644 --- a/src/currency_core/currency_core.h +++ b/src/currency_core/currency_core.h @@ -41,6 +41,7 @@ namespace currency core(i_currency_protocol* pprotocol); bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, currency_connection_context& context)const ; bool on_idle(); + bool handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ = null_hash); bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block); bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true); bool handle_incoming_block(const block& b, block_verification_context& bvc, bool update_miner_blocktemplate = true); diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index 43bde8d0..aef7e603 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -20,6 +20,7 @@ #include "warnings.h" #include "crypto/hash.h" #include "profile_tools.h" +#include "common/db_backend_selector.h" DISABLE_VS_WARNINGS(4244 4345 4503) //'boost::foreach_detail_::or_' : decorated name length exceeded, name was truncated @@ -1163,11 +1164,15 @@ namespace currency //--------------------------------------------------------------------------------- bool tx_memory_pool::init(const std::string& config_folder, const boost::program_options::variables_map& vm) { - if (!select_db_engine_from_arg(vm, m_db)) + tools::db::db_backend_selector dbbs; + dbbs.init(vm); + auto p_backend = dbbs.create_backend(); + if (!p_backend) { - LOG_PRINT_RED_L0("Failed to select db engine"); + LOG_PRINT_RED_L0("Failed to create db engine"); return false; } + m_db.reset_backend(p_backend); LOG_PRINT_L0("DB ENGINE USED BY POOL: " << m_db.get_backend()->name()); m_config_folder = config_folder; @@ -1183,7 +1188,7 @@ namespace currency boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path)); } - const std::string db_folder_path = m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + m_db.get_backend()->name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX; + const std::string db_folder_path = dbbs.get_pool_db_folder_path(); LOG_PRINT_L0("Loading blockchain from " << db_folder_path << "..."); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index f81d1a5a..a9b9ad23 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -27,6 +27,7 @@ using namespace epee; #include "version.h" #include "currency_core/core_tools.h" #include "common/callstack_helper.h" +#include "common/pre_download.h" #include @@ -148,6 +149,11 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, command_line::arg_disable_stop_on_low_free_space); command_line::add_arg(desc_cmd_sett, command_line::arg_enable_offers_service); + command_line::add_arg(desc_cmd_sett, command_line::arg_no_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_force_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link); + arg_market_disable.default_value = true; arg_market_disable.not_use_default = false; @@ -159,7 +165,7 @@ int main(int argc, char* argv[]) currency::miner::init_options(desc_cmd_sett); bc_services::bc_offers_service::init_options(desc_cmd_sett); currency::stratum_server::init_options(desc_cmd_sett); - + tools::db::db_backend_selector::init_options(desc_cmd_sett); po::options_description desc_options("Allowed options"); desc_options.add(desc_cmd_only).add(desc_cmd_sett); @@ -270,8 +276,25 @@ int main(int argc, char* argv[]) LOG_PRINT_L0(generate_reference << ENDL << "----------------------------------------" << ENDL << json_rpc_reference); } - bool res = false; + + //do pre_download if needed + if (!command_line::has_arg(vm, command_line::arg_no_predownload) || command_line::has_arg(vm, command_line::arg_force_predownload)) + { + auto is_stop_signal_sent = [&p2psrv]() -> bool { + return static_cast::connection_context>*>(&p2psrv)->is_stop_signal_sent(); + }; + + if (!tools::process_predownload(vm, [&](uint64_t total_bytes, uint64_t received_bytes) { return is_stop_signal_sent(); })) + { + return EXIT_FAILURE; + } + + if (is_stop_signal_sent()) + return 1; + } + + //initialize objects LOG_PRINT_L0("Initializing p2p server..."); res = p2psrv.init(vm); diff --git a/src/gui/qt-daemon/application/mainwindow.cpp b/src/gui/qt-daemon/application/mainwindow.cpp index 5d0a148c..6da2ec90 100644 --- a/src/gui/qt-daemon/application/mainwindow.cpp +++ b/src/gui/qt-daemon/application/mainwindow.cpp @@ -12,6 +12,7 @@ #include "string_coding.h" #include "gui_utils.h" #include "notification_helper.h" +#include "common/config_encrypt_helper.h" #define PREPARE_ARG_FROM_JSON(arg_type, var_name) \ arg_type var_name = AUTO_VAL_INIT(var_name); \ @@ -1047,6 +1048,35 @@ void MainWindow::on_clear_events() CATCH_ENTRY2(void()); } + +QString MainWindow::store_secure_app_data(const QString& param) +{ + TRY_ENTRY(); + LOG_API_TIMING(); + if (!tools::create_directories_if_necessary(m_backend.get_config_folder())) + { + view::api_response ar; + LOG_PRINT_L0("Failed to create data directory: " << m_backend.get_config_folder()); + ar.error_code = API_RETURN_CODE_FAIL; + return MAKE_RESPONSE(ar); + } + + view::api_response ar = AUTO_VAL_INIT(ar); + ar.error_code = tools::store_encrypted_file(m_backend.get_config_folder() + "/" + GUI_SECURE_CONFIG_FILENAME, + m_master_password, param.toStdString(), APP_DATA_FILE_BINARY_SIGNATURE); + if (ar.error_code != API_RETURN_CODE_OK) + { + return MAKE_RESPONSE(ar); + } + + crypto::hash master_password_pre_hash = crypto::cn_fast_hash(m_master_password.c_str(), m_master_password.length()); + crypto::hash master_password_hash = crypto::cn_fast_hash(&master_password_pre_hash, sizeof master_password_pre_hash); + LOG_PRINT_L0("store_secure_app_data, r = " << ar.error_code << ", pass hash: " << master_password_hash); + + return MAKE_RESPONSE(ar); + CATCH_ENTRY_FAIL_API_RESPONCE(); +} + QString MainWindow::get_secure_app_data(const QString& param) { TRY_ENTRY(); @@ -1060,41 +1090,21 @@ QString MainWindow::get_secure_app_data(const QString& param) return MAKE_RESPONSE(ar); } - std::string app_data_buff; std::string filename = m_backend.get_config_folder() + "/" + GUI_SECURE_CONFIG_FILENAME; - bool r = file_io_utils::load_file_to_string(filename, app_data_buff); - if (!r) + std::string res_body; + std::string rsp_code = tools::load_encrypted_file(filename, pwd.pass, res_body, APP_DATA_FILE_BINARY_SIGNATURE); + if (rsp_code != API_RETURN_CODE_OK) { - LOG_PRINT_L1("gui secure config was not loaded from " << filename); - return ""; - } - - if (app_data_buff.size() < sizeof(app_data_file_binary_header)) - { - LOG_ERROR("app_data_buff.size() < sizeof(app_data_file_binary_header) check failed while loading from " << filename); view::api_response ar; - ar.error_code = API_RETURN_CODE_FAIL; + ar.error_code = rsp_code; return MAKE_RESPONSE(ar); } - - crypto::chacha_crypt(app_data_buff, pwd.pass); - - const app_data_file_binary_header* phdr = reinterpret_cast(app_data_buff.data()); - if (phdr->m_signature != APP_DATA_FILE_BINARY_SIGNATURE) - { - LOG_ERROR("gui secure config: password missmatch while loading from " << filename); - view::api_response ar; - ar.error_code = API_RETURN_CODE_WRONG_PASSWORD; - return MAKE_RESPONSE(ar); - } - m_master_password = pwd.pass; - crypto::hash master_password_pre_hash = crypto::cn_fast_hash(m_master_password.c_str(), m_master_password.length()); crypto::hash master_password_hash = crypto::cn_fast_hash(&master_password_pre_hash, sizeof master_password_pre_hash); LOG_PRINT_L0("gui secure config loaded ok from " << filename << ", pass hash: " << master_password_hash); - return app_data_buff.substr(sizeof(app_data_file_binary_header)).c_str(); + return res_body.c_str(); CATCH_ENTRY2(API_RETURN_CODE_INTERNAL_ERROR); } @@ -1267,40 +1277,6 @@ QString MainWindow::get_app_data() CATCH_ENTRY2(API_RETURN_CODE_INTERNAL_ERROR); } -QString MainWindow::store_secure_app_data(const QString& param) -{ - TRY_ENTRY(); - LOG_API_TIMING(); - if (!tools::create_directories_if_necessary(m_backend.get_config_folder())) - { - view::api_response ar; - LOG_PRINT_L0("Failed to create data directory: " << m_backend.get_config_folder()); - return MAKE_RESPONSE(ar); - } - - - std::string buff(sizeof(app_data_file_binary_header), 0); - app_data_file_binary_header* phdr = (app_data_file_binary_header*)buff.data(); - phdr->m_signature = APP_DATA_FILE_BINARY_SIGNATURE; - phdr->m_cb_body = 0; // for future use - - buff.append(param.toStdString()); - crypto::chacha_crypt(buff, m_master_password); - - bool r = file_io_utils::save_string_to_file(m_backend.get_config_folder() + "/" + GUI_SECURE_CONFIG_FILENAME, buff); - view::api_response ar; - if (r) - ar.error_code = API_RETURN_CODE_OK; - else - ar.error_code = API_RETURN_CODE_FAIL; - - crypto::hash master_password_pre_hash = crypto::cn_fast_hash(m_master_password.c_str(), m_master_password.length()); - crypto::hash master_password_hash = crypto::cn_fast_hash(&master_password_pre_hash, sizeof master_password_pre_hash); - LOG_PRINT_L0("store_secure_app_data, r = " << r << ", pass hash: " << master_password_hash); - - return MAKE_RESPONSE(ar); - CATCH_ENTRY_FAIL_API_RESPONCE(); -} QString MainWindow::have_secure_app_data() { diff --git a/src/gui/qt-daemon/application/mainwindow.h b/src/gui/qt-daemon/application/mainwindow.h index 2acdd34f..4ea84e1d 100644 --- a/src/gui/qt-daemon/application/mainwindow.h +++ b/src/gui/qt-daemon/application/mainwindow.h @@ -8,6 +8,7 @@ #include #include "wallet/view_iface.h" + #ifndef Q_MOC_RUN #include "wallet/wallets_manager.h" #include "currency_core/offers_services_helpers.h" @@ -18,13 +19,7 @@ class QWebEngineView; class QLineEdit; QT_END_NAMESPACE -#pragma pack(push, 1) -struct app_data_file_binary_header -{ - uint64_t m_signature; - uint64_t m_cb_body; -}; -#pragma pack (pop) + #define APP_DATA_FILE_BINARY_SIGNATURE 0x1000111101101021LL diff --git a/src/gui/qt-daemon/main.cpp b/src/gui/qt-daemon/main.cpp index 28e7a564..94efe369 100644 --- a/src/gui/qt-daemon/main.cpp +++ b/src/gui/qt-daemon/main.cpp @@ -60,6 +60,11 @@ int main(int argc, char *argv[]) #endif #endif + log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); + log_space::get_set_need_thread_id(true, true); + log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,p2p,wallet"); + + QApplication app(argc, argv); MainWindow viewer; if (!viewer.init_backend(argc, argv)) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index dc00ae0c..005f68f1 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -386,6 +386,9 @@ namespace currency return true; } + res.tx_expiration_ts_median = m_core.get_blockchain_storage().get_tx_expiration_median(); + + for(auto& tx: txs) { res.txs.push_back(t_serializable_object_to_blob(tx)); @@ -393,6 +396,7 @@ namespace currency res.status = CORE_RPC_STATUS_OK; return true; } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_scan_pos(const COMMAND_RPC_SCAN_POS::request& req, COMMAND_RPC_SCAN_POS::response& res, connection_context& cntx) { CHECK_CORE_READY(); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index c0cb4990..2d7130bc 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -181,10 +181,12 @@ namespace currency struct response { std::list txs; //transactions blobs + uint64_t tx_expiration_ts_median; std::string status; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) + KV_SERIALIZE(tx_expiration_ts_median) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; @@ -614,7 +616,8 @@ namespace currency daemon_network_state_online = 2, daemon_network_state_loading_core = 3, daemon_network_state_internal_error = 4, - daemon_network_state_unloading_core = 5 + daemon_network_state_unloading_core = 5, + daemon_network_state_downloading_database = 6 }; struct response diff --git a/src/wallet/core_default_rpc_proxy.cpp b/src/wallet/core_default_rpc_proxy.cpp index ae6123bd..4db9330d 100644 --- a/src/wallet/core_default_rpc_proxy.cpp +++ b/src/wallet/core_default_rpc_proxy.cpp @@ -11,6 +11,11 @@ using namespace epee; #include "currency_core/currency_format_utils.h" #include "currency_core/alias_helper.h" +#undef LOG_DEFAULT_CHANNEL +#define LOG_DEFAULT_CHANNEL "rpc_proxy" +ENABLE_CHANNEL_BY_DEFAULT("rpc_proxy") + + namespace tools { bool default_http_core_proxy::set_connection_addr(const std::string& url) @@ -131,7 +136,13 @@ namespace tools epee::net_utils::parse_url(m_daemon_address, u); if (!u.port) u.port = 8081; - return m_http_client.connect(u.host, std::to_string(u.port), WALLET_RCP_CONNECTION_TIMEOUT); + bool r = m_http_client.connect(u.host, std::to_string(u.port), m_connection_timeout); + if (r) + { + *m_plast_daemon_is_disconnected = false; + m_last_success_interract_time = time(nullptr); + } + return r; } //------------------------------------------------------------------------------------------------------------------------------ bool default_http_core_proxy::call_COMMAND_RPC_GET_ALL_ALIASES(currency::COMMAND_RPC_GET_ALL_ALIASES::response& res) @@ -166,7 +177,17 @@ namespace tools m_plast_daemon_is_disconnected = plast_daemon_is_disconnected ? plast_daemon_is_disconnected : &m_last_daemon_is_disconnected_stub; } //------------------------------------------------------------------------------------------------------------------------------ - default_http_core_proxy::default_http_core_proxy():m_plast_daemon_is_disconnected(&m_last_daemon_is_disconnected_stub) + bool default_http_core_proxy::set_connectivity(unsigned int connection_timeout, size_t repeats_count) + { + m_connection_timeout = connection_timeout; + m_attempts_count = repeats_count; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + default_http_core_proxy::default_http_core_proxy() :m_plast_daemon_is_disconnected(&m_last_daemon_is_disconnected_stub), + m_last_success_interract_time(0), + m_connection_timeout(WALLET_RCP_CONNECTION_TIMEOUT), + m_attempts_count(WALLET_RCP_COUNT_ATTEMNTS) { } diff --git a/src/wallet/core_default_rpc_proxy.h b/src/wallet/core_default_rpc_proxy.h index c855b212..f180ff26 100644 --- a/src/wallet/core_default_rpc_proxy.h +++ b/src/wallet/core_default_rpc_proxy.h @@ -11,9 +11,11 @@ #include "core_rpc_proxy.h" #include "storages/http_abstract_invoke.h" -#define WALLET_RCP_CONNECTION_TIMEOUT 10000 +#define WALLET_RCP_CONNECTION_TIMEOUT 3000 #define WALLET_RCP_COUNT_ATTEMNTS 3 + + namespace tools { class default_http_core_proxy final : public i_core_proxy @@ -22,6 +24,7 @@ namespace tools bool set_connection_addr(const std::string& url) override; + bool set_connectivity(unsigned int connection_timeout, size_t repeats_count); bool call_COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES(const currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& rqt, currency::COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& rsp) override; bool call_COMMAND_RPC_GET_BLOCKS_FAST(const currency::COMMAND_RPC_GET_BLOCKS_FAST::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_FAST::response& rsp) override; bool call_COMMAND_RPC_GET_BLOCKS_DIRECT(const currency::COMMAND_RPC_GET_BLOCKS_DIRECT::request& rqt, currency::COMMAND_RPC_GET_BLOCKS_DIRECT::response& rsp) override; @@ -58,13 +61,16 @@ namespace tools CRITICAL_REGION_LOCAL(m_lock); bool ret = false; - for(size_t i = WALLET_RCP_COUNT_ATTEMNTS; i && !ret; --i) + for(size_t i = m_attempts_count; i && !ret; --i) { ret = request(); } if (ret) + { m_last_success_interract_time = time(nullptr); + *m_plast_daemon_is_disconnected = false; + } else *m_plast_daemon_is_disconnected = true; return ret; @@ -74,7 +80,14 @@ namespace tools inline bool invoke_http_json_rpc_update_is_disconnect(const std::string& method_name, const t_request& req, t_response& res) { return call_request([&](){ - return epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client); +#ifdef MOBILE_WALLET_BUILD + LOG_PRINT_L0("[INVOKE_JSON_METHOD] ---> " << method_name) +#endif + bool r = epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client); +#ifdef MOBILE_WALLET_BUILD + LOG_PRINT_L0("[INVOKE_JSON_METHOD] <---" << method_name) +#endif + return r; }); } @@ -82,7 +95,14 @@ namespace tools inline bool invoke_http_bin_remote_command2_update_is_disconnect(const std::string& url, const t_request& req, t_response& res) { return call_request([&](){ - return epee::net_utils::invoke_http_bin_remote_command2(m_daemon_address + url, req, res, m_http_client, WALLET_RCP_CONNECTION_TIMEOUT); +#ifdef MOBILE_WALLET_BUILD + LOG_PRINT_L0("[INVOKE_BIN] --->" << typeid(t_request).name()) +#endif + bool r = epee::net_utils::invoke_http_bin_remote_command2(m_daemon_address + url, req, res, m_http_client, m_connection_timeout); +#ifdef MOBILE_WALLET_BUILD + LOG_PRINT_L0("[INVOKE_BIN] <---" << typeid(t_request).name()) +#endif + return r; }); } @@ -90,7 +110,14 @@ namespace tools inline bool invoke_http_json_remote_command2_update_is_disconnect(const std::string& url, const t_request& req, t_response& res) { return call_request([&](){ - return epee::net_utils::invoke_http_json_remote_command2(m_daemon_address + url, req, res, m_http_client, WALLET_RCP_CONNECTION_TIMEOUT); +#ifdef MOBILE_WALLET_BUILD + LOG_PRINT_L0("[INVOKE_JSON_URL] --->" << typeid(t_request).name() ) +#endif + bool r = epee::net_utils::invoke_http_json_remote_command2(m_daemon_address + url, req, res, m_http_client, m_connection_timeout); +#ifdef MOBILE_WALLET_BUILD + LOG_PRINT_L0("[INVOKE_JSON_URL] <---" << typeid(t_request).name()) +#endif + return r; }); } //------------------------------------------------------------------------------------------------------------------------------ @@ -105,6 +132,8 @@ namespace tools std::atomic m_last_success_interract_time; std::atomic *m_plast_daemon_is_disconnected; std::atomic m_last_daemon_is_disconnected_stub; + unsigned int m_connection_timeout; + size_t m_attempts_count; }; } diff --git a/src/wallet/plain_wallet_api.cpp b/src/wallet/plain_wallet_api.cpp index 6103cf0c..c76ef533 100644 --- a/src/wallet/plain_wallet_api.cpp +++ b/src/wallet/plain_wallet_api.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include "plain_wallet_api.h" #include "plain_wallet_api_impl.h" #include "currency_core/currency_config.h" @@ -10,6 +11,8 @@ #include "string_tools.h" #include "currency_core/currency_format_utils.h" #include "wallets_manager.h" +#include "common/base58.h" +#include "common/config_encrypt_helper.h" #define ANDROID_PACKAGE_NAME "com.zano_mobile" #ifdef IOS_BUILD @@ -17,9 +20,13 @@ #elif ANDROID_BUILD #define HOME_FOLDER "files" #else - #define HOME_FOLDER "" + #define HOME_FOLDER "logs" #endif #define WALLETS_FOLDER_NAME "wallets" +#define APP_CONFIG_FOLDER "app_config" +#define APP_CONFIG_FILENAME "app_cfg.bin" + +#define MOBILE_APP_DATA_FILE_BINARY_SIGNATURE 0x1000111201101011LL //Bender's nightmare #define GENERAL_INTERNAL_ERRROR_INSTANCE "GENERAL_INTERNAL_ERROR: WALLET INSTNACE NOT FOUND" #define GENERAL_INTERNAL_ERRROR_INIT "Failed to intialize library" @@ -39,14 +46,15 @@ namespace plain_wallet std::string get_bundle_root_dir() { #ifdef WIN32 - return ""; -#endif // WIN32 -#ifdef IOS_BUILD + return boost::dll::program_location().parent_path().string(); +#elif IOS_BUILD char* env = getenv("HOME"); return env ? env : ""; #elif ANDROID_BUILD /// data/data/com.zano_mobile/files return "/data/data/" ANDROID_PACKAGE_NAME; +#else + return ""; #endif } @@ -60,11 +68,26 @@ namespace plain_wallet #endif // WIN32 } - void initialize_logs() + std::string get_app_config_folder() + { +#ifdef WIN32 + return ""; +#else + std::string path = get_bundle_root_dir() + "/" + HOME_FOLDER + "/" + APP_CONFIG_FOLDER + "/"; + return path; +#endif // WIN32 + } + + + + void initialize_logs(int log_level) { std::string log_dir = get_bundle_root_dir(); log_dir += "/" HOME_FOLDER; - epee::log_space::get_set_log_detalisation_level(true, LOG_LEVEL_2); + + log_space::get_set_need_thread_id(true, true); + log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,p2p,wallet"); + epee::log_space::get_set_log_detalisation_level(true, log_level); epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); epee::log_space::log_singletone::add_logger(LOGGER_FILE, "plain_wallet.log", log_dir.c_str()); LOG_PRINT_L0("Plain wallet initialized: " << CURRENCY_NAME << " v" << PROJECT_VERSION_LONG << ", log location: " << log_dir + "/plain_wallet.log"); @@ -72,23 +95,32 @@ namespace plain_wallet //glogs_initialized = true; } - std::string init(const std::string& ip, const std::string& port) + std::string set_log_level(int log_level) + { + epee::log_space::get_set_log_detalisation_level(true, log_level); + return "{}"; + } + + std::string init(const std::string& ip, const std::string& port, int log_level) { if (initialized) { LOG_ERROR("Double-initialization in plain_wallet detected."); - //throw std::runtime_error("Double-initialization in plain_wallet detected."); - return "Already initialized!"; + epee::json_rpc::response ok_response = AUTO_VAL_INIT(ok_response); + ok_response.result.return_code = API_RETURN_CODE_ALREADY_EXISTS; + return epee::serialization::store_t_to_json(ok_response); } - initialize_logs(); + initialize_logs(log_level); std::string argss_1 = std::string("--remote-node=") + ip + ":" + port; - char * args[3]; + std::string argss_2 = std::string("--disable-logs-init"); + char * args[4]; args[0] = "stub"; args[1] = const_cast(argss_1.c_str()); - args[2] = nullptr; - if (!gwm.init(2, args, nullptr)) + args[2] = const_cast(argss_2.c_str()); + args[3] = nullptr; + if (!gwm.init(3, args, nullptr)) { LOG_ERROR("Failed to init wallets_manager"); return GENERAL_INTERNAL_ERRROR_INIT; @@ -100,13 +132,84 @@ namespace plain_wallet return GENERAL_INTERNAL_ERRROR_INIT; } - std::string wallet_folder = get_wallets_folder(); + std::string wallets_folder = get_wallets_folder(); boost::system::error_code ec; - boost::filesystem::create_directories(wallet_folder, ec); + boost::filesystem::create_directories(wallets_folder, ec); + if (ec) + { + error_response err_result = AUTO_VAL_INIT(err_result); + err_result.error.code = API_RETURN_CODE_INTERNAL_ERROR; + err_result.error.message = LOCATION_STR + " \nmessage:" + ec.message(); + return epee::serialization::store_t_to_json(err_result); + } + + std::string app_config_folder = get_app_config_folder(); + boost::filesystem::create_directories(app_config_folder, ec); + if (ec) + { + error_response err_result = AUTO_VAL_INIT(err_result); + err_result.error.code = API_RETURN_CODE_INTERNAL_ERROR; + err_result.error.message = LOCATION_STR + " \nmessage:" + ec.message(); + return epee::serialization::store_t_to_json(err_result); + } + initialized = true; - return API_RETURN_CODE_OK; + epee::json_rpc::response ok_response = AUTO_VAL_INIT(ok_response); + ok_response.result.return_code = API_RETURN_CODE_OK; + return epee::serialization::store_t_to_json(ok_response); } + std::string get_appconfig(const std::string& encryption_key) + { + std::string res_str; + std::string app_config_config_path = get_app_config_folder() + APP_CONFIG_FILENAME; + std::string ret_code = tools::load_encrypted_file(app_config_config_path, encryption_key, res_str, MOBILE_APP_DATA_FILE_BINARY_SIGNATURE); + if (ret_code != API_RETURN_CODE_OK) + { + error_response err_result = AUTO_VAL_INIT(err_result); + err_result.error.code = ret_code; + return epee::serialization::store_t_to_json(err_result); + } + return res_str; + } + std::string set_appconfig(const std::string& conf_str, const std::string& encryption_key) + { + std::string app_config_config_path = get_app_config_folder() + APP_CONFIG_FILENAME; + std::string ret_code = tools::store_encrypted_file(app_config_config_path, encryption_key, conf_str, MOBILE_APP_DATA_FILE_BINARY_SIGNATURE); + if (ret_code != API_RETURN_CODE_OK) + { + error_response err_result = AUTO_VAL_INIT(err_result); + err_result.error.code = ret_code; + return epee::serialization::store_t_to_json(err_result); + } + else + { + epee::json_rpc::response ok_response = AUTO_VAL_INIT(ok_response); + ok_response.result.return_code = API_RETURN_CODE_OK; + return epee::serialization::store_t_to_json(ok_response); + } + } + + std::string generate_random_key(uint64_t lenght) + { + std::string buff; + buff.resize(lenght); + crypto::generate_random_bytes(lenght, const_cast(buff.data())); + return tools::base58::encode(buff); + } + + std::string get_logs_buffer() + { + return epee::log_space::log_singletone::copy_logs_to_buffer(); + } + + std::string truncate_log() + { + epee::log_space::log_singletone::truncate_log_files(); + epee::json_rpc::response ok_response = AUTO_VAL_INIT(ok_response); + ok_response.result.return_code = API_RETURN_CODE_OK; + return epee::serialization::store_t_to_json(ok_response); + } std::string get_version() diff --git a/src/wallet/plain_wallet_api.h b/src/wallet/plain_wallet_api.h index 01dc044d..45abefde 100644 --- a/src/wallet/plain_wallet_api.h +++ b/src/wallet/plain_wallet_api.h @@ -10,10 +10,17 @@ namespace plain_wallet { typedef int64_t hwallet; - std::string init(const std::string& ip, const std::string& port); + std::string init(const std::string& ip, const std::string& port, int log_level); + std::string set_log_level(int log_level); std::string get_version(); std::string get_wallet_files(); + std::string get_appconfig(const std::string& encryption_key); + std::string set_appconfig(const std::string& conf_str, const std::string& encryption_key); + std::string generate_random_key(uint64_t lenght); + std::string get_logs_buffer(); + std::string truncate_log(); + std::string open(const std::string& path, const std::string& password); std::string restore(const std::string& seed, const std::string& path, const std::string& password); std::string generate(const std::string& path, const std::string& password); diff --git a/src/wallet/view_iface.h b/src/wallet/view_iface.h index 41d64ced..5d6f4aea 100644 --- a/src/wallet/view_iface.h +++ b/src/wallet/view_iface.h @@ -139,6 +139,8 @@ public: uint64_t last_build_displaymode; uint64_t alias_count; std::string last_build_available; + uint64_t downloaded_bytes; + uint64_t download_total_data_size; //std::list last_blocks; bool is_pos_allowed; uint64_t expiration_median_timestamp; @@ -160,6 +162,8 @@ public: KV_SERIALIZE(last_build_available) //KV_SERIALIZE(last_blocks) KV_SERIALIZE(alias_count) + KV_SERIALIZE(downloaded_bytes) + KV_SERIALIZE(download_total_data_size) KV_SERIALIZE(is_pos_allowed) KV_SERIALIZE(expiration_median_timestamp) KV_SERIALIZE(is_disconnected) @@ -775,6 +779,7 @@ public: #define API_RETURN_CODE_CORE_BUSY "CORE_BUSY" #define API_RETURN_CODE_OVERFLOW "OVERFLOW" #define API_RETURN_CODE_BUSY "BUSY" +#define API_RETURN_CODE_INVALID_FILE "INVALID_FILE" #define API_MAX_ALIASES_COUNT 10000 diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 229181dd..08c5d8ec 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1356,7 +1356,7 @@ void wallet2::scan_tx_pool(bool& has_related_alias_in_unconfirmed) std::unordered_map> unconfirmed_multisig_transfers_from_tx_pool; has_related_alias_in_unconfirmed = false; - uint64_t tx_expiration_ts_median = get_tx_expiration_median(); + uint64_t tx_expiration_ts_median = res.tx_expiration_ts_median; //get_tx_expiration_median(); for (const auto &tx_blob : res.txs) { currency::transaction tx; @@ -2208,7 +2208,7 @@ void wallet2::store_watch_only(const std::wstring& path_to_save, const std::stri continue; const crypto::public_key& out_key = boost::get(out_t).key; wo.m_pending_key_images.insert(std::make_pair(out_key, td.m_key_image)); - wo.m_pending_key_images_file_container.push_back(tools::out_key_to_ki(out_key, td.m_key_image)); + wo.m_pending_key_images_file_container.push_back(tools::out_key_to_ki{ out_key, td.m_key_image }); WLT_LOG_L1("preparing watch-only wallet: added pending ki (" << out_key << ", " << td.m_key_image << ")"); } @@ -2552,7 +2552,7 @@ void wallet2::submit_transfer(const std::string& signed_tx_blob, currency::trans else { m_pending_key_images[p.first] = p.second; - m_pending_key_images_file_container.push_back(tools::out_key_to_ki(p.first, p.second)); + m_pending_key_images_file_container.push_back(tools::out_key_to_ki{ p.first, p.second }); LOG_PRINT_L2("for tx " << tx_hash << " pending key image added (" << p.first << ", " << p.second << ")"); } } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index cdd64fd6..2cc41907 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -123,8 +123,6 @@ namespace tools #pragma pack(push, 1) struct out_key_to_ki { - out_key_to_ki() {} - out_key_to_ki(const crypto::public_key& out_key, const crypto::key_image& key_image) : out_key(out_key), key_image(key_image) {} crypto::public_key out_key; crypto::key_image key_image; }; diff --git a/src/wallet/wallets_manager.cpp b/src/wallet/wallets_manager.cpp index 06b7a3c6..ef3e6716 100644 --- a/src/wallet/wallets_manager.cpp +++ b/src/wallet/wallets_manager.cpp @@ -14,6 +14,8 @@ #include "string_coding.h" #include "wallet_helpers.h" #include "core_default_rpc_proxy.h" +#include "common/db_backend_selector.h" +#include "common/pre_download.h" #define GET_WALLET_OPT_BY_ID(wallet_id, name) \ CRITICAL_REGION_LOCAL(m_wallets_lock); \ @@ -29,7 +31,9 @@ if (it == m_wallets.end()) \ return API_RETURN_CODE_WALLET_WRONG_ID; \ auto& name = it->second.w; -#define DAEMON_IDLE_UPDATE_TIME_MS 1000 +#define DAEMON_IDLE_UPDATE_TIME_MS 2000 +#define HTTP_PROXY_TIMEOUT 2000 +#define HTTP_PROXY_ATTEMPTS_COUNT 1 wallets_manager::wallets_manager():m_pview(&m_view_stub), m_stop_singal_sent(false), @@ -65,6 +69,9 @@ const command_line::arg_descriptor arg_enable_gui_debug_mode = { "gui-debu const command_line::arg_descriptor arg_qt_remote_debugging_port = { "remote-debugging-port", "Specify port for Qt remote debugging", 30333, true }; const command_line::arg_descriptor arg_remote_node = { "remote-node", "Switch GUI to work with remote node instead of local daemon", "", true }; const command_line::arg_descriptor arg_enable_qt_logs = { "enable-qt-logs", "Forward Qt log messages into main log", false, true }; +const command_line::arg_descriptor arg_disable_logs_init("disable-logs-init", "Disable log initialization in GUI"); +//const command_line::arg_descriptor arg_disable_logs_init = { "disable-logs-init", "Disable log initialization in GUI" }; + void wallet_lock_time_watching_policy::watch_lock_time(uint64_t lock_time) { @@ -98,10 +105,6 @@ bool wallets_manager::init(int argc, char* argv[], view::i_view* pview_handler) dsi.pos_difficulty = dsi.pow_difficulty = "---"; m_pview->update_daemon_status(dsi); - log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); - log_space::get_set_need_thread_id(true, true); - log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,p2p,wallet"); - tools::signal_handler::install_fatal([](int sig_number, void* address) { LOG_ERROR("\n\nFATAL ERROR\nsig: " << sig_number << ", address: " << address); std::fflush(nullptr); // all open output streams are flushed @@ -140,6 +143,12 @@ bool wallets_manager::init(int argc, char* argv[], view::i_view* pview_handler) command_line::add_arg(desc_cmd_sett, arg_qt_remote_debugging_port); command_line::add_arg(desc_cmd_sett, arg_remote_node); command_line::add_arg(desc_cmd_sett, arg_enable_qt_logs); + command_line::add_arg(desc_cmd_sett, arg_disable_logs_init); + command_line::add_arg(desc_cmd_sett, command_line::arg_no_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_force_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_validate_predownload); + command_line::add_arg(desc_cmd_sett, command_line::arg_predownload_link); + #ifndef MOBILE_WALLET_BUILD currency::core::init_options(desc_cmd_sett); @@ -147,6 +156,7 @@ bool wallets_manager::init(int argc, char* argv[], view::i_view* pview_handler) nodetool::node_server >::init_options(desc_cmd_sett); currency::miner::init_options(desc_cmd_sett); bc_services::bc_offers_service::init_options(desc_cmd_sett); + tools::db::db_backend_selector::init_options(desc_cmd_sett); #endif po::options_description desc_options("Allowed options"); @@ -225,19 +235,23 @@ bool wallets_manager::init(int argc, char* argv[], view::i_view* pview_handler) path_to_html = command_line::get_arg(m_vm, arg_html_folder); } - log_space::log_singletone::add_logger(LOGGER_FILE, log_file_name.c_str(), log_dir.c_str()); - LOG_PRINT_L0(CURRENCY_NAME << " v" << PROJECT_VERSION_LONG); - LOG_PRINT("Module folder: " << argv[0], LOG_LEVEL_0); - if (command_line::has_arg(m_vm, arg_remote_node)) { m_remote_node_mode = true; auto proxy_ptr = new tools::default_http_core_proxy(); proxy_ptr->set_plast_daemon_is_disconnected(&m_last_daemon_is_disconnected); + proxy_ptr->set_connectivity(HTTP_PROXY_TIMEOUT, HTTP_PROXY_ATTEMPTS_COUNT); m_rpc_proxy.reset(proxy_ptr); m_rpc_proxy->set_connection_addr(command_line::get_arg(m_vm, arg_remote_node)); } + if(!command_line::has_arg(m_vm, arg_disable_logs_init)) + { + log_space::log_singletone::add_logger(LOGGER_FILE, log_file_name.c_str(), log_dir.c_str()); + LOG_PRINT_L0(CURRENCY_NAME << " v" << PROJECT_VERSION_LONG); + LOG_PRINT("Module folder: " << argv[0], LOG_LEVEL_0); + } + m_qt_logs_enbaled = command_line::get_arg(m_vm, arg_enable_qt_logs); m_pview->init(path_to_html); @@ -309,12 +323,42 @@ bool wallets_manager::init_local_daemon() dsi.daemon_network_state = currency::COMMAND_RPC_GET_INFO::daemon_network_state_loading_core; m_pview->update_daemon_status(dsi); + // pre-downloading handling + tools::db::db_backend_selector dbbs; + bool res = dbbs.init(m_vm); + CHECK_AND_ASSERT_AND_SET_GUI(res, "Failed to initialize db_backend_selector"); + if (!command_line::has_arg(m_vm, command_line::arg_no_predownload) || command_line::has_arg(m_vm, command_line::arg_force_predownload)) + { + auto last_update = std::chrono::system_clock::now(); + bool r = tools::process_predownload(m_vm, [&](uint64_t total_bytes, uint64_t received_bytes){ + auto dif = std::chrono::system_clock::now() - last_update; + if (dif > std::chrono::milliseconds(300)) + { + dsi.download_total_data_size = total_bytes; + dsi.downloaded_bytes = received_bytes; + dsi.daemon_network_state = currency::COMMAND_RPC_GET_INFO::daemon_network_state_downloading_database; + m_pview->update_daemon_status(dsi); + last_update = std::chrono::system_clock::now(); + } + + return static_cast(m_stop_singal_sent); + }); + /*if (m_stop_singal_sent) + { + dsi.daemon_network_state = currency::COMMAND_RPC_GET_INFO::daemon_network_state_deintializing; + m_pview->update_daemon_status(dsi); + }*/ + } + + + //initialize core here LOG_PRINT_L0("Initializing core..."); + dsi.daemon_network_state = currency::COMMAND_RPC_GET_INFO::daemon_network_state_loading_core; //dsi.text_state = "Initializing core"; m_pview->update_daemon_status(dsi); - bool res = m_ccore.init(m_vm); + res = m_ccore.init(m_vm); CHECK_AND_ASSERT_AND_SET_GUI(res, "Failed to initialize core"); LOG_PRINT_L0("Core initialized OK"); @@ -490,7 +534,7 @@ void wallets_manager::main_worker(const po::variables_map& m_vm) } m_pview->on_backend_stopped(); - CATCH_ENTRY_L0("daemon_backend::main_worker", void()); + CATCH_ENTRY_L0("wallets_manager::main_worker", void()); } bool wallets_manager::update_state_info() @@ -505,10 +549,9 @@ bool wallets_manager::update_state_info() dsi.is_disconnected = true; m_last_daemon_network_state = dsi.daemon_network_state; m_pview->update_daemon_status(dsi); - LOG_ERROR("Failed to call get_info"); + LOG_PRINT_RED_L0("Failed to call get_info"); return false; } - m_is_pos_allowed = inf.pos_allowed; dsi.alias_count = inf.alias_count; dsi.pow_difficulty = std::to_string(inf.pow_difficulty); diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 6acc5d5b..68ff8c68 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -13,6 +13,7 @@ #include "currency_core/bc_offers_service.h" #include "random_helper.h" #include "core_state_helper.h" +#include "common/db_backend_selector.h" #define TX_BLOBSIZE_CHECKER_LOG_FILENAME "get_object_blobsize(tx).log" @@ -666,6 +667,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_options, arg_enable_debug_asserts); command_line::add_arg(desc_options, command_line::arg_data_dir, std::string(".")); currency::core::init_options(desc_options); + tools::db::db_backend_selector::init_options(desc_options); bool r = command_line::handle_error_helper(desc_options, [&]() { diff --git a/tests/functional_tests/plain_wallet_tests.cpp b/tests/functional_tests/plain_wallet_tests.cpp index 0b626ea3..00f1047d 100644 --- a/tests/functional_tests/plain_wallet_tests.cpp +++ b/tests/functional_tests/plain_wallet_tests.cpp @@ -30,77 +30,92 @@ struct try_pull_result_open_response void run_plain_wallet_api_test() { LOG_PRINT_L0("Creating instance..."); - std::string s = plain_wallet::init("195.201.107.230", "11211"); + std::string s = plain_wallet::init("195.201.107.230", "11211", 1); - LOG_PRINT_L0("Generating wallet..."); - view::open_wallet_request owr = AUTO_VAL_INIT(owr); - owr.path = "E:\\tmp\\zano_testwallet_745ss65030.zan"; - owr.pass = ""; - std::string job_id_str = plain_wallet::async_call("open", 0, epee::serialization::store_t_to_json(owr)); - - - try_pull_result_open_response rsp = AUTO_VAL_INIT(rsp); - - while (true) + std::string key = plain_wallet::generate_random_key(10); + std::string test_data = "1234567890 test test "; + std::string res = plain_wallet::set_appconfig(test_data, key); + std::string test_data2 = plain_wallet::get_appconfig(key); + if (test_data2 != test_data) { - std::string res = plain_wallet::try_pull_result(1); - LOG_PRINT_L0("[try_pull_result] RESPONSE:" << ENDL << res); - - if (!epee::serialization::load_t_from_json(rsp, res)) - { - LOG_ERROR("Failed to parse try_pull_result response: " << res); - return; - } - epee::misc_utils::sleep_no_w(1000); - if(!rsp.delivered) - continue; - break; + LOG_ERROR("Error"); } + return; + //std::string fres = plain_wallet::get_logs_buffer(); + //std::string fres2 = plain_wallet::truncate_log(); + //std::string fres3 = plain_wallet::get_logs_buffer(); +// LOG_PRINT_L0("Generating wallet..."); +// view::open_wallet_request owr = AUTO_VAL_INIT(owr); +// owr.path = "E:\\tmp\\zano_testwallet_745ss65030.zan"; +// owr.pass = ""; +// std::string job_id_str = plain_wallet::async_call("open", 0, epee::serialization::store_t_to_json(owr)); +// +// +// try_pull_result_open_response rsp = AUTO_VAL_INIT(rsp); +// +// while (true) +// { +// std::string res = plain_wallet::try_pull_result(1); +// LOG_PRINT_L0("[try_pull_result] RESPONSE:" << ENDL << res); +// +// if (!epee::serialization::load_t_from_json(rsp, res)) +// { +// LOG_ERROR("Failed to parse try_pull_result response: " << res); +// return; +// } +// epee::misc_utils::sleep_no_w(1000); +// if(!rsp.delivered) +// continue; +// break; +// } +// +// + //std::string rsp = plain_wallet::open(std::string("E:\\tmp\\zano_testwallet_745ss65030.zan"), ""); //LOG_PRINT_L0("RESPONSE:" << ENDL << rsp); //epee::json_rpc::response ok_response = AUTO_VAL_INIT(ok_response); //epee::serialization::load_t_from_json(ok_response, rsp); - size_t count = 0; - while (count < 10) - { - std::string prog = plain_wallet::get_wallet_status(rsp.result.result.wallet_id); - LOG_PRINT_L0("Progress: " << ENDL << prog); - view::wallet_sync_status_info wsi = AUTO_VAL_INIT(wsi); - if (!epee::serialization::load_t_from_json(wsi, prog)) - { - LOG_ERROR("Failed to get_wallet_status()"); - return; - } - if (!wsi.is_in_long_refresh) - break; - epee::misc_utils::sleep_no_w(1000); - } - - std::string job_id_str2 = plain_wallet::async_call("close", rsp.result.result.wallet_id, ""); - try_pull_result_open_response rsp2 = AUTO_VAL_INIT(rsp2); - - while (true) - { - std::string res = plain_wallet::try_pull_result(2); - LOG_PRINT_L0("[try_pull_result] RESPONSE:" << ENDL << res); - - if (!epee::serialization::load_t_from_json(rsp2, res)) - { - LOG_ERROR("Failed to parse try_pull_result response: " << res); - return; - } - epee::misc_utils::sleep_no_w(1000); - if (!rsp2.delivered) - continue; - break; - } - - - LOG_PRINT_L0("OK"); +// size_t count = 0; +// while (count < 10) +// { +// std::string prog = plain_wallet::get_wallet_status(rsp.result.result.wallet_id); +// LOG_PRINT_L0("Progress: " << ENDL << prog); +// view::wallet_sync_status_info wsi = AUTO_VAL_INIT(wsi); +// if (!epee::serialization::load_t_from_json(wsi, prog)) +// { +// LOG_ERROR("Failed to get_wallet_status()"); +// return; +// } +// if (!wsi.is_in_long_refresh) +// break; +// epee::misc_utils::sleep_no_w(1000); +// } +// +// std::string job_id_str2 = plain_wallet::async_call("close", rsp.result.result.wallet_id, ""); +// try_pull_result_open_response rsp2 = AUTO_VAL_INIT(rsp2); +// +// while (true) +// { +// std::string res = plain_wallet::try_pull_result(2); +// LOG_PRINT_L0("[try_pull_result] RESPONSE:" << ENDL << res); +// +// if (!epee::serialization::load_t_from_json(rsp2, res)) +// { +// LOG_ERROR("Failed to parse try_pull_result response: " << res); +// return; +// } +// epee::misc_utils::sleep_no_w(1000); +// if (!rsp2.delivered) +// continue; +// break; +// } +// +// +// LOG_PRINT_L0("OK"); }