From 7bb3c766f9f92d1a5643b7f355faa2d8b76fcd52 Mon Sep 17 00:00:00 2001 From: sowle Date: Mon, 9 Mar 2020 23:50:55 +0300 Subject: [PATCH] epee: gzip_encoder_lambda, gzip_decoder_lambda --- contrib/epee/include/gzip_encoding.h | 462 +++++++++++++++++---------- 1 file changed, 299 insertions(+), 163 deletions(-) diff --git a/contrib/epee/include/gzip_encoding.h b/contrib/epee/include/gzip_encoding.h index 2be51e77..d4fb7e50 100644 --- a/contrib/epee/include/gzip_encoding.h +++ b/contrib/epee/include/gzip_encoding.h @@ -31,7 +31,6 @@ #define _GZIP_ENCODING_H_ #include "net/http_client_base.h" #include "zlib/zlib.h" -//#include "http.h" namespace epee @@ -41,186 +40,323 @@ namespace net_utils - class content_encoding_gzip: public i_sub_handler - { - public: - /*! \brief - * Function content_encoding_gzip : Constructor - * - */ - inline - content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false):m_powner_filter(powner_filter), - m_is_stream_ended(false), - m_is_deflate_mode(is_deflate_mode), - m_is_first_update_in(true) - { - memset(&m_zstream_in, 0, sizeof(m_zstream_in)); - memset(&m_zstream_out, 0, sizeof(m_zstream_out)); - int ret = 0; - if(is_deflate_mode) - { - ret = inflateInit(&m_zstream_in); - ret = deflateInit(&m_zstream_out, Z_DEFAULT_COMPRESSION); - }else - { - ret = inflateInit2(&m_zstream_in, 0x1F); - ret = deflateInit2(&m_zstream_out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY); - } - } - /*! \brief - * Function content_encoding_gzip : Destructor - * - */ - inline - ~content_encoding_gzip() - { - inflateEnd(& m_zstream_in ); - deflateEnd(& m_zstream_out ); - } - /*! \brief - * Function update_in : Entry point for income data - * - */ - inline - virtual bool update_in( std::string& piece_of_transfer) - { + class content_encoding_gzip: public i_sub_handler + { + public: + /*! \brief + * Function content_encoding_gzip : Constructor + * + */ + inline + content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false):m_powner_filter(powner_filter), + m_is_stream_ended(false), + m_is_deflate_mode(is_deflate_mode), + m_is_first_update_in(true) + { + memset(&m_zstream_in, 0, sizeof(m_zstream_in)); + memset(&m_zstream_out, 0, sizeof(m_zstream_out)); + int ret = 0; + if(is_deflate_mode) + { + ret = inflateInit(&m_zstream_in); + ret = deflateInit(&m_zstream_out, Z_DEFAULT_COMPRESSION); + }else + { + ret = inflateInit2(&m_zstream_in, 0x1F); + ret = deflateInit2(&m_zstream_out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY); + } + } + /*! \brief + * Function content_encoding_gzip : Destructor + * + */ + inline + ~content_encoding_gzip() + { + inflateEnd(& m_zstream_in ); + deflateEnd(& m_zstream_out ); + } + /*! \brief + * Function update_in : Entry point for income data + * + */ + inline + virtual bool update_in( std::string& piece_of_transfer) + { - bool is_first_time_here = m_is_first_update_in; - m_is_first_update_in = false; + bool is_first_time_here = m_is_first_update_in; + m_is_first_update_in = false; - if(m_pre_decode.size()) - m_pre_decode += piece_of_transfer; - else - m_pre_decode.swap(piece_of_transfer); - piece_of_transfer.clear(); + if(m_pre_decode.size()) + m_pre_decode += piece_of_transfer; + else + m_pre_decode.swap(piece_of_transfer); + piece_of_transfer.clear(); - std::string decode_summary_buff; + std::string decode_summary_buff; - size_t ungzip_size = m_pre_decode.size() * 0x30; - std::string current_decode_buff(ungzip_size, 'X'); + size_t ungzip_size = m_pre_decode.size() * 0x30; + std::string current_decode_buff(ungzip_size, 'X'); - //Here the cycle is introduced where we unpack the buffer, the cycle is required - //because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error - bool continue_unpacking = true; - bool first_step = true; - while(m_pre_decode.size() && continue_unpacking) - { + //Here the cycle is introduced where we unpack the buffer, the cycle is required + //because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error + bool continue_unpacking = true; + bool first_step = true; + while(m_pre_decode.size() && continue_unpacking) + { - //fill buffers - m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); - m_zstream_in.avail_in = (uInt)m_pre_decode.size(); - m_zstream_in.next_out = (Bytef*)current_decode_buff.data(); - m_zstream_in.avail_out = (uInt)ungzip_size; + //fill buffers + m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); + m_zstream_in.avail_in = (uInt)m_pre_decode.size(); + m_zstream_in.next_out = (Bytef*)current_decode_buff.data(); + m_zstream_in.avail_out = (uInt)ungzip_size; - int flag = Z_SYNC_FLUSH; - int ret = inflate(&m_zstream_in, flag); - CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. err = " << ret); + int flag = Z_SYNC_FLUSH; + int ret = inflate(&m_zstream_in, flag); + CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. err = " << ret); - if(Z_STREAM_END == ret) - m_is_stream_ended = true; - else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step) - { - // some servers (notably Apache with mod_deflate) don't generate zlib headers - // insert a dummy header and try again - static char dummy_head[2] = - { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, - }; - inflateReset(&m_zstream_in); - m_zstream_in.next_in = (Bytef*) dummy_head; - m_zstream_in.avail_in = sizeof(dummy_head); + if(Z_STREAM_END == ret) + m_is_stream_ended = true; + else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step) + { + // some servers (notably Apache with mod_deflate) don't generate zlib headers + // insert a dummy header and try again + static char dummy_head[2] = + { + 0x8 + 0x7 * 0x10, + (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, + }; + inflateReset(&m_zstream_in); + m_zstream_in.next_in = (Bytef*) dummy_head; + m_zstream_in.avail_in = sizeof(dummy_head); - ret = inflate(&m_zstream_in, Z_NO_FLUSH); - if (ret != Z_OK) - { - LOCAL_ASSERT(0); - m_pre_decode.swap(piece_of_transfer); - return false; - } - m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); - m_zstream_in.avail_in = (uInt)m_pre_decode.size(); + ret = inflate(&m_zstream_in, Z_NO_FLUSH); + if (ret != Z_OK) + { + LOCAL_ASSERT(0); + m_pre_decode.swap(piece_of_transfer); + return false; + } + m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); + m_zstream_in.avail_in = (uInt)m_pre_decode.size(); - ret = inflate(&m_zstream_in, Z_NO_FLUSH); - if (ret != Z_OK) - { - LOCAL_ASSERT(0); - m_pre_decode.swap(piece_of_transfer); - return false; - } - } + ret = inflate(&m_zstream_in, Z_NO_FLUSH); + if (ret != Z_OK) + { + LOCAL_ASSERT(0); + m_pre_decode.swap(piece_of_transfer); + return false; + } + } - //leave only unpacked part in the output buffer to start with it the next time - m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in); - //if decoder gave nothing to return, then everything is ahead, now simply break - if(ungzip_size == m_zstream_in.avail_out) - break; + //leave only unpacked part in the output buffer to start with it the next time + m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in); + //if decoder gave nothing to return, then everything is ahead, now simply break + if(ungzip_size == m_zstream_in.avail_out) + break; - //decode_buff currently stores data parts that were unpacked, fix this size - current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out); - if(decode_summary_buff.size()) - decode_summary_buff += current_decode_buff; - else - current_decode_buff.swap(decode_summary_buff); + //decode_buff currently stores data parts that were unpacked, fix this size + current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out); + if(decode_summary_buff.size()) + decode_summary_buff += current_decode_buff; + else + current_decode_buff.swap(decode_summary_buff); - current_decode_buff.resize(ungzip_size); - first_step = false; - } + current_decode_buff.resize(ungzip_size); + first_step = false; + } - //Process these data if required - bool res = true; + //Process these data if required + bool res = true; - res = m_powner_filter->handle_target_data(decode_summary_buff); + res = m_powner_filter->handle_target_data(decode_summary_buff); - return true; + return true; - } - /*! \brief - * Function stop : Entry point for stop signal and flushing cached data buffer. - * - */ - inline - virtual void stop(std::string& OUT collect_remains) - { - } - protected: - private: - /*! \brief - * Pointer to parent HTTP-parser - */ - i_target_handler* m_powner_filter; - /*! \brief - * ZLIB object for income stream - */ - z_stream m_zstream_in; - /*! \brief - * ZLIB object for outcome stream - */ - z_stream m_zstream_out; - /*! \brief - * Data that could not be unpacked immediately, left to wait for the next packet of data - */ - std::string m_pre_decode; - /*! \brief - * The data are accumulated for a package in the buffer to send the web client - */ - std::string m_pre_encode; - /*! \brief - * Signals that stream looks like ended - */ - bool m_is_stream_ended; - /*! \brief - * If this flag is set, income data is in HTTP-deflate mode - */ - bool m_is_deflate_mode; - /*! \brief - * Marks that it is a first data packet - */ - bool m_is_first_update_in; - }; -} -} + } + /*! \brief + * Function stop : Entry point for stop signal and flushing cached data buffer. + * + */ + inline + virtual void stop(std::string& OUT collect_remains) + { + } + protected: + private: + /*! \brief + * Pointer to parent HTTP-parser + */ + i_target_handler* m_powner_filter; + /*! \brief + * ZLIB object for income stream + */ + z_stream m_zstream_in; + /*! \brief + * ZLIB object for outcome stream + */ + z_stream m_zstream_out; + /*! \brief + * Data that could not be unpacked immediately, left to wait for the next packet of data + */ + std::string m_pre_decode; + /*! \brief + * The data are accumulated for a package in the buffer to send the web client + */ + std::string m_pre_encode; + /*! \brief + * Signals that stream looks like ended + */ + bool m_is_stream_ended; + /*! \brief + * If this flag is set, income data is in HTTP-deflate mode + */ + bool m_is_deflate_mode; + /*! \brief + * Marks that it is a first data packet + */ + bool m_is_first_update_in; + }; // class content_encoding_gzip + + struct abstract_callback_base + { + virtual bool do_call(const std::string& piece_of_transfer) = 0; + }; + + template + struct abstract_callback : public abstract_callback_base + { + callback_t m_cb; + + abstract_callback(callback_t cb) : m_cb(cb){} + virtual bool do_call(const std::string& piece_of_transfer) + { + return m_cb(piece_of_transfer); + } + }; + + class gzip_decoder_lambda : public content_encoding_gzip, + public i_target_handler + { + std::shared_ptr m_pcb; + + virtual bool handle_target_data(std::string& piece_of_transfer) + { + bool r = m_pcb->do_call(piece_of_transfer); + piece_of_transfer.clear(); + return r; + } + public: + gzip_decoder_lambda() : content_encoding_gzip(this, true, Z_BEST_COMPRESSION) + {} + + template + bool update_in(std::string& piece_of_transfer, callback_t cb) + { + m_pcb.reset(new abstract_callback(cb)); + return content_encoding_gzip::update_in(piece_of_transfer); + } + template + bool stop(callback_t cb) + {return true;} + }; // class gzip_decoder_lambda + + class gzip_encoder_lyambda + { + bool m_initialized; + z_stream m_zstream; + public: + gzip_encoder_lyambda(int compression_level = Z_DEFAULT_COMPRESSION) :m_initialized(false), m_zstream(AUTO_VAL_INIT(m_zstream)) + { + int ret = deflateInit(&m_zstream, compression_level); + if (ret == Z_OK) + m_initialized = true; + } + + ~gzip_encoder_lyambda() + { + deflateEnd(&m_zstream); + } + + template + bool update_in(const std::string& target, callback_t cb) + { + if (!m_initialized) + { + return false; + } + + if (!target.size()) + { + return true; + } + + std::string result_packed_buff; + //theoretically it supposed to be smaller + result_packed_buff.resize(target.size(), 'X'); + while (true) + { + m_zstream.next_in = (Bytef*)target.data(); + m_zstream.avail_in = (uInt)target.size(); + m_zstream.next_out = (Bytef*)result_packed_buff.data(); + m_zstream.avail_out = (uInt)result_packed_buff.size(); + + int ret = deflate(&m_zstream, Z_NO_FLUSH); + CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate. err = " << ret); + if (m_zstream.avail_out == 0) + { + //twice bigger buffer + result_packed_buff.resize(result_packed_buff.size()*2); + continue; + } + if (result_packed_buff.size() != m_zstream.avail_out) + result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out); + break; + } + + return cb(result_packed_buff); + } + + template + bool stop(callback_t cb) + { + if (!m_initialized) + { + return false; + } + + std::string result_packed_buff; + //theoretically it supposed to be smaller + result_packed_buff.resize(1000000, 'X'); + while (true) + { + m_zstream.next_in = nullptr; + m_zstream.avail_in = 0; + m_zstream.next_out = (Bytef*)result_packed_buff.data(); + m_zstream.avail_out = (uInt)result_packed_buff.size(); + + int ret = deflate(&m_zstream, Z_FINISH); + CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate at finish. err = " << ret); + if (ret != Z_STREAM_END) + { + //twice bigger buffer + result_packed_buff.resize(result_packed_buff.size() * 2); + continue; + } + if (result_packed_buff.size() != m_zstream.avail_out) + result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out); + m_initialized = false; + break; + } + + return cb(result_packed_buff); + } + + }; // class gzip_encoder_lyambda + +} // namespace net_utils +} // namespace epee