lthn/blockchain
6
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

epee: gzip_encoder_lambda, gzip_decoder_lambda

Browse source
This commit is contained in:
sowle 2020-03-09 23:50:55 +03:00
parent b034328a9f
commit 7bb3c766f9
No known key found for this signature in database
GPG key ID: C07A24B2D89D49FC
1 changed files with 299 additions and 163 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

462
contrib/epee/include/gzip_encoding.h
View file

@ -31,7 +31,6 @@
#define _GZIP_ENCODING_H_ #define _GZIP_ENCODING_H_
#include "net/http_client_base.h" #include "net/http_client_base.h"
#include "zlib/zlib.h" #include "zlib/zlib.h"
//#include "http.h"
namespace epee namespace epee
@ -41,186 +40,323 @@ namespace net_utils
class content_encoding_gzip: public i_sub_handler class content_encoding_gzip: public i_sub_handler
{ {
public: public:
/*! \brief /*! \brief
* Function content_encoding_gzip : Constructor * Function content_encoding_gzip : Constructor
* *
*/ */
inline inline
content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false):m_powner_filter(powner_filter), content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false):m_powner_filter(powner_filter),
m_is_stream_ended(false), m_is_stream_ended(false),
m_is_deflate_mode(is_deflate_mode), m_is_deflate_mode(is_deflate_mode),
m_is_first_update_in(true) m_is_first_update_in(true)
{ {
memset(&m_zstream_in, 0, sizeof(m_zstream_in)); memset(&m_zstream_in, 0, sizeof(m_zstream_in));
memset(&m_zstream_out, 0, sizeof(m_zstream_out)); memset(&m_zstream_out, 0, sizeof(m_zstream_out));
int ret = 0; int ret = 0;
if(is_deflate_mode) if(is_deflate_mode)
{ {
ret = inflateInit(&m_zstream_in); ret = inflateInit(&m_zstream_in);
ret = deflateInit(&m_zstream_out, Z_DEFAULT_COMPRESSION); ret = deflateInit(&m_zstream_out, Z_DEFAULT_COMPRESSION);
}else }else
{ {
ret = inflateInit2(&m_zstream_in, 0x1F); ret = inflateInit2(&m_zstream_in, 0x1F);
ret = deflateInit2(&m_zstream_out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY); ret = deflateInit2(&m_zstream_out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY);
} }
} }
/*! \brief /*! \brief
* Function content_encoding_gzip : Destructor * Function content_encoding_gzip : Destructor
* *
*/ */
inline inline
~content_encoding_gzip() ~content_encoding_gzip()
{ {
inflateEnd(& m_zstream_in ); inflateEnd(& m_zstream_in );
deflateEnd(& m_zstream_out ); deflateEnd(& m_zstream_out );
} }
/*! \brief /*! \brief
* Function update_in : Entry point for income data * Function update_in : Entry point for income data
* *
*/ */
inline inline
virtual bool update_in( std::string& piece_of_transfer) virtual bool update_in( std::string& piece_of_transfer)
{ {
bool is_first_time_here = m_is_first_update_in; bool is_first_time_here = m_is_first_update_in;
m_is_first_update_in = false; m_is_first_update_in = false;
if(m_pre_decode.size()) if(m_pre_decode.size())
m_pre_decode += piece_of_transfer; m_pre_decode += piece_of_transfer;
else else
m_pre_decode.swap(piece_of_transfer); m_pre_decode.swap(piece_of_transfer);
piece_of_transfer.clear(); piece_of_transfer.clear();
std::string decode_summary_buff; std::string decode_summary_buff;
size_t ungzip_size = m_pre_decode.size() * 0x30; size_t ungzip_size = m_pre_decode.size() * 0x30;
std::string current_decode_buff(ungzip_size, 'X'); std::string current_decode_buff(ungzip_size, 'X');
//Here the cycle is introduced where we unpack the buffer, the cycle is required //Here the cycle is introduced where we unpack the buffer, the cycle is required
//because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error //because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error
bool continue_unpacking = true; bool continue_unpacking = true;
bool first_step = true; bool first_step = true;
while(m_pre_decode.size() && continue_unpacking) while(m_pre_decode.size() && continue_unpacking)
{ {
//fill buffers //fill buffers
m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); m_zstream_in.next_in = (Bytef*)m_pre_decode.data();
m_zstream_in.avail_in = (uInt)m_pre_decode.size(); m_zstream_in.avail_in = (uInt)m_pre_decode.size();
m_zstream_in.next_out = (Bytef*)current_decode_buff.data(); m_zstream_in.next_out = (Bytef*)current_decode_buff.data();
m_zstream_in.avail_out = (uInt)ungzip_size; m_zstream_in.avail_out = (uInt)ungzip_size;
int flag = Z_SYNC_FLUSH; int flag = Z_SYNC_FLUSH;
int ret = inflate(&m_zstream_in, flag); int ret = inflate(&m_zstream_in, flag);
CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. err = " << ret); CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. err = " << ret);
if(Z_STREAM_END == ret) if(Z_STREAM_END == ret)
m_is_stream_ended = true; m_is_stream_ended = true;
else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step) else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step)
{ {
// some servers (notably Apache with mod_deflate) don't generate zlib headers // some servers (notably Apache with mod_deflate) don't generate zlib headers
// insert a dummy header and try again // insert a dummy header and try again
static char dummy_head[2] = static char dummy_head[2] =
{ {
0x8 + 0x7 * 0x10, 0x8 + 0x7 * 0x10,
(((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF,
}; };
inflateReset(&m_zstream_in); inflateReset(&m_zstream_in);
m_zstream_in.next_in = (Bytef*) dummy_head; m_zstream_in.next_in = (Bytef*) dummy_head;
m_zstream_in.avail_in = sizeof(dummy_head); m_zstream_in.avail_in = sizeof(dummy_head);
ret = inflate(&m_zstream_in, Z_NO_FLUSH); ret = inflate(&m_zstream_in, Z_NO_FLUSH);
if (ret != Z_OK) if (ret != Z_OK)
{ {
LOCAL_ASSERT(0); LOCAL_ASSERT(0);
m_pre_decode.swap(piece_of_transfer); m_pre_decode.swap(piece_of_transfer);
return false; return false;
} }
m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); m_zstream_in.next_in = (Bytef*)m_pre_decode.data();
m_zstream_in.avail_in = (uInt)m_pre_decode.size(); m_zstream_in.avail_in = (uInt)m_pre_decode.size();
ret = inflate(&m_zstream_in, Z_NO_FLUSH); ret = inflate(&m_zstream_in, Z_NO_FLUSH);
if (ret != Z_OK) if (ret != Z_OK)
{ {
LOCAL_ASSERT(0); LOCAL_ASSERT(0);
m_pre_decode.swap(piece_of_transfer); m_pre_decode.swap(piece_of_transfer);
return false; return false;
} }
} }
//leave only unpacked part in the output buffer to start with it the next time //leave only unpacked part in the output buffer to start with it the next time
m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in); m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in);
//if decoder gave nothing to return, then everything is ahead, now simply break //if decoder gave nothing to return, then everything is ahead, now simply break
if(ungzip_size == m_zstream_in.avail_out) if(ungzip_size == m_zstream_in.avail_out)
break; break;
//decode_buff currently stores data parts that were unpacked, fix this size //decode_buff currently stores data parts that were unpacked, fix this size
current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out); current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out);
if(decode_summary_buff.size()) if(decode_summary_buff.size())
decode_summary_buff += current_decode_buff; decode_summary_buff += current_decode_buff;
else else
current_decode_buff.swap(decode_summary_buff); current_decode_buff.swap(decode_summary_buff);
current_decode_buff.resize(ungzip_size); current_decode_buff.resize(ungzip_size);
first_step = false; first_step = false;
} }
//Process these data if required //Process these data if required
bool res = true; bool res = true;
res = m_powner_filter->handle_target_data(decode_summary_buff); res = m_powner_filter->handle_target_data(decode_summary_buff);
return true; return true;
} }
/*! \brief /*! \brief
* Function stop : Entry point for stop signal and flushing cached data buffer. * Function stop : Entry point for stop signal and flushing cached data buffer.
* *
*/ */
inline inline
virtual void stop(std::string& OUT collect_remains) virtual void stop(std::string& OUT collect_remains)
{ {
} }
protected: protected:
private: private:
/*! \brief /*! \brief
* Pointer to parent HTTP-parser * Pointer to parent HTTP-parser
*/ */
i_target_handler* m_powner_filter; i_target_handler* m_powner_filter;
/*! \brief /*! \brief
* ZLIB object for income stream * ZLIB object for income stream
*/ */
z_stream m_zstream_in; z_stream m_zstream_in;
/*! \brief /*! \brief
* ZLIB object for outcome stream * ZLIB object for outcome stream
*/ */
z_stream m_zstream_out; z_stream m_zstream_out;
/*! \brief /*! \brief
* Data that could not be unpacked immediately, left to wait for the next packet of data * Data that could not be unpacked immediately, left to wait for the next packet of data
*/ */
std::string m_pre_decode; std::string m_pre_decode;
/*! \brief /*! \brief
* The data are accumulated for a package in the buffer to send the web client * The data are accumulated for a package in the buffer to send the web client
*/ */
std::string m_pre_encode; std::string m_pre_encode;
/*! \brief /*! \brief
* Signals that stream looks like ended * Signals that stream looks like ended
*/ */
bool m_is_stream_ended; bool m_is_stream_ended;
/*! \brief /*! \brief
* If this flag is set, income data is in HTTP-deflate mode * If this flag is set, income data is in HTTP-deflate mode
*/ */
bool m_is_deflate_mode; bool m_is_deflate_mode;
/*! \brief /*! \brief
* Marks that it is a first data packet * Marks that it is a first data packet
*/ */
bool m_is_first_update_in; bool m_is_first_update_in;
}; }; // class content_encoding_gzip
}
} struct abstract_callback_base
{
virtual bool do_call(const std::string& piece_of_transfer) = 0;
};
template <typename callback_t>
struct abstract_callback : public abstract_callback_base
{
callback_t m_cb;
abstract_callback(callback_t cb) : m_cb(cb){}
virtual bool do_call(const std::string& piece_of_transfer)
{
return m_cb(piece_of_transfer);
}
};
class gzip_decoder_lambda : public content_encoding_gzip,
public i_target_handler
{
std::shared_ptr<abstract_callback_base> m_pcb;
virtual bool handle_target_data(std::string& piece_of_transfer)
{
bool r = m_pcb->do_call(piece_of_transfer);
piece_of_transfer.clear();
return r;
}
public:
gzip_decoder_lambda() : content_encoding_gzip(this, true, Z_BEST_COMPRESSION)
{}
template<class callback_t>
bool update_in(std::string& piece_of_transfer, callback_t cb)
{
m_pcb.reset(new abstract_callback<callback_t>(cb));
return content_encoding_gzip::update_in(piece_of_transfer);
}
template<class callback_t>
bool stop(callback_t cb)
{return true;}
}; // class gzip_decoder_lambda
class gzip_encoder_lyambda
{
bool m_initialized;
z_stream m_zstream;
public:
gzip_encoder_lyambda(int compression_level = Z_DEFAULT_COMPRESSION) :m_initialized(false), m_zstream(AUTO_VAL_INIT(m_zstream))
{
int ret = deflateInit(&m_zstream, compression_level);
if (ret == Z_OK)
m_initialized = true;
}
~gzip_encoder_lyambda()
{
deflateEnd(&m_zstream);
}
template<typename callback_t>
bool update_in(const std::string& target, callback_t cb)
{
if (!m_initialized)
{
return false;
}
if (!target.size())
{
return true;
}
std::string result_packed_buff;
//theoretically it supposed to be smaller
result_packed_buff.resize(target.size(), 'X');
while (true)
{
m_zstream.next_in = (Bytef*)target.data();
m_zstream.avail_in = (uInt)target.size();
m_zstream.next_out = (Bytef*)result_packed_buff.data();
m_zstream.avail_out = (uInt)result_packed_buff.size();
int ret = deflate(&m_zstream, Z_NO_FLUSH);
CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate. err = " << ret);
if (m_zstream.avail_out == 0)
{
//twice bigger buffer
result_packed_buff.resize(result_packed_buff.size()*2);
continue;
}
if (result_packed_buff.size() != m_zstream.avail_out)
result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out);
break;
}
return cb(result_packed_buff);
}
template<typename callback_t>
bool stop(callback_t cb)
{
if (!m_initialized)
{
return false;
}
std::string result_packed_buff;
//theoretically it supposed to be smaller
result_packed_buff.resize(1000000, 'X');
while (true)
{
m_zstream.next_in = nullptr;
m_zstream.avail_in = 0;
m_zstream.next_out = (Bytef*)result_packed_buff.data();
m_zstream.avail_out = (uInt)result_packed_buff.size();
int ret = deflate(&m_zstream, Z_FINISH);
CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate at finish. err = " << ret);
if (ret != Z_STREAM_END)
{
//twice bigger buffer
result_packed_buff.resize(result_packed_buff.size() * 2);
continue;
}
if (result_packed_buff.size() != m_zstream.avail_out)
result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out);
m_initialized = false;
break;
}
return cb(result_packed_buff);
}
}; // class gzip_encoder_lyambda
} // namespace net_utils
} // namespace epee