Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity
blockchain/contrib/epee/include/net/http_client.h
cryptozoidberg 4ad93162e8
https enabled in remote node
2025-01-30 14:50:51 +04:00

1226 lines
No EOL
46 KiB
C++
Raw Permalink Blame History

// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of the Andrey N. Sabelnikov nor the
// names of its contributors may be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
#pragma once
#include <boost/shared_ptr.hpp>
#include <boost/regex.hpp>
#include <boost/lexical_cast.hpp>
//#include <mbstring.h>
#include <algorithm>
#include <cctype>
#include <functional>
#include "net_helper.h"
#include "http_client_base.h"
//#ifdef HTTP_ENABLE_GZIP
#include "gzip_encoding.h"
//#endif
#include "string_tools.h"
#include "reg_exp_definer.h"
#include "http_base.h"
#include "to_nonconst_iterator.h"
#include "net_parse_helpers.h"
//#include "shlwapi.h"
//#pragma comment(lib, "shlwapi.lib")
extern epee::critical_section gregexp_lock;
namespace epee
{
namespace net_utils
{
using namespace std;
/*struct url
{
public:
void parse(const std::string& url_s)
{
const string prot_end("://");
string::const_iterator prot_i = search(url_s.begin(), url_s.end(),
prot_end.begin(), prot_end.end());
protocol_.reserve(distance(url_s.begin(), prot_i));
transform(url_s.begin(), prot_i,
back_inserter(protocol_),
ptr_fun<int,int>(tolower)); // protocol is icase
if( prot_i == url_s.end() )
return;
advance(prot_i, prot_end.length());
string::const_iterator path_i = find(prot_i, url_s.end(), '/');
host_.reserve(distance(prot_i, path_i));
transform(prot_i, path_i,
back_inserter(host_),
ptr_fun<int,int>(tolower)); // host is icase
string::const_iterator query_i = find(path_i, url_s.end(), '?');
path_.assign(path_i, query_i);
if( query_i != url_s.end() )
++query_i;
query_.assign(query_i, url_s.end());
}
std::string protocol_;
std::string host_;
std::string path_;
std::string query_;
};*/
//---------------------------------------------------------------------------
static inline const char* get_hex_vals()
{
static char hexVals[16] = { '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' };
return hexVals;
}
static inline const char* get_unsave_chars()
{
//static char unsave_chars[] = "\"<>%\\^[]`+$,@:;/!#?=&";
static char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&";
return unsave_chars;
}
static inline bool is_unsafe(unsigned char compare_char)
{
if (compare_char <= 32 || compare_char >= 123)
return true;
const char* punsave = get_unsave_chars();
for (int ichar_pos = 0; 0 != punsave[ichar_pos]; ichar_pos++)
if (compare_char == punsave[ichar_pos])
return true;
return false;
}
static inline
std::string dec_to_hex(char num, int radix)
{
int temp = 0;
std::string csTmp;
int num_char;
num_char = (int)num;
if (num_char < 0)
num_char = 256 + num_char;
while (num_char >= radix)
{
temp = num_char % radix;
num_char = (int)floor((float)num_char / (float)radix);
csTmp = get_hex_vals()[temp];
}
csTmp += get_hex_vals()[num_char];
if (csTmp.size() < 2)
{
csTmp += '0';
}
std::reverse(csTmp.begin(), csTmp.end());
//_mbsrev((unsigned char*)csTmp.data());
return csTmp;
}
static inline std::string convert(char val)
{
std::string csRet;
csRet += "%";
csRet += dec_to_hex(val, 16);
return csRet;
}
static inline std::string conver_to_url_format(const std::string& uri)
{
std::string result;
for (size_t i = 0; i != uri.size(); i++)
{
if (is_unsafe(uri[i]))
result += convert(uri[i]);
else
result += uri[i];
}
return result;
}
static inline std::string convert_to_url_format_force_all(const std::string& uri)
{
std::string result;
for (size_t i = 0; i != uri.size(); i++)
{
result += convert(uri[i]);
}
return result;
}
namespace http
{
struct i_http_client
{
virtual void set_host_name(const std::string& name) = 0;
virtual boost::asio::ip::tcp::socket& get_socket() = 0;
virtual bool connect(const std::string& host, int port, unsigned int timeout) = 0;
virtual bool set_timeouts(unsigned int connection_timeout, unsigned int recv_timeout) = 0;
virtual bool connect(const std::string& host, std::string port) = 0;
virtual bool connect(const std::string& host, const std::string& port, unsigned int timeout) = 0;
virtual bool disconnect() = 0;
virtual bool is_connected() = 0;
virtual bool invoke_get(const std::string& uri, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
virtual bool invoke(const std::string& uri, const std::string& method, const std::string& body, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
virtual bool invoke_post(const std::string& uri, const std::string& body, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
};
template<bool is_ssl>
class http_simple_client_t : public i_target_handler,
public i_http_client
{
public:
private:
enum reciev_machine_state
{
reciev_machine_state_header,
reciev_machine_state_body_content_len,
reciev_machine_state_body_connection_close,
reciev_machine_state_body_chunked,
reciev_machine_state_done,
reciev_machine_state_error
};
enum chunked_state {
http_chunked_state_chunk_head,
http_chunked_state_chunk_body,
http_chunked_state_done,
http_chunked_state_undefined
};
blocked_mode_client_t<is_ssl> m_net_client;
std::string m_host_buff;
std::string m_port;
//unsigned int m_timeout;
unsigned int m_connection_timeout;
unsigned int m_recv_timeout;
std::string m_header_cache;
http_response_info m_response_info;
//std::string* m_ptarget_buffer;
boost::shared_ptr<i_sub_handler> m_pcontent_encoding_handler;
reciev_machine_state m_state;
chunked_state m_chunked_state;
std::string m_chunked_cache;
critical_section m_lock;
protected:
uint64_t m_len_in_summary;
uint64_t m_len_in_remain;
public:
void set_host_name(const std::string& name)
{
CRITICAL_REGION_LOCAL(m_lock);
m_host_buff = name;
}
boost::asio::ip::tcp::socket& get_socket()
{
return m_net_client.get_socket();
}
bool connect(const std::string& host, int port, unsigned int timeout)
{
return connect(host, std::to_string(port), timeout);
}
bool set_timeouts(unsigned int connection_timeout, unsigned int recv_timeout)
{
m_connection_timeout = connection_timeout;
m_recv_timeout = recv_timeout;
return true;
}
bool connect(const std::string& host, std::string port)
{
CRITICAL_REGION_LOCAL(m_lock);
m_host_buff = host;
m_port = port;
return m_net_client.connect(host, port, m_connection_timeout, m_recv_timeout);
}
bool connect(const std::string& host, const std::string& port, unsigned int timeout)
{
m_connection_timeout = m_recv_timeout = timeout;
return connect(host, port);
}
//---------------------------------------------------------------------------
bool disconnect()
{
CRITICAL_REGION_LOCAL(m_lock);
return m_net_client.disconnect();
}
//---------------------------------------------------------------------------
bool is_connected()
{
CRITICAL_REGION_LOCAL(m_lock);
return m_net_client.is_connected();
}
//---------------------------------------------------------------------------
virtual bool handle_target_data(std::string& piece_of_transfer)
{
CRITICAL_REGION_LOCAL(m_lock);
m_response_info.m_body += piece_of_transfer;
piece_of_transfer.clear();
return true;
}
//---------------------------------------------------------------------------
inline
bool invoke_get(const std::string& uri, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
{
CRITICAL_REGION_LOCAL(m_lock);
return invoke(uri, "GET", body, ppresponse_info, additional_params);
}
//---------------------------------------------------------------------------
inline bool invoke(const std::string& uri, const std::string& method, const std::string& body, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
{
CRITICAL_REGION_LOCAL(m_lock);
if (!is_connected())
{
LOG_PRINT("Reconnecting...", LOG_LEVEL_3);
if (!connect(m_host_buff, m_port))
{
LOG_PRINT("Failed to connect to " << m_host_buff << ":" << m_port, LOG_LEVEL_3);
return false;
}
}
m_response_info.clear();
std::string req_buff = method + " ";
req_buff += uri + " HTTP/1.1\r\n" +
"Host: " + m_host_buff + "\r\n" + "Content-Length: " + boost::lexical_cast<std::string>(body.size()) + "\r\n";
//handle "additional_params"
for (fields_list::const_iterator it = additional_params.begin(); it != additional_params.end(); it++)
req_buff += it->first + ": " + it->second + "\r\n";
req_buff += "\r\n";
//--
bool res = m_net_client.send(req_buff);
CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
if (body.size())
res = m_net_client.send(body);
CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
if (ppresponse_info)
*ppresponse_info = &m_response_info;
m_state = reciev_machine_state_header;
return handle_reciev();
}
//---------------------------------------------------------------------------
inline bool invoke_post(const std::string& uri, const std::string& body, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
{
CRITICAL_REGION_LOCAL(m_lock);
return invoke(uri, "POST", body, ppresponse_info, additional_params);
}
private:
//---------------------------------------------------------------------------
inline bool handle_reciev()
{
CRITICAL_REGION_LOCAL(m_lock);
bool keep_handling = true;
bool need_more_data = true;
std::string recv_buffer;
while (keep_handling)
{
if (need_more_data)
{
if (!m_net_client.recv(recv_buffer))
{
LOG_PRINT("Unexpected reciec fail", LOG_LEVEL_3);
m_state = reciev_machine_state_error;
}
if (!recv_buffer.size())
{
//connection is going to be closed
if (reciev_machine_state_body_connection_close != m_state)
{
m_state = reciev_machine_state_error;
}
}
need_more_data = false;
}
switch (m_state)
{
case reciev_machine_state_header:
keep_handling = handle_header(recv_buffer, need_more_data);
break;
case reciev_machine_state_body_content_len:
keep_handling = handle_body_content_len(recv_buffer, need_more_data);
break;
case reciev_machine_state_body_connection_close:
keep_handling = handle_body_connection_close(recv_buffer, need_more_data);
break;
case reciev_machine_state_body_chunked:
keep_handling = handle_body_body_chunked(recv_buffer, need_more_data);
break;
case reciev_machine_state_done:
keep_handling = false;
break;
case reciev_machine_state_error:
keep_handling = false;
break;
}
}
m_header_cache.clear();
if (m_state != reciev_machine_state_error)
{
if (m_response_info.m_header_info.m_connection.size() && !string_tools::compare_no_case("close", m_response_info.m_header_info.m_connection))
disconnect();
return true;
}
else
{
LOG_PRINT_L3("Returning false because of wrong state machine. state: " << m_state);
return false;
}
}
//---------------------------------------------------------------------------
inline
bool handle_header(std::string& recv_buff, bool& need_more_data)
{
CRITICAL_REGION_LOCAL(m_lock);
if (!recv_buff.size())
{
LOG_ERROR("Connection closed at handle_header");
m_state = reciev_machine_state_error;
return false;
}
m_header_cache += recv_buff;
recv_buff.clear();
std::string::size_type pos = m_header_cache.find("\r\n\r\n");
if (pos != std::string::npos)
{
recv_buff.assign(m_header_cache.begin() + pos + 4, m_header_cache.end());
m_header_cache.erase(m_header_cache.begin() + pos + 4, m_header_cache.end());
analize_cached_header_and_invoke_state();
m_header_cache.clear();
if (!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done))
need_more_data = true;
return true;
}
else
need_more_data = true;
return true;
}
//---------------------------------------------------------------------------
inline
bool handle_body_content_len(std::string& recv_buff, bool& need_more_data)
{
CRITICAL_REGION_LOCAL(m_lock);
if (!recv_buff.size())
{
LOG_PRINT("Warning: Content-Len mode, but connection unexpectedly closed", LOG_LEVEL_3);
m_state = reciev_machine_state_done;
return true;
}
CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()");
m_len_in_remain -= recv_buff.size();
bool r = m_pcontent_encoding_handler->update_in(recv_buff);
//CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_pcontent_encoding_handler->update_in returned false");
if (!r)
{
m_state = reciev_machine_state_error;
disconnect();
return false;
}
if (m_len_in_remain == 0)
m_state = reciev_machine_state_done;
else
need_more_data = true;
return true;
}
//---------------------------------------------------------------------------
inline
bool handle_body_connection_close(std::string& recv_buff, bool& need_more_data)
{
CRITICAL_REGION_LOCAL(m_lock);
if (!recv_buff.size())
{
m_state = reciev_machine_state_done;
return true;
}
need_more_data = true;
m_pcontent_encoding_handler->update_in(recv_buff);
return true;
}
//---------------------------------------------------------------------------
inline bool is_hex_symbol(char ch)
{
if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f'))
return true;
else
return false;
}
//---------------------------------------------------------------------------
inline
bool get_len_from_chunk_head(const std::string &chunk_head, uint64_t& result_size)
{
std::stringstream str_stream;
str_stream << std::hex;
if (!(str_stream << chunk_head && str_stream >> result_size))
return false;
return true;
}
//---------------------------------------------------------------------------
inline
bool get_chunk_head(std::string& buff, uint64_t& chunk_size, bool& is_matched)
{
is_matched = false;
size_t offset = 0;
for (std::string::iterator it = buff.begin(); it != buff.end(); it++, offset++)
{
if (!is_hex_symbol(*it))
{
if (*it == '\r' || *it == ' ')
{
offset--;
continue;
}
else if (*it == '\n')
{
std::string chunk_head = buff.substr(0, offset);
if (!get_len_from_chunk_head(chunk_head, chunk_size))
return false;
if (0 == chunk_size)
{
//Here is a small confusion
//In breif - if the chunk is the last one we need to get terminating sequence
//along with the cipher, generally in the "ddd\r\n\r\n" form
for (it++; it != buff.end(); it++)
{
if ('\r' == *it)
continue;
else if ('\n' == *it)
break;
else
{
LOG_ERROR("http_stream_filter: Wrong last chunk terminator");
return false;
}
}
if (it == buff.end())
return true;
}
buff.erase(buff.begin(), ++it);
is_matched = true;
return true;
}
else
return false;
}
}
return true;
}
//---------------------------------------------------------------------------
inline
bool handle_body_body_chunked(std::string& recv_buff, bool& need_more_data)
{
CRITICAL_REGION_LOCAL(m_lock);
if (!recv_buff.size())
{
LOG_PRINT("Warning: CHUNKED mode, but connection unexpectedly closed", LOG_LEVEL_3);
m_state = reciev_machine_state_done;
return true;
}
m_chunked_cache += recv_buff;
recv_buff.clear();
bool is_matched = false;
while (true)
{
if (!m_chunked_cache.size())
{
need_more_data = true;
break;
}
switch (m_chunked_state)
{
case http_chunked_state_chunk_head:
if (m_chunked_cache[0] == '\n' || m_chunked_cache[0] == '\r')
{
//optimize a bit
if (m_chunked_cache[0] == '\r' && m_chunked_cache.size()>1 && m_chunked_cache[1] == '\n')
m_chunked_cache.erase(0, 2);
else
m_chunked_cache.erase(0, 1);
break;
}
if (!get_chunk_head(m_chunked_cache, m_len_in_remain, is_matched))
{
LOG_ERROR("http_stream_filter::handle_chunked(*) Failed to get length from chunked head:" << m_chunked_cache);
m_state = reciev_machine_state_error;
return false;
}
if (!is_matched)
{
need_more_data = true;
return true;
}
else
{
m_chunked_state = http_chunked_state_chunk_body;
if (m_len_in_remain == 0)
{//last chunk, let stop the stream and fix the chunk queue.
m_state = reciev_machine_state_done;
return true;
}
m_chunked_state = http_chunked_state_chunk_body;
break;
}
break;
case http_chunked_state_chunk_body:
{
std::string chunk_body;
if (m_len_in_remain >= m_chunked_cache.size())
{
m_len_in_remain -= m_chunked_cache.size();
chunk_body.swap(m_chunked_cache);
}
else
{
chunk_body.assign(m_chunked_cache, 0, m_len_in_remain);
m_chunked_cache.erase(0, m_len_in_remain);
m_len_in_remain = 0;
}
m_pcontent_encoding_handler->update_in(chunk_body);
if (!m_len_in_remain)
m_chunked_state = http_chunked_state_chunk_head;
}
break;
case http_chunked_state_done:
m_state = reciev_machine_state_done;
return true;
case http_chunked_state_undefined:
default:
LOG_ERROR("http_stream_filter::handle_chunked(): Wrong state" << m_chunked_state);
return false;
}
}
return true;
}
//---------------------------------------------------------------------------
inline
bool parse_header(http_header_info& body_info, const std::string& m_cache_to_process)
{
LOG_FRAME("http_stream_filter::parse_cached_header(*)", LOG_LEVEL_4);
STATIC_REGEXP_EXPR_1(rexp_mach_field,
"\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)"
// 12 3 4 5 6 7 8 9
"|([\\w-]+?)) ?: ?((.*?)(\r?\n))[^\t ]",
//10 1112 13
boost::regex::icase | boost::regex::normal);
boost::smatch result;
std::string::const_iterator it_current_bound = m_cache_to_process.begin();
std::string::const_iterator it_end_bound = m_cache_to_process.end();
//lookup all fields and fill well-known fields
while (boost::regex_search(it_current_bound, it_end_bound, result, rexp_mach_field, boost::match_default) && result[0].matched)
{
const size_t field_val = 12;
//const size_t field_etc_name = 10;
int i = 2; //start position = 2
if (result[i++].matched)//"Connection"
body_info.m_connection = result[field_val];
else if (result[i++].matched)//"Referrer"
body_info.m_referer = result[field_val];
else if (result[i++].matched)//"Content-Length"
body_info.m_content_length = result[field_val];
else if (result[i++].matched)//"Content-Type"
body_info.m_content_type = result[field_val];
else if (result[i++].matched)//"Transfer-Encoding"
body_info.m_transfer_encoding = result[field_val];
else if (result[i++].matched)//"Content-Encoding"
body_info.m_content_encoding = result[field_val];
else if (result[i++].matched)//"Host"
{
body_info.m_host = result[field_val];
string_tools::trim(body_info.m_host);
}
else if (result[i++].matched)//"Cookie"
body_info.m_cookie = result[field_val];
else if (result[i++].matched)//e.t.c (HAVE TO BE MATCHED!)
{
;
}
else
{
CHECK_AND_ASSERT_MES(false, false, "http_stream_filter::parse_cached_header() not matched last entry in:" << m_cache_to_process);
}
it_current_bound = result[(int)result.size() - 1].first;
}
return true;
}
inline
bool analize_first_response_line()
{
//First line response, look like this: "HTTP/1.1 200 OK"
STATIC_REGEXP_EXPR_1(rexp_match_first_response_line, "^HTTP/(\\d+).(\\d+) ((\\d)\\d{2})( [^\n]*)?\r?\n", boost::regex::icase | boost::regex::normal);
// 1 2 34 5
//size_t match_len = 0;
boost::smatch result;
if (boost::regex_search(m_header_cache, result, rexp_match_first_response_line, boost::match_default) && result[0].matched)
{
CHECK_AND_ASSERT_MES(result[1].matched&&result[2].matched, false, "http_stream_filter::handle_invoke_reply_line() assert failed...");
m_response_info.m_http_ver_hi = boost::lexical_cast<int>(result[1]);
m_response_info.m_http_ver_lo = boost::lexical_cast<int>(result[2]);
m_response_info.m_response_code = boost::lexical_cast<int>(result[3]);
m_header_cache.erase(to_nonsonst_iterator(m_header_cache, result[0].first), to_nonsonst_iterator(m_header_cache, result[0].second));
return true;
}
else
{
LOG_ERROR("http_stream_filter::handle_invoke_reply_line(): Failed to match first response line:" << m_header_cache);
return false;
}
}
inline
bool set_reply_content_encoder()
{
STATIC_REGEXP_EXPR_1(rexp_match_gzip, "^.*?((gzip)|(deflate))", boost::regex::icase | boost::regex::normal);
boost::smatch result; // 12 3
if (boost::regex_search(m_response_info.m_header_info.m_content_encoding, result, rexp_match_gzip, boost::match_default) && result[0].matched)
{
#ifdef HTTP_ENABLE_GZIP
m_pcontent_encoding_handler.reset(new content_encoding_gzip(this, result[3].matched));
#else
m_pcontent_encoding_handler.reset(new do_nothing_sub_handler(this));
LOG_ERROR("GZIP encoding not supported in this build, please add zlib to your project and define HTTP_ENABLE_GZIP");
return false;
#endif
}
else
{
m_pcontent_encoding_handler.reset(new do_nothing_sub_handler(this));
}
return true;
}
inline
bool analize_cached_header_and_invoke_state()
{
m_response_info.clear();
analize_first_response_line();
std::string fake_str; //gcc error workaround
bool res = parse_header(m_response_info.m_header_info, m_header_cache);
CHECK_AND_ASSERT_MES(res, false, "http_stream_filter::analize_cached_reply_header_and_invoke_state(): failed to anilize reply header: " << m_header_cache);
set_reply_content_encoder();
m_len_in_summary = 0;
bool content_len_valid = false;
if (m_response_info.m_header_info.m_content_length.size())
content_len_valid = string_tools::get_xtype_from_string(m_len_in_summary, m_response_info.m_header_info.m_content_length);
if (!m_len_in_summary && ((m_response_info.m_response_code >= 100 && m_response_info.m_response_code<200)
|| 204 == m_response_info.m_response_code
|| 304 == m_response_info.m_response_code))
{//There will be no response body, server will display the local page with error
m_state = reciev_machine_state_done;
return true;
}
else if (m_response_info.m_header_info.m_transfer_encoding.size())
{
string_tools::trim(m_response_info.m_header_info.m_transfer_encoding);
if (string_tools::compare_no_case(m_response_info.m_header_info.m_transfer_encoding, "chunked"))
{
LOG_ERROR("Wrong Transfer-Encoding:" << m_response_info.m_header_info.m_transfer_encoding);
m_state = reciev_machine_state_error;
return false;
}
m_state = reciev_machine_state_body_chunked;
m_chunked_state = http_chunked_state_chunk_head;
return true;
}
else if (!m_response_info.m_header_info.m_content_length.empty())
{
//In the response header the length was specified
if (!content_len_valid)
{
LOG_ERROR("http_stream_filter::analize_cached_reply_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length=" << m_response_info.m_header_info.m_content_length);
m_state = reciev_machine_state_error;
return false;
}
if (!m_len_in_summary)
{
m_state = reciev_machine_state_done;
return true;
}
else
{
m_len_in_remain = m_len_in_summary;
m_state = reciev_machine_state_body_content_len;
return true;
}
}
else if (!m_response_info.m_header_info.m_connection.empty() && is_connection_close_field(m_response_info.m_header_info.m_connection))
{ //By indirect signs we suspect that data transfer will end with a connection break
m_state = reciev_machine_state_body_connection_close;
}
else if (is_multipart_body(m_response_info.m_header_info, fake_str))
{
m_state = reciev_machine_state_error;
LOG_ERROR("Unsupported MULTIPART BODY.");
return false;
}
else
{ //Apparently there are no signs of the form of transfer, will receive data until the connection is closed
m_state = reciev_machine_state_body_connection_close;
LOG_PRINT("Undefinded transfer type, consider http_body_transfer_connection_close method. header: " << m_header_cache, LOG_LEVEL_2);
return false;
}
return false;
}
inline
bool is_connection_close_field(const std::string& str)
{
STATIC_REGEXP_EXPR_1(rexp_match_close, "^\\s*close", boost::regex::icase | boost::regex::normal);
boost::smatch result;
if (boost::regex_search(str, result, rexp_match_close, boost::match_default) && result[0].matched)
return true;
else
return false;
}
inline
bool is_multipart_body(const http_header_info& head_info, OUT std::string& boundary)
{
//Check whether this is multi part - if yes, capture boundary immediately
STATIC_REGEXP_EXPR_1(rexp_match_multipart_type, "^\\s*multipart/([\\w\\-]+); boundary=((\"(.*?)\")|(\\\\\"(.*?)\\\\\")|([^\\s;]*))", boost::regex::icase | boost::regex::normal);
boost::smatch result;
if (boost::regex_search(head_info.m_content_type, result, rexp_match_multipart_type, boost::match_default) && result[0].matched)
{
if (result[4].matched)
boundary = result[4];
else if (result[6].matched)
boundary = result[6];
else if (result[7].matched)
boundary = result[7];
else
{
LOG_ERROR("Failed to match boundary in content-type=" << head_info.m_content_type);
return false;
}
return true;
}
else
return false;
return true;
}
};
// class http_simple_client
typedef http_simple_client_t<false> http_simple_client;
typedef http_simple_client_t<true> https_simple_client;
//suitable for both http and https
class http_universal_client: public i_http_client
{
public:
http_universal_client(): m_pclient(new http_simple_client())
{}
// Forward all calls to m_pclient
void set_host_name(const std::string& name) override
{
m_pclient->set_host_name(name);
}
bool connect(const std::string& host, int port, unsigned int timeout) override
{
return m_pclient->connect(host, port, timeout);
}
boost::asio::ip::tcp::socket& get_socket() override { return m_pclient->get_socket(); }
bool set_timeouts(unsigned int connection_timeout, unsigned int recv_timeout) override { return m_pclient->set_timeouts(connection_timeout, recv_timeout); }
bool connect(const std::string& host, std::string port) override { return m_pclient->connect(host, port); }
bool connect(const std::string& host, const std::string& port, unsigned int timeout) override { return m_pclient->connect(host, port, timeout); }
bool disconnect() override { return m_pclient->disconnect(); }
bool is_connected() override { return m_pclient->is_connected(); }
bool invoke_get(const std::string& uri, const std::string& body = std::string(), const http_response_info** ppresponse_info = nullptr, const fields_list& additional_params = fields_list()) override { return m_pclient->invoke_get(uri, body, ppresponse_info, additional_params); }
bool invoke(const std::string& uri, const std::string& method, const std::string& body, const http_response_info** ppresponse_info = nullptr, const fields_list& additional_params = fields_list()) override { return m_pclient->invoke(uri, method, body, ppresponse_info, additional_params); }
bool invoke_post(const std::string& uri, const std::string& body, const http_response_info** ppresponse_info = nullptr, const fields_list& additional_params = fields_list()) override { return m_pclient->invoke_post(uri, body, ppresponse_info, additional_params); }
void set_is_ssl(bool is_ssl)
{
if (m_is_ssl != is_ssl)
{
if (is_ssl)
{
m_pclient.reset(new https_simple_client());
}
else
{
m_pclient.reset(new http_simple_client());
}
m_is_ssl = is_ssl;
}
}
private:
bool m_is_ssl = false;
std::shared_ptr<i_http_client> m_pclient;
};
template<typename transport>
void configure_transport(const std::string schema, transport& tr)
{}
inline void configure_transport(const std::string schema, http_universal_client& tr)
{
if (schema == "https")
tr.set_is_ssl(true);
else
tr.set_is_ssl(false);
}
/************************************************************************/
/* */
/************************************************************************/
template<class t_transport>
bool invoke_request(const std::string& url, t_transport& tr, unsigned int timeout, const http_response_info** ppresponse_info, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list())
{
http::url_content u_c{};
bool r = parse_url(url, u_c);
CHECK_AND_ASSERT_MES(tr.is_connected() || u_c.host.empty() || r, false, "failed to parse url: " << url);
r = invoke_request(u_c, tr, timeout, ppresponse_info, method, body, additional_params);
return r;
}
template<class t_transport>
bool invoke_request(const http::url_content& u_c, t_transport& tr, unsigned int timeout, const http_response_info** ppresponse_info, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list())
{
if (!tr.is_connected() && !u_c.host.empty())
{
int port = static_cast<int>(u_c.port);
if (!port)
port = 80;//default for http
configure_transport(u_c.schema, tr);
if (!tr.connect(u_c.host, port, timeout))
{
LOG_PRINT_L2("invoke_request: cannot connect to " << u_c.host << ":" << port);
return false;
}
}
return tr.invoke(u_c.uri, method, body, ppresponse_info, additional_params);
}
struct idle_handler_base
{
virtual bool do_call(const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) = 0;
virtual ~idle_handler_base() {}
};
template <typename callback_t>
struct idle_handler : public idle_handler_base
{
callback_t m_cb;
idle_handler(callback_t cb) : m_cb(cb) {}
virtual bool do_call(const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes)
{
return m_cb(piece_of_data, total_bytes, received_bytes);
}
};
class http_https_simple_client_wrapper : virtual public http_simple_client, virtual public https_simple_client
{
public:
http_https_simple_client_wrapper(bool is_ssl, std::shared_ptr<idle_handler_base> ihb_cb)
: m_ssl(is_ssl)
, m_ihb_cb(ihb_cb)
{}
bool invoke_request(const http::url_content& u_c, unsigned int timeout, const http_response_info** ppresponse_info, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list())
{
bool r = false;
if (m_ssl)
r = epee::net_utils::http::invoke_request(u_c, static_cast<https_simple_client&>(*this), timeout, ppresponse_info, method, body, additional_params);
else
r = epee::net_utils::http::invoke_request(u_c, static_cast<http_simple_client&>(*this), timeout, ppresponse_info, method, body, additional_params);
return r;
}
private:
// class i_target_handler
virtual bool handle_target_data(std::string& piece_of_transfer) override
{
bool r = false;
if (m_ssl)
r = m_ihb_cb->do_call(piece_of_transfer, https_simple_client::m_len_in_summary, https_simple_client::m_len_in_summary - https_simple_client::m_len_in_remain);
else
r = m_ihb_cb->do_call(piece_of_transfer, http_simple_client::m_len_in_summary, http_simple_client::m_len_in_summary - http_simple_client::m_len_in_remain);
piece_of_transfer.clear();
return r;
}
bool m_ssl;
std::shared_ptr<idle_handler_base> m_ihb_cb;
};
class interruptible_http_client
{
bool m_permanent_error = false;
public:
template<typename callback_t>
bool invoke_cb(callback_t cb, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list())
{
http::url_content uc{};
if (!parse_url(url, uc))
{
LOG_PRINT_L0("HTTP request to " << url << " failed because the URL couldn't be parsed.");
m_permanent_error = true;
return false;
}
bool is_ssl = uc.schema == "https";
http_https_simple_client_wrapper wrapper(is_ssl, std::make_shared<idle_handler<callback_t>>(cb));
const http_response_info* p_hri = nullptr;
bool r = wrapper.invoke_request(uc, timeout, &p_hri, method, body, additional_params);
if (p_hri && !(p_hri->m_response_code >= 200 && p_hri->m_response_code < 300))
{
LOG_PRINT_L0(boost::to_upper_copy(uc.schema) << " request to " << url << " failed with code: " << p_hri->m_response_code);
m_permanent_error = true;
return false;
}
return r;
}
template<typename callback_t>
bool download(callback_t cb, const std::string& path_for_file, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list())
{
std::ofstream fs;
fs.open(path_for_file, std::ios::binary | std::ios::out | std::ios::trunc);
if (!fs.is_open())
{
LOG_ERROR("Fsiled to open " << path_for_file);
return false;
}
auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes)
{
fs.write(piece_of_data.data(), piece_of_data.size());
return cb(total_bytes, received_bytes);
};
bool r = this->invoke_cb(local_cb, url, timeout, method, body, additional_params);
fs.close();
return r;
}
//
template<typename callback_t>
bool download_and_unzip(callback_t cb, const std::string& path_for_file, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), uint64_t fails_count = 1000, const fields_list& additional_params = fields_list())
{
std::ofstream fs;
fs.open(path_for_file, std::ios::binary | std::ios::out | std::ios::trunc);
if (!fs.is_open())
{
LOG_ERROR("Fsiled to open " << path_for_file);
return false;
}
std::string buff;
gzip_decoder_lambda zip_decoder;
uint64_t state_total_bytes = 0;
uint64_t state_received_bytes_base = 0;
uint64_t state_received_bytes_current = 0;
bool stopped = false;
auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes)
{
//remember total_bytes only for first attempt, where fetched full lenght of the file
if (!state_total_bytes)
state_total_bytes = total_bytes;
buff += piece_of_data;
return zip_decoder.update_in(buff, [&](const std::string& unpacked_buff)
{
state_received_bytes_current = received_bytes;
fs.write(unpacked_buff.data(), unpacked_buff.size());
stopped = !cb(unpacked_buff, state_total_bytes, state_received_bytes_base + received_bytes);
return !stopped;
});
};
uint64_t current_err_count = 0;
bool r = false;
m_permanent_error = false;
while (!r && current_err_count < fails_count)
{
LOG_PRINT_L0("Attempt " << current_err_count + 1 << "/" << fails_count << " to get " << url << " (offset:" << state_received_bytes_base << ")");
fields_list additional_params_local = additional_params;
additional_params_local.push_back(std::make_pair<std::string, std::string>("Range", std::string("bytes=") + std::to_string(state_received_bytes_base) + "-"));
r = this->invoke_cb(local_cb, url, timeout, method, body, additional_params_local);
if (!r)
{
if (stopped || m_permanent_error)
break;
current_err_count++;
state_received_bytes_base += state_received_bytes_current;
state_received_bytes_current = 0;
boost::this_thread::sleep_for(boost::chrono::milliseconds(2000));
}
}
if (current_err_count >= fails_count)
{
LOG_PRINT_YELLOW("Downloading from " << url << " FAILED as it's reached maximum (" << fails_count << ") number of attempts. Downloaded " << state_received_bytes_base << " bytes.", LOG_LEVEL_0);
}
else if (m_permanent_error)
{
LOG_PRINT_YELLOW("Downloading from " << url << " FAILED due to permanent HTTP error. Downloaded " << state_received_bytes_base << " bytes.", LOG_LEVEL_0);
}
fs.close();
return r;
}
};
template<typename transport_t>
bool fetch_url_t(transport_t& tr, const url_content& u_c, std::string& response_body, const std::string& method = "GET", const std::string& request_body = "", unsigned int timeout = 1000)
{
fields_list additional_params;
if (!tr.is_connected() && !u_c.host.empty())
{
if (!tr.connect(u_c.host, static_cast<int>(u_c.port), timeout))
{
LOG_PRINT_L2("invoke_request: cannot connect to " << u_c.host << ":" << u_c.port);
return false;
}
}
const http_response_info* ppresponse_info = nullptr;
if (tr.invoke(u_c.uri, "GET", request_body, &ppresponse_info, additional_params) && ppresponse_info)
{
response_body = ppresponse_info->m_body;
return true;
}
return false;
}
inline
bool fetch_url(const std::string& url, std::string& response_body, const std::string& method = "GET", const std::string& request_body = "", unsigned int timeout = 1000)
{
try
{
url_content u_c = AUTO_VAL_INIT(u_c);
bool res = epee::net_utils::parse_url(url, u_c);
CHECK_AND_ASSERT_MES(res, false, "failed to parse url: " << url);
if (u_c.schema == "https")
{
https_simple_client tr;
return fetch_url_t(tr, u_c, response_body, method, request_body, timeout);
}
else
{
http_simple_client tr;
return fetch_url_t(tr, u_c, response_body, method, request_body, timeout);
}
}
catch (...)
{
return false;
}
}
} // namespace http
} // namespace net_utils
} // namespace epee
View git blame Copy permalink