From 389f3660db285b334c6a9004cdfda0ffba5f4592 Mon Sep 17 00:00:00 2001 From: selsta Date: Wed, 20 May 2026 00:24:18 +0200 Subject: [PATCH] http: share deterministic header field parser --- contrib/epee/include/net/http_base.h | 40 ++- contrib/epee/include/net/http_client.h | 100 +++---- .../include/net/http_protocol_handler.inl | 87 +++--- tests/unit_tests/http.cpp | 252 ++++++++++++++++++ 4 files changed, 376 insertions(+), 103 deletions(-) diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index b53766780..32256c2d9 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -29,7 +29,7 @@ #pragma once #include "memwipe.h" -#include +#include #include #include @@ -58,7 +58,7 @@ namespace net_utils typedef std::list > fields_list; - static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value) + static inline void add_field(std::string& out, boost::string_view name, boost::string_view value) { out.append(name.data(), name.size()).append(": "); out.append(value.data(), value.size()).append("\r\n"); @@ -68,6 +68,42 @@ namespace net_utils add_field(out, field.first, field.second); } + namespace detail + { + inline bool parse_header_line(boost::string_view line, boost::string_view& name, boost::string_view& value) + { + if(!line.empty() && line.back() == '\r') + line.remove_suffix(1); + if(line.empty()) + return false; + if(line.front() == ' ' || line.front() == '\t') + return false; + + const size_t colon = line.find(':'); + if(colon == boost::string_view::npos || colon == 0) + return false; + + name = line.substr(0, colon); + value = line.substr(colon + 1); + while(!value.empty() && (value.front() == ' ' || value.front() == '\t')) + value.remove_prefix(1); + while(!value.empty() && (value.back() == ' ' || value.back() == '\t')) + value.remove_suffix(1); + + for(char c : name) + { + const bool alnum = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); + const bool allowed = alnum || + c == '!' || c == '#' || c == '$' || c == '%' || c == '&' || + c == '\'' || c == '*' || c == '+' || c == '-' || c == '.' || + c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; + if(!allowed) + return false; + } + + return true; + } + } struct http_header_info { diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 66788c88d..90ac5882f 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -361,7 +361,11 @@ namespace net_utils recv_buff.assign(m_header_cache.begin()+pos+4, m_header_cache.end()); m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end()); - analize_cached_header_and_invoke_state(); + if(!analize_cached_header_and_invoke_state()) + { + m_state = reciev_machine_state_error; + return false; + } if (!on_header(m_response_info)) { MDEBUG("Connection cancelled by on_header"); @@ -594,64 +598,46 @@ namespace net_utils { MTRACE("http_stream_filter::parse_cached_header(*)"); - const char *ptr = m_cache_to_process.c_str(); - while (ptr[0] != '\r' || ptr[1] != '\n') + size_t cur = 0; + while(cur < m_cache_to_process.size()) { - // optional \n - if (*ptr == '\n') - ++ptr; - // an identifier composed of letters or - - const char *key_pos = ptr; - while (isalnum(*ptr) || *ptr == '_' || *ptr == '-') - ++ptr; - const char *key_end = ptr; - // optional space (not in RFC, but in previous code) - if (*ptr == ' ') - ++ptr; - CHECK_AND_ASSERT_MES(*ptr == ':', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); - ++ptr; - // optional whitespace, but not newlines - line folding is obsolete, let's ignore it - while (isblank(*ptr)) - ++ptr; - const char *value_pos = ptr; - while (*ptr != '\r' && *ptr != '\n') - ++ptr; - const char *value_end = ptr; - // optional trailing whitespace - while (value_end > value_pos && isblank(*(value_end-1))) - --value_end; - if (*ptr == '\r') - ++ptr; - CHECK_AND_ASSERT_MES(*ptr == '\n', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); - ++ptr; + const size_t line_end = m_cache_to_process.find('\n', cur); + CHECK_AND_ASSERT_MES(line_end != std::string::npos, false, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); - const std::string key = std::string(key_pos, key_end - key_pos); - const std::string value = std::string(value_pos, value_end - value_pos); - if (!key.empty()) - { - if (!string_tools::compare_no_case(key, "Connection")) - body_info.m_connection = value; - else if(!string_tools::compare_no_case(key, "Referrer")) - body_info.m_referer = value; - else if(!string_tools::compare_no_case(key, "Content-Length")) - body_info.m_content_length = value; - else if(!string_tools::compare_no_case(key, "Content-Type")) - body_info.m_content_type = value; - else if(!string_tools::compare_no_case(key, "Transfer-Encoding")) - body_info.m_transfer_encoding = value; - else if(!string_tools::compare_no_case(key, "Content-Encoding")) - body_info.m_content_encoding = value; - else if(!string_tools::compare_no_case(key, "Host")) - body_info.m_host = value; - else if(!string_tools::compare_no_case(key, "Cookie")) - body_info.m_cookie = value; - else if(!string_tools::compare_no_case(key, "User-Agent")) - body_info.m_user_agent = value; - else if(!string_tools::compare_no_case(key, "Origin")) - body_info.m_origin = value; - else - body_info.m_etc_fields.emplace_back(key, value); - } + boost::string_view line(m_cache_to_process.data() + cur, line_end - cur); + cur = line_end + 1; + + if(line == "\r" || line.empty()) + break; + + boost::string_view name; + boost::string_view value; + CHECK_AND_ASSERT_MES(detail::parse_header_line(line, name, value), false, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); + + std::string key(name.data(), name.size()); + std::string val(value.data(), value.size()); + if (!string_tools::compare_no_case(key, "Connection")) + body_info.m_connection = std::move(val); + else if(!string_tools::compare_no_case(key, "Referer")) + body_info.m_referer = std::move(val); + else if(!string_tools::compare_no_case(key, "Content-Length")) + body_info.m_content_length = std::move(val); + else if(!string_tools::compare_no_case(key, "Content-Type")) + body_info.m_content_type = std::move(val); + else if(!string_tools::compare_no_case(key, "Transfer-Encoding")) + body_info.m_transfer_encoding = std::move(val); + else if(!string_tools::compare_no_case(key, "Content-Encoding")) + body_info.m_content_encoding = std::move(val); + else if(!string_tools::compare_no_case(key, "Host")) + body_info.m_host = std::move(val); + else if(!string_tools::compare_no_case(key, "Cookie")) + body_info.m_cookie = std::move(val); + else if(!string_tools::compare_no_case(key, "User-Agent")) + body_info.m_user_agent = std::move(val); + else if(!string_tools::compare_no_case(key, "Origin")) + body_info.m_origin = std::move(val); + else + body_info.m_etc_fields.emplace_back(std::move(key), std::move(val)); } return true; } diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index f682c8bda..623bd40ef 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -25,8 +25,9 @@ // -#include +#include #include +#include #include "http_protocol_handler.h" #include "string_tools.h" #include "file_io_utils.h" @@ -545,54 +546,52 @@ namespace net_utils template bool simple_http_connection_handler::parse_cached_header(http_header_info& body_info, const std::string& m_cache_to_process, size_t pos) { - static const boost::regex rexp_mach_field( - "\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)|(User-Agent)|(Origin)" - // 12 3 4 5 6 7 8 9 10 11 - "|([\\w-]+?)) ?: ?((.*?)(\r?\n))[^\t ]", - //11 1213 14 - boost::regex::icase | boost::regex::normal); - - boost::smatch result; - std::string::const_iterator it_current_bound = m_cache_to_process.begin(); - std::string::const_iterator it_end_bound = m_cache_to_process.begin()+pos; - body_info.clear(); + if(pos > m_cache_to_process.size() || pos > HTTP_MAX_HEADER_LEN) + return false; - //lookup all fields and fill well-known fields - while( boost::regex_search( it_current_bound, it_end_bound, result, rexp_mach_field, boost::match_default) && result[0].matched) + size_t cur = 0; + + while(cur < pos) { - const size_t field_val = 14; - const size_t field_etc_name = 12; + const size_t line_end = m_cache_to_process.find('\n', cur); + if(line_end == std::string::npos || line_end >= pos) + break; - int i = 2; //start position = 2 - if(result[i++].matched)//"Connection" - body_info.m_connection = result[field_val]; - else if(result[i++].matched)//"Referer" - body_info.m_referer = result[field_val]; - else if(result[i++].matched)//"Content-Length" - body_info.m_content_length = result[field_val]; - else if(result[i++].matched)//"Content-Type" - body_info.m_content_type = result[field_val]; - else if(result[i++].matched)//"Transfer-Encoding" - body_info.m_transfer_encoding = result[field_val]; - else if(result[i++].matched)//"Content-Encoding" - body_info.m_content_encoding = result[field_val]; - else if(result[i++].matched)//"Host" - body_info.m_host = result[field_val]; - else if(result[i++].matched)//"Cookie" - body_info.m_cookie = result[field_val]; - else if(result[i++].matched)//"User-Agent" - body_info.m_user_agent = result[field_val]; - else if(result[i++].matched)//"Origin" - body_info.m_origin = result[field_val]; - else if(result[i++].matched)//e.t.c (HAVE TO BE MATCHED!) - body_info.m_etc_fields.push_back(std::pair(result[field_etc_name], result[field_val])); + boost::string_view line(m_cache_to_process.data() + cur, line_end - cur); + cur = line_end + 1; + + // End of header block. + if(line == "\r" || line.empty()) + break; + + boost::string_view name; + boost::string_view value; + if(!detail::parse_header_line(line, name, value)) + return false; + + if(boost::iequals(name, "Connection")) + body_info.m_connection = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Referer")) + body_info.m_referer = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Content-Length")) + body_info.m_content_length = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Content-Type")) + body_info.m_content_type = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Transfer-Encoding")) + body_info.m_transfer_encoding = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Content-Encoding")) + body_info.m_content_encoding = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Host")) + body_info.m_host = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Cookie")) + body_info.m_cookie = std::string(value.data(), value.size()); + else if(boost::iequals(name, "User-Agent")) + body_info.m_user_agent = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Origin")) + body_info.m_origin = std::string(value.data(), value.size()); else - { - LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::parse_cached_header() not matched last entry in:" << m_cache_to_process); - } - - it_current_bound = result[(int)result.size()-1]. first; + body_info.m_etc_fields.push_back(std::make_pair(std::string(name.data(), name.size()), std::string(value.data(), value.size()))); } return true; } diff --git a/tests/unit_tests/http.cpp b/tests/unit_tests/http.cpp index 4ded76c9d..74fe41293 100644 --- a/tests/unit_tests/http.cpp +++ b/tests/unit_tests/http.cpp @@ -28,6 +28,9 @@ #include "gtest/gtest.h" #include "net/http_auth.h" +#include "net/http_client.h" +#include "syncobj.h" +#include "net/http_protocol_handler.h" #include #include @@ -51,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +71,111 @@ namespace http = epee::net_utils::http; using fields = std::unordered_map; using auth_responses = std::vector; +class test_http_endpoint final : public epee::net_utils::i_service_endpoint +{ +public: + bool do_send(epee::byte_slice message) override + { + sent.append(reinterpret_cast(message.data()), message.size()); + return true; + } + bool close(const bool) override { return true; } + bool send_done() override { return true; } + bool call_run_once_service_io() override { return true; } + bool request_callback() override { return true; } + boost::asio::io_context& get_io_context() override { return io_context; } + bool add_ref() override { return true; } + bool release() override { return true; } + + boost::asio::io_context io_context; + std::string sent; +}; + +class dummy_client +{ +public: + bool connect(const std::string&, int, std::chrono::milliseconds, bool = false, const std::string& = "0.0.0.0") { return true; } + bool connect(const std::string&, const std::string&, std::chrono::milliseconds, bool = false, const std::string& = "0.0.0.0") { return true; } + bool disconnect() { return true; } + bool send(const boost::string_ref, std::chrono::milliseconds) { return true; } + bool send(const void*, size_t) { return true; } + bool recv(std::string& buff, std::chrono::milliseconds) + { + buff = data; + data.clear(); + return true; + } + void set_ssl(epee::net_utils::ssl_options_t) { } + bool is_connected(bool *ssl = NULL) { return true; } + uint64_t get_bytes_sent() const { return 1; } + uint64_t get_bytes_received() const { return 1; } + + void set_test_data(const std::string& s) { data = s; } + +private: + std::string data; +}; + +class test_http_client final : public http::http_simple_client_template +{ +public: + bool on_header(const http::http_response_info& headers) override + { + ++headers_seen; + last_headers = headers; + return true; + } + + http::http_response_info last_headers; + unsigned headers_seen = 0; +}; + +class capturing_http_handler final : public http::i_http_server_handler +{ +public: + bool handle_http_request( + const http::http_request_info& query_info, + http::http_response_info& response, + epee::net_utils::connection_context_base&) override + { + requests.push_back(query_info); + response.m_response_code = 200; + response.m_response_comment = "OK"; + response.m_mime_tipe = "text/plain"; + return true; + } + + std::vector requests; +}; + +struct http_request_capture +{ + std::vector results; + std::vector requests; + std::string sent; +}; + +http_request_capture feed_http_request(const std::vector& chunks) +{ + capturing_http_handler handler; + test_http_endpoint endpoint; + epee::net_utils::connection_context_base context; + http::custum_handler_config config; + config.m_phandler = &handler; + + http::http_custom_handler connection(&endpoint, config, context); + std::vector results; + for (const std::string& chunk : chunks) + results.push_back(connection.handle_recv(chunk.data(), chunk.size())); + + return {results, handler.requests, endpoint.sent}; +} + +http_request_capture feed_http_request(const std::string& request) +{ + return feed_http_request(std::vector{request}); +} + void rng(size_t len, uint8_t *ptr) { crypto::rand(len, ptr); @@ -701,6 +810,149 @@ TEST(HTTP_Client_Auth, MD5_auth) } +TEST(HTTP, Parse_Header_Line) +{ + boost::string_view name; + boost::string_view value; + + ASSERT_TRUE(http::detail::parse_header_line("Host: example.com\r", name, value)); + EXPECT_EQ("Host", name); + EXPECT_EQ("example.com", value); + + ASSERT_TRUE(http::detail::parse_header_line("Content-Type:\t application/json \t", name, value)); + EXPECT_EQ("Content-Type", name); + EXPECT_EQ("application/json", value); + + ASSERT_TRUE(http::detail::parse_header_line("Host:example.com", name, value)); + EXPECT_EQ("Host", name); + EXPECT_EQ("example.com", value); + + ASSERT_TRUE(http::detail::parse_header_line("X!#$%&'*+-.^_`|~: value", name, value)); + EXPECT_EQ("X!#$%&'*+-.^_`|~", name); + EXPECT_EQ("value", value); + + EXPECT_FALSE(http::detail::parse_header_line("", name, value)); + EXPECT_FALSE(http::detail::parse_header_line("Host : example.com", name, value)); + EXPECT_FALSE(http::detail::parse_header_line(" Content-Length: 1", name, value)); + EXPECT_FALSE(http::detail::parse_header_line("\tContent-Length: 1", name, value)); + EXPECT_FALSE(http::detail::parse_header_line("Bad Header", name, value)); + EXPECT_FALSE(http::detail::parse_header_line("Host example.com", name, value)); + EXPECT_FALSE(http::detail::parse_header_line("GET / HTTP/1.1", name, value)); +} + +TEST(HTTP, Server_Parses_Content_Length_First_Header) +{ + const std::string body = "0123456789"; + const auto capture = feed_http_request( + "POST /json_rpc HTTP/1.1\r\n" + "Content-Length: 10\r\n" + "Host: example.com\r\n" + "\r\n" + body + ); + + ASSERT_EQ(1u, capture.results.size()); + ASSERT_TRUE(capture.results.front()); + ASSERT_EQ(1u, capture.requests.size()); + EXPECT_STREQ("10", capture.requests.front().m_header_info.m_content_length.c_str()); + EXPECT_EQ(body, capture.requests.front().m_body); +} + +TEST(HTTP, Server_Parses_First_Header_After_Split_Request_Line) +{ + const std::string body = "0123456789"; + const auto capture = feed_http_request({ + "POST /json_rpc HTTP/1.1\r\n", + "Content-Length: 10\r\n" + "Host: example.com\r\n" + "\r\n" + body + }); + + ASSERT_EQ(2u, capture.results.size()); + EXPECT_TRUE(capture.results[0]); + EXPECT_TRUE(capture.results[1]); + ASSERT_EQ(1u, capture.requests.size()); + EXPECT_STREQ("10", capture.requests.front().m_header_info.m_content_length.c_str()); + EXPECT_EQ(body, capture.requests.front().m_body); +} + +TEST(HTTP, Server_Rejects_Malformed_First_Header) +{ + const auto capture = feed_http_request( + "GET / HTTP/1.1\r\n" + "Bad Header Without Colon\r\n" + "Host: example.com\r\n" + "\r\n" + ); + + ASSERT_EQ(1u, capture.results.size()); + EXPECT_FALSE(capture.results.front()); + EXPECT_TRUE(capture.requests.empty()); +} + +TEST(HTTP, Server_Rejects_Malformed_Later_Header) +{ + const auto capture = feed_http_request( + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Bad Header Without Colon\r\n" + "\r\n" + ); + + ASSERT_EQ(1u, capture.results.size()); + EXPECT_FALSE(capture.results.front()); + EXPECT_TRUE(capture.requests.empty()); +} + +TEST(HTTP, Server_Keeps_Unknown_First_Header) +{ + const auto capture = feed_http_request( + "GET / HTTP/1.1\r\n" + "X-Test: abc\r\n" + "Host: example.com\r\n" + "\r\n" + ); + + ASSERT_EQ(1u, capture.results.size()); + ASSERT_TRUE(capture.results.front()); + ASSERT_EQ(1u, capture.requests.size()); + ASSERT_EQ(1u, capture.requests.front().m_header_info.m_etc_fields.size()); + EXPECT_STREQ("X-Test", capture.requests.front().m_header_info.m_etc_fields.front().first.c_str()); + EXPECT_STREQ("abc", capture.requests.front().m_header_info.m_etc_fields.front().second.c_str()); +} + +TEST(HTTP, Client_Keeps_Unknown_Header) +{ + test_http_client client; + const bool result = client.test( + "HTTP/1.1 200 OK\r\n" + "X-Test: abc\r\n" + "Content-Length: 0\r\n" + "\r\n", + std::chrono::milliseconds(1000) + ); + + ASSERT_TRUE(result); + EXPECT_EQ(1u, client.headers_seen); + ASSERT_EQ(1u, client.last_headers.m_header_info.m_etc_fields.size()); + EXPECT_STREQ("X-Test", client.last_headers.m_header_info.m_etc_fields.front().first.c_str()); + EXPECT_STREQ("abc", client.last_headers.m_header_info.m_etc_fields.front().second.c_str()); +} + +TEST(HTTP, Client_Rejects_Malformed_Response_Header) +{ + test_http_client client; + const bool result = client.test( + "HTTP/1.1 200 OK\r\n" + "Bad Header Without Colon\r\n" + "Content-Length: 0\r\n" + "\r\n", + std::chrono::milliseconds(1000) + ); + + EXPECT_FALSE(result); + EXPECT_EQ(0u, client.headers_seen); +} + TEST(HTTP, Add_Field) { std::string str{"leading text"};