From 0ea772d667a17d66215cd415eb6c9497b34621a2 Mon Sep 17 00:00:00 2001 From: lihuiba Date: Fri, 3 Nov 2023 17:46:35 +0800 Subject: [PATCH 1/4] minor improvements to http --- common/estring.h | 8 +--- net/http/headers.cpp | 5 --- net/http/headers.h | 6 ++- net/http/message.cpp | 35 ++++++++-------- net/http/message.h | 2 +- net/http/status.cpp | 98 +++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 121 insertions(+), 33 deletions(-) diff --git a/common/estring.h b/common/estring.h index 10464b21..9e57214f 100644 --- a/common/estring.h +++ b/common/estring.h @@ -321,12 +321,8 @@ class rstring_view // relative string_view, that stores values relative } public: - constexpr rstring_view() = default; // note: don't give _offset or _length - // an initial value and do not - // assigned them in default ctor() - // it would caused verb_init() error - rstring_view(uint64_t offset, uint64_t length) - { + constexpr rstring_view() = default; + constexpr rstring_view(uint64_t offset, uint64_t length) { assert(offset <= std::numeric_limits::max()); assert(length <= std::numeric_limits::max()); _offset = (OffsetType)offset; diff --git a/net/http/headers.cpp b/net/http/headers.cpp index de85a649..39f262e4 100644 --- a/net/http/headers.cpp +++ b/net/http/headers.cpp @@ -64,11 +64,6 @@ HeadersBase::iterator HeadersBase::find(std::string_view key) const { return {this, (uint16_t)(it - kv_begin())}; } -void buf_append(char*& ptr, std::string_view sv) { - memcpy(ptr, sv.data(), sv.size()); - ptr += sv.size(); -} - void buf_append(char*& ptr, uint64_t x) { auto begin = ptr; do { diff --git a/net/http/headers.h b/net/http/headers.h index 3153d165..05b2fd64 100644 --- a/net/http/headers.h +++ b/net/http/headers.h @@ -29,9 +29,13 @@ namespace photon { namespace net { namespace http { -void buf_append(char*& ptr, std::string_view sv); void buf_append(char*& ptr, uint64_t x); +inline void buf_append(char*& ptr, std::string_view sv) { + memcpy(ptr, sv.data(), sv.size()); + ptr += sv.size(); +} + class HeadersBase { public: HeadersBase() = default; diff --git a/net/http/message.cpp b/net/http/message.cpp index 395d10e4..ab8e3179 100644 --- a/net/http/message.cpp +++ b/net/http/message.cpp @@ -41,7 +41,7 @@ Message::~Message() { delete m_stream; } - if (m_buf_ownership && m_buf) + if (m_buf_ownership) free(m_buf); } @@ -105,11 +105,14 @@ int Message::append_bytes(uint16_t size) { return parse_res; } - LOG_DEBUG("add headers, header buf size `", m_buf + m_buf_capacity - p.cur()); - if (headers.reset(p.cur(), m_buf + m_buf_capacity - p.cur(), m_buf + m_buf_size - p.cur()) < 0) + auto buf_cap = m_buf + m_buf_capacity - p.cur(); + auto buf_size = m_buf + m_buf_size - p.cur(); + LOG_DEBUG("add headers: ", VALUE(buf_cap), VALUE(buf_size)); + if (headers.reset(p.cur(), buf_cap, buf_size) < 0) LOG_ERRNO_RETURN(0, -1, "failed to parse headers"); - m_abandon = (headers.get_value("Connection") == "close") || (!headers.get_value("Trailer").empty()); + m_abandon = (headers["Connection"] == "close" || + !headers["Trailer"].empty()); message_status = HEADER_PARSED; LOG_DEBUG("header parsed"); return 0; @@ -118,11 +121,9 @@ int Message::append_bytes(uint16_t size) { int Message::send_header(net::ISocketStream* stream) { if (stream != nullptr) m_stream = stream; // update stream if needed - if (m_keep_alive) - headers.insert("Connection", "keep-alive"); - else - headers.insert("Connection", "close"); - + using SV = std::string_view; + headers.insert("Connection", m_keep_alive ? SV("keep-alive") : + SV("close")); if (headers.space_remain() < 2) LOG_ERRNO_RETURN(ENOBUFS, -1, "no buffer"); @@ -322,17 +323,15 @@ int Request::reset(Verb v, std::string_view url, bool enable_proxy) { int Request::redirect(Verb v, estring_view location, bool enable_proxy) { estring full_location; - if (!location.starts_with(http_url_scheme) && (!location.starts_with(https_url_scheme))) { + if (!location.starts_with(http_url_scheme) && + !location.starts_with(https_url_scheme)) { full_location.appends(secure() ? https_url_scheme : http_url_scheme, host(), location); location = full_location; } StoredURL u(location); - auto new_request_line_size = verbstr[v].size() + sizeof(" HTTP/1.1\r\n"); - if (enable_proxy) - new_request_line_size += full_url_size(u); - else - new_request_line_size += u.target().size(); + auto new_request_line_size = verbstr[v].size() + sizeof(" HTTP/1.1\r\n") + + enable_proxy ? full_url_size(u) : u.target().size(); auto delta = new_request_line_size - m_buf_size; LOG_DEBUG(VALUE(delta)); @@ -381,11 +380,9 @@ int Response::set_result(int code, std::string_view reason) { char* buf = m_buf; m_status_code = code; buf_append(buf, "HTTP/1.1 "); - buf_append(buf, std::to_string(code)); + buf_append(buf, code); buf_append(buf, " "); - auto message = reason; - if (message.empty()) message = obsolete_reason(code); - buf_append(buf, message); + buf_append(buf, reason.size() ? reason : obsolete_reason(code)); buf_append(buf, "\r\n"); m_buf_size = buf - m_buf; headers.reset(m_buf + m_buf_size, m_buf_capacity - m_buf_size); diff --git a/net/http/message.h b/net/http/message.h index e8e8859f..95c5e747 100644 --- a/net/http/message.h +++ b/net/http/message.h @@ -57,7 +57,7 @@ class Message : public IStream { void reset(void* buf, uint16_t buf_capacity, bool buf_ownership = false, ISocketStream* s = nullptr, bool stream_ownership = false) { reset(s, stream_ownership); - if (m_buf_ownership && m_buf) { + if (m_buf_ownership) { free(m_buf); } m_buf = (char*)buf; diff --git a/net/http/status.cpp b/net/http/status.cpp index ef0ba126..8dcec7ec 100644 --- a/net/http/status.cpp +++ b/net/http/status.cpp @@ -17,7 +17,102 @@ limitations under the License. #include "photon/common/string_view.h" #include "message.h" +struct SV : public std::string_view { + template + constexpr SV(const char(&s)[N]) : SV(s, N) { } + constexpr SV(const char* s, size_t n) : std::string_view(s, n) { } +}; + +constexpr static SV code_str[] = { + /*100*/ "Continue", + /*101*/ "Switching Protocols", + /*102*/ "Processing", + /*103*/ "Early Hints", + + /*200*/ "OK", + /*201*/ "Created", + /*202*/ "Accepted", + /*203*/ "Non-Authoritative Information", + /*204*/ "No Content", + /*205*/ "Reset Content", + /*206*/ "Partial Content", + /*207*/ "Multi-Status", + /*208*/ "Already Reported", +// /*226*/ "IM Used", + + /*300*/ "Multiple Choices", + /*301*/ "Moved Permanently", + /*302*/ "Found", + /*303*/ "See Other", + /*304*/ "Not Modified", + /*305*/ "Use Proxy", + /*307*/ "Temporary Redirect", + /*308*/ "Permanent Redirect", + + /*400*/ "Bad Request", + /*401*/ "Unauthorized", + /*402*/ "Payment Required", + /*403*/ "Forbidden", + /*404*/ "Not Found", + /*405*/ "Method Not Allowed", + /*406*/ "Not Acceptable", + /*407*/ "Proxy Authentication Required", + /*408*/ "Request Timeout", + /*409*/ "Conflict", + /*410*/ "Gone", + /*411*/ "Length Required", + /*412*/ "Precondition Failed", + /*413*/ "Content Too Large", + /*414*/ "URI Too Long", + /*415*/ "Unsupported Media Type", + /*416*/ "Range Not Satisfiable", + /*417*/ "Expectation Failed", + /*418*/ "I'm a teapot", + /*419*/ {0, 0}, + /*420*/ {0, 0}, + /*421*/ "Misdirected Request", + /*422*/ "Unprocessable Content", + /*423*/ "Locked", + /*424*/ "Failed Dependency", + /*425*/ "Too Early", + /*426*/ "Upgrade Required", + /*427*/ {0, 0}, + /*428*/ "Precondition Required", + /*429*/ "Too Many Requests", + /*430*/ {0, 0}, + /*431*/ "Request Header Fields Too Large", +// /*451*/ "Unavailable For Legal Reasons", + + /*500*/ "Internal Server Error", + /*501*/ "Not Implemented", + /*502*/ "Bad Gateway", + /*503*/ "Service Unavailable", + /*504*/ "Gateway Timeout", + /*505*/ "HTTP Version Not Supported", + /*506*/ "Variant Also Negotiates", + /*507*/ "Insufficient Storage", + /*508*/ "Loop Detected", + /*509*/ {0, 0}, + /*510*/ "Not Extended", + /*511*/ "Network Authentication Required", +}; + +const static uint8_t LEN1xx = 4; +const static uint8_t LEN2xx = 9 + LEN1xx; +const static uint8_t LEN3xx = 9 + LEN2xx; +const static uint8_t LEN4xx = 32 + LEN3xx; +const static uint8_t LEN5xx = 12 + LEN4xx; +uint8_t entries[] = {0, LEN1xx, LEN2xx, LEN3xx, LEN4xx, LEN5xx}; + std::string_view photon::net::http::obsolete_reason(int code) { + uint8_t major = code / 100 - 1; + uint8_t minor = code % 100; + if (unlikely(major > 4)) return {}; + uint8_t max = entries[major+1] - entries[major]; + if (unlikely(minor >= max)) return {}; + return code_str[entries[major] + minor]; + +/* switch (code){ case 100: return "Continue"; @@ -87,6 +182,7 @@ std::string_view photon::net::http::obsolete_reason(int code) { case 510: return "Not Extended"; case 511: return "Network Authentication Required"; - default: return ""; + default: return { }; } +*/ } \ No newline at end of file From c088754c675db84fba566dd1dc417bb3a5ad5e94 Mon Sep 17 00:00:00 2001 From: lihuiba Date: Mon, 6 Nov 2023 17:22:55 +0800 Subject: [PATCH 2/4] fix --- net/http/status.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/http/status.cpp b/net/http/status.cpp index 8dcec7ec..849474fc 100644 --- a/net/http/status.cpp +++ b/net/http/status.cpp @@ -46,6 +46,7 @@ constexpr static SV code_str[] = { /*303*/ "See Other", /*304*/ "Not Modified", /*305*/ "Use Proxy", + /*306*/ {0, 0}, /*307*/ "Temporary Redirect", /*308*/ "Permanent Redirect", @@ -185,4 +186,4 @@ std::string_view photon::net::http::obsolete_reason(int code) { default: return { }; } */ -} \ No newline at end of file +} From fb53a2fe311dd10d705c299d9d04ac2a8f30db31 Mon Sep 17 00:00:00 2001 From: lihuiba Date: Thu, 16 Nov 2023 17:32:47 +0800 Subject: [PATCH 3/4] fix --- common/estring.cpp | 30 ++++++++++++++++++------------ common/estring.h | 34 ++++++++++++++++++++++++++++++---- common/test/test.cpp | 9 +++++++++ net/http/message.cpp | 2 +- 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/common/estring.cpp b/common/estring.cpp index 6822f891..2498c14d 100644 --- a/common/estring.cpp +++ b/common/estring.cpp @@ -55,27 +55,33 @@ size_t estring_view::find_last_not_of(const charset& set) const bool estring_view::to_uint64_check(uint64_t* v) const { - v ? (*v = 0) : 0; + uint64_t val = 0; for (unsigned char c : *this) { - if (c > '9' || c < '0') + c -= '0'; + if (c > 9) return false; - v ? (*v = *v * 10 + (c - '0')) : 0; + val = val * 10 + c; } + if (v) *v = val; return true; } -uint64_t estring_view::hex_to_uint64() const -{ +uint64_t estring_view::hex_to_uint64() const { uint64_t ret = 0; for (unsigned char c : *this) { - if (c >= '0' && c <= '9') { - ret = ret * 16 + (c - '0'); - } else if (c >= 'A' && c <= 'F') { - ret = ret * 16 + (c - 'A' + 10); - } else if (c >= 'a' && c <= 'f') { - ret = ret * 16 + (c - 'a' + 10); + unsigned char cc = c - '0'; + if (cc < 10) { + ret = ret * 16 + cc; } else { - return ret; + const unsigned char mask = 'a' - 'A'; + static_assert(mask == 32, "..."); // single digit + c |= mask; // unified to 'a'..'f' + cc = c - 'a'; + if (cc < 6) { + ret = ret * 16 + cc + 10; + } else { + break; // invalid char + } } } return ret; diff --git a/common/estring.h b/common/estring.h index 9e57214f..50d1024a 100644 --- a/common/estring.h +++ b/common/estring.h @@ -18,6 +18,7 @@ limitations under the License. #include #include #include +#include #include #include #include @@ -289,8 +290,33 @@ class estring_view : public std::string_view uint64_t to_uint64(uint64_t default_val = 0) const { uint64_t val; - to_uint64_check(&val); - return val; + return to_uint64_check(&val) ? val : default_val; + } + bool to_int64_check(int64_t* v = nullptr) const + { + if (this->empty()) return false; + if (this->front() != '-') return to_uint64_check((uint64_t*)v); + bool ret = this->substr(1).to_uint64_check((uint64_t*)v); + if (ret) *v = -*v; + return ret; + } + int64_t to_int64(int64_t default_val = 0) const + { + int64_t val; + return to_int64_check(&val) ? val : default_val; + } + bool to_double_check(double* v = nullptr) + { + char buf[32]; + auto len = std::max(this->size(), sizeof(buf) - 1 ); + memcpy(buf, data(), len); + buf[len] = '0'; + return sscanf(buf, "%lf", v) == 1; + } + double to_double(double default_val = NAN) + { + double val; + return to_double_check(&val) ? val : default_val; } // do not support 0x/0X prefix uint64_t hex_to_uint64() const; @@ -312,8 +338,8 @@ class rstring_view // relative string_view, that stores values relative protected: static_assert(std::is_integral::value, "..."); static_assert(std::is_integral::value, "..."); - OffsetType _offset; - LengthType _length; + OffsetType _offset = 0; + LengthType _length = 0; estring_view to_abs(const char* s) const { diff --git a/common/test/test.cpp b/common/test/test.cpp index 7c87049b..e9b2b27e 100644 --- a/common/test/test.cpp +++ b/common/test/test.cpp @@ -867,6 +867,15 @@ TEST(estring, test) estring as = " \tasdf \t\r\n"; auto trimmed = as.trim(); EXPECT_EQ(trimmed, "asdf"); + + EXPECT_EQ(estring_view("234423").to_uint64(), 234423); + EXPECT_EQ(estring_view("-234423").to_int64(), -234423); + EXPECT_EQ(estring_view("asfdsf").to_uint64(32), 32); + EXPECT_EQ(estring_view("-3.14").to_double(), -3.14); + EXPECT_EQ(estring_view("1e10").to_double(), 1e10); + + EXPECT_EQ(estring_view("1").hex_to_uint64(), 0x1); + EXPECT_EQ(estring_view("1a2b3d4e5f").hex_to_uint64(), 0x1a2b3d4e5f); } TEST(generator, example) diff --git a/net/http/message.cpp b/net/http/message.cpp index ab8e3179..82ffbd22 100644 --- a/net/http/message.cpp +++ b/net/http/message.cpp @@ -331,7 +331,7 @@ int Request::redirect(Verb v, estring_view location, bool enable_proxy) { } StoredURL u(location); auto new_request_line_size = verbstr[v].size() + sizeof(" HTTP/1.1\r\n") + - enable_proxy ? full_url_size(u) : u.target().size(); + (enable_proxy ? full_url_size(u) : u.target().size()); auto delta = new_request_line_size - m_buf_size; LOG_DEBUG(VALUE(delta)); From 5d5ecdc7b5a8a0ecb2b7ffd8e7e2255339b5a15b Mon Sep 17 00:00:00 2001 From: lihuiba Date: Fri, 17 Nov 2023 14:01:26 +0800 Subject: [PATCH 4/4] fix --- common/estring.cpp | 47 ++++++++++++++------------ common/estring.h | 9 +++-- net/http/message.cpp | 2 +- net/http/test/client_function_test.cpp | 10 +++--- net/utils.cpp | 40 ++++++++++------------ 5 files changed, 55 insertions(+), 53 deletions(-) diff --git a/common/estring.cpp b/common/estring.cpp index 2498c14d..54166380 100644 --- a/common/estring.cpp +++ b/common/estring.cpp @@ -55,36 +55,39 @@ size_t estring_view::find_last_not_of(const charset& set) const bool estring_view::to_uint64_check(uint64_t* v) const { - uint64_t val = 0; - for (unsigned char c : *this) { + if (this->empty()) return false; + uint64_t val = (*this)[0] - '0'; + if (val > 9) return false; + for (unsigned char c : this->substr(1)) { c -= '0'; - if (c > 9) - return false; + if (c > 9) break; val = val * 10 + c; } if (v) *v = val; return true; } -uint64_t estring_view::hex_to_uint64() const { - uint64_t ret = 0; - for (unsigned char c : *this) { - unsigned char cc = c - '0'; - if (cc < 10) { - ret = ret * 16 + cc; - } else { - const unsigned char mask = 'a' - 'A'; - static_assert(mask == 32, "..."); // single digit - c |= mask; // unified to 'a'..'f' - cc = c - 'a'; - if (cc < 6) { - ret = ret * 16 + cc + 10; - } else { - break; // invalid char - } - } +inline char hex_char_to_digit(char c) { + unsigned char cc = c - '0'; + if (cc < 10) return cc; + const unsigned char mask = 'a' - 'A'; + static_assert(mask == 32, "..."); // single digit + c |= mask; // unified to 'a'..'f' + cc = c - 'a'; + return (cc < 6) ? (cc + 10) : -1; +} + +bool estring_view::hex_to_uint64_check(uint64_t* v) const { + if (this->empty()) return false; + uint64_t val = hex_char_to_digit((*this)[0]); + if (val == -1ul) return false; + for (unsigned char c : this->substr(1)) { + auto d = hex_char_to_digit(c); + if (d == -1) break; + val = val * 16 + d; } - return ret; + if (v) *v = val; + return true; } estring& estring::append(uint64_t x) diff --git a/common/estring.h b/common/estring.h index 50d1024a..cdd90af7 100644 --- a/common/estring.h +++ b/common/estring.h @@ -318,8 +318,13 @@ class estring_view : public std::string_view double val; return to_double_check(&val) ? val : default_val; } - // do not support 0x/0X prefix - uint64_t hex_to_uint64() const; + // not including 0x/0X prefix + bool hex_to_uint64_check(uint64_t* v = nullptr) const; + uint64_t hex_to_uint64(uint64_t default_val = 0) const + { + uint64_t val; + return hex_to_uint64_check(&val) ? val : default_val; + } }; inline bool operator == (const std::string_view& sv, const std::string& s) diff --git a/net/http/message.cpp b/net/http/message.cpp index 82ffbd22..481c059f 100644 --- a/net/http/message.cpp +++ b/net/http/message.cpp @@ -368,7 +368,7 @@ int Response::parse_status_line(Parser &p) { p.skip_chars(' '); auto code = p.extract_integer(); if (code <= 0 || code >= 1000) - LOG_ERROR_RETURN(0, -1, "invalid status code"); + LOG_ERROR_RETURN(0, -1, "invalid status code ", code); m_status_code = (uint16_t)code; p.skip_chars(' '); m_status_message = p.extract_until_char('\r'); diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp index 7af2d2df..062f34c0 100644 --- a/net/http/test/client_function_test.cpp +++ b/net/http/test/client_function_test.cpp @@ -413,9 +413,9 @@ TEST(http_client, debug) { server->setsockopt(SOL_SOCKET, SO_REUSEPORT, 1); server->set_handler({nullptr, &chunked_handler_debug}); auto ret = server->bind(ep.port, ep.addr); - if (ret < 0) LOG_ERROR(VALUE(errno)); + if (ret < 0) LOG_ERROR(ERRNO()); ret |= server->listen(100); - if (ret < 0) LOG_ERROR(VALUE(errno)); + if (ret < 0) LOG_ERROR(ERRNO()); EXPECT_EQ(0, ret); LOG_INFO("Ready to accept"); server->start_loop(); @@ -438,13 +438,13 @@ TEST(http_client, debug) { memset((void*)buf.data(), '0', std_data_size); ret = op_test->resp.read((void*)buf.data(), std_data_size); EXPECT_EQ(std_data_size, ret); - EXPECT_EQ(true, buf == std_data); + EXPECT_TRUE(buf == std_data); for (int i = 0; i < buf.size(); i++) { if (buf[i] != std_data[i]) { - std::cout << i << std::endl; + LOG_ERROR("first occurrence of difference at: ", i); + break; } } - std::cout << "new" << std::endl; } int sleep_handler(void*, ISocketStream* sock) { photon::thread_sleep(3); diff --git a/net/utils.cpp b/net/utils.cpp index e1de9661..82985ffb 100644 --- a/net/utils.cpp +++ b/net/utils.cpp @@ -101,22 +101,25 @@ int _gethostbyname(const char* name, Delegate append_op) { return idx; } -inline __attribute__((always_inline)) void base64_translate_3to4(const char *in, char *out) { - struct xlator { - unsigned char _; - unsigned char a : 6; - unsigned char b : 6; - unsigned char c : 6; - unsigned char d : 6; - } __attribute__((packed)); - static_assert(sizeof(xlator) == 4, "..."); +struct xlator { + unsigned char _; + unsigned char a : 6; + unsigned char b : 6; + unsigned char c : 6; + unsigned char d : 6; +} __attribute__((packed)); +static_assert(sizeof(xlator) == 4, "..."); + +inline __attribute__((always_inline)) +void base64_translate_3to4(const char *in, char *out) { static const unsigned char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; auto v = htonl(*(uint32_t *)in); - auto x = *(xlator *)(&v); - *(uint32_t *)out = ((tbl[x.a] << 24) + (tbl[x.b] << 16) + - (tbl[x.c] << 8) + (tbl[x.d] << 0)); + auto x = (xlator*) &v; + *(uint32_t *)out = ((tbl[x->a] << 24) + (tbl[x->b] << 16) + + (tbl[x->c] << 8) + (tbl[x->d] << 0)); } + void Base64Encode(std::string_view in, std::string &out) { auto main = in.size() / 3; auto remain = in.size() % 3; @@ -137,7 +140,6 @@ void Base64Encode(std::string_view in, std::string &out) { base64_translate_3to4(_in + 9, _out + 12); } - for (; _in < end; _in += 3, _out += 4) { base64_translate_3to4(_in, _out); } @@ -191,16 +193,8 @@ static unsigned char get_index_of(char val, bool &ok) { } #undef EI +inline bool base64_translate_4to3(const char *in, char *out) { - struct xlator { - unsigned char _; - unsigned char a : 6; - unsigned char b : 6; - unsigned char c : 6; - unsigned char d : 6; - } __attribute__((packed)); - static_assert(sizeof(xlator) == 4, "..."); - xlator v; bool f1, f2, f3, f4; v.a = get_index_of(*(in+3), f1); @@ -208,10 +202,10 @@ bool base64_translate_4to3(const char *in, char *out) { v.c = get_index_of(*(in+1), f3); v.d = get_index_of(*(in), f4); - *(uint32_t *)out = ntohl(*(uint32_t *)&v); return (f1 && f2 && f3 && f4); } + bool Base64Decode(std::string_view in, std::string &out) { #define GSIZE 4 //Size of each group auto in_size = in.size();