diff --git a/CMake/build-from-src.cmake b/CMake/build-from-src.cmake index 646dc9f3..dad13278 100644 --- a/CMake/build-from-src.cmake +++ b/CMake/build-from-src.cmake @@ -12,7 +12,7 @@ function(build_from_src [dep]) URL_MD5 605237f35de238dfacc83bcae406d95d BUILD_IN_SOURCE ON CONFIGURE_COMMAND "" - BUILD_COMMAND make prefix=${BINARY_DIR} install -j + BUILD_COMMAND $(MAKE) prefix=${BINARY_DIR} install INSTALL_COMMAND "" ) set(AIO_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE) @@ -26,8 +26,8 @@ function(build_from_src [dep]) URL_MD5 9b8aa094c4e5765dabf4da391f00d15c BUILD_IN_SOURCE ON CONFIGURE_COMMAND CFLAGS=-fPIC ./configure --prefix=${BINARY_DIR} --static - BUILD_COMMAND make -j - INSTALL_COMMAND make install + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install ) set(ZLIB_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE) set(ZLIB_LIBRARIES ${BINARY_DIR}/lib/libz.a PARENT_SCOPE) @@ -40,8 +40,8 @@ function(build_from_src [dep]) URL_MD5 2e8c3c23795415475654346484f5c4b8 BUILD_IN_SOURCE ON CONFIGURE_COMMAND ./configure --prefix=${BINARY_DIR} - BUILD_COMMAND V=1 CFLAGS=-fPIC make -C src - INSTALL_COMMAND make install + BUILD_COMMAND V=1 CFLAGS=-fPIC $(MAKE) -C src + INSTALL_COMMAND $(MAKE) install ) set(URING_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE) set(URING_LIBRARIES ${BINARY_DIR}/lib/liburing.a PARENT_SCOPE) @@ -83,10 +83,12 @@ function(build_from_src [dep]) URL_MD5 bad68bb6bd9908da75e2c8dedc536b29 BUILD_IN_SOURCE ON CONFIGURE_COMMAND ./config -fPIC --prefix=${BINARY_DIR} --openssldir=${BINARY_DIR} shared - BUILD_COMMAND make -j ${NumCPU} - INSTALL_COMMAND make install + BUILD_COMMAND make -j 1 # https://github.com/openssl/openssl/issues/5762#issuecomment-376622684 + INSTALL_COMMAND $(MAKE) install + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON ) - ExternalProject_Get_Property(openssl SOURCE_DIR) set(OPENSSL_ROOT_DIR ${BINARY_DIR} PARENT_SCOPE) set(OPENSSL_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE) set(OPENSSL_LIBRARIES ${BINARY_DIR}/lib/libssl.a ${BINARY_DIR}/lib/libcrypto.a PARENT_SCOPE) @@ -101,7 +103,7 @@ function(build_from_src [dep]) URL ${PHOTON_CURL_SOURCE} URL_MD5 a66270f11e3fbfad709600bbd1686704 BUILD_IN_SOURCE ON - CONFIGURE_COMMAND autoreconf -i && ./configure --with-ssl=${OPENSSL_ROOT_DIR} + CONFIGURE_COMMAND autoreconf -i COMMAND ./configure --with-ssl=${OPENSSL_ROOT_DIR} --without-libssh2 --enable-static --enable-shared=no --enable-optimize --disable-manual --without-libidn --disable-ftp --disable-file --disable-ldap --disable-ldaps @@ -109,8 +111,12 @@ function(build_from_src [dep]) --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --without-nghttp2 --enable-http --disable-verbose --with-pic=PIC --prefix=${BINARY_DIR} - BUILD_COMMAND make -j ${NumCPU} - INSTALL_COMMAND make install + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + DEPENDS openssl + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON ) set(CURL_INCLUDE_DIRS ${BINARY_DIR}/include PARENT_SCOPE) set(CURL_LIBRARIES ${BINARY_DIR}/lib/libcurl.a PARENT_SCOPE) diff --git a/common/utility.h b/common/utility.h index afeb6fb0..3363d102 100644 --- a/common/utility.h +++ b/common/utility.h @@ -74,12 +74,15 @@ ptr_array_t ptr_array(T* pbegin, size_t n) return {pbegin, pbegin + n}; } +#define __INLINE__ __attribute__((always_inline)) +#define __FORCE_INLINE__ __INLINE__ inline + template class Defer { public: - Defer(T fn) : m_func(fn) {} - ~Defer() { m_func(); } + Defer(T fn) __INLINE__ : m_func(fn) {} + ~Defer() __INLINE__ { m_func(); } void operator=(const Defer&) = delete; operator bool () { return true; } // if (DEFER(...)) { ... } @@ -87,12 +90,9 @@ class Defer T m_func; }; -template +template __FORCE_INLINE__ Defer make_defer(T func) { return Defer(func); } -#define __INLINE__ __attribute__((always_inline)) -#define __FORCE_INLINE__ __INLINE__ inline - #define _CONCAT_(a, b) a##b #define _CONCAT(a, b) _CONCAT_(a, b) diff --git a/fs/fuse_adaptor.cpp b/fs/fuse_adaptor.cpp index 655eb55a..f6821712 100644 --- a/fs/fuse_adaptor.cpp +++ b/fs/fuse_adaptor.cpp @@ -80,13 +80,21 @@ namespace photon { namespace fs{ static IFileSystem* fs = nullptr; +static uid_t cuid = 0, cgid = 0; #define CHECK_FS() if (!fs) return -EFAULT; static void* xmp_init(struct fuse_conn_info *conn) { REPORT_PERF(xmp_init, 1) - (void) conn; + cuid = geteuid(); + cgid = getegid(); + if ((unsigned int)conn->capable & FUSE_CAP_ATOMIC_O_TRUNC) { + conn->want |= FUSE_CAP_ATOMIC_O_TRUNC; + } + if ((unsigned int)conn->capable & FUSE_CAP_BIG_WRITES) { + conn->want |= FUSE_CAP_BIG_WRITES; + } return NULL; } @@ -102,9 +110,13 @@ static int xmp_getattr(const char *path, struct stat *stbuf) CHECK_FS(); errno = 0; int res = fs->lstat(path, stbuf); - if(res) \ - LOG_ERROR_RETURN(0, errno ? -errno : res, VALUE(path)); - errno = 0; + if(res) return -errno; + stbuf->st_blocks = stbuf->st_size / 512 + 1; // du relies on this + stbuf->st_blksize = 4096; + // use current user/group when backendfs doesn't support uid/gid + if (stbuf->st_uid == 0) stbuf->st_uid = cuid; + if (stbuf->st_gid == 0) stbuf->st_gid = cgid; + if (stbuf->st_nlink == 0) stbuf->st_nlink = 1; return 0; } diff --git a/net/http/client.cpp b/net/http/client.cpp index 59f08faf..57fac913 100644 --- a/net/http/client.cpp +++ b/net/http/client.cpp @@ -91,7 +91,7 @@ ISocketStream* PooledDialer::dial(std::string_view host, uint16_t port, bool sec if (ipaddr.undefined()) LOG_DEBUG("No connectable resolve result"); // When failed, remove resolved result from dns cache so that following retries can try // different ips. - resolver->discard_cache(strhost.c_str()); + resolver->discard_cache(strhost.c_str(), ipaddr); return nullptr; } @@ -120,9 +120,7 @@ class ClientImpl : public Client { CommonHeaders<> m_common_headers; ICookieJar *m_cookie_jar; ClientImpl(ICookieJar *cookie_jar, TLSContext *tls_ctx) : - m_cookie_jar(cookie_jar), - m_dialer(tls_ctx) { - } + m_dialer(tls_ctx), m_cookie_jar(cookie_jar) { } using SocketStream_ptr = std::unique_ptr; int redirect(Operation* op) { diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp index 1fb7cf9f..8e0b48cd 100644 --- a/net/http/test/client_function_test.cpp +++ b/net/http/test/client_function_test.cpp @@ -513,6 +513,16 @@ TEST(http_client, partial_body) { EXPECT_EQ(true, buf == "http_clien"); } +TEST(DISABLED_http_client, ipv6) { // make sure runing in a ipv6-ready environment + auto client = new_http_client(); + DEFER(delete client); + // here is an ipv6-only website + auto op = client->new_operation(Verb::GET, "http://test6.ustc.edu.cn"); + DEFER(delete op); + op->call(); + EXPECT_EQ(200, op->resp.status_code()); +} + TEST(url, url_escape_unescape) { EXPECT_EQ( url_escape("?a=x:b&b=cd&c= feg&d=2/1[+]@alibaba.com&e='!bad';"), diff --git a/net/kernel_socket.cpp b/net/kernel_socket.cpp index 59749dd8..08715afa 100644 --- a/net/kernel_socket.cpp +++ b/net/kernel_socket.cpp @@ -191,9 +191,7 @@ class KernelSocketStream : public SocketStreamBase { class KernelSocketClient : public SocketClientBase { public: - KernelSocketClient(int socket_family, bool nonblocking) : - m_socket_family(socket_family), - m_nonblocking(nonblocking) {} + KernelSocketClient(bool nonblocking) : m_nonblocking(nonblocking) {} ISocketStream* connect(const char* path, size_t count) override { struct sockaddr_un addr_un; @@ -211,11 +209,10 @@ class KernelSocketClient : public SocketClientBase { } protected: - int m_socket_family; bool m_nonblocking; - virtual KernelSocketStream* create_stream() { - return new KernelSocketStream(m_socket_family, m_nonblocking); + virtual KernelSocketStream* create_stream(int socket_family) { + return new KernelSocketStream(socket_family, m_nonblocking); } virtual int fd_connect(int fd, const sockaddr* remote, socklen_t addrlen) { @@ -224,7 +221,7 @@ class KernelSocketClient : public SocketClientBase { ISocketStream* do_connect(const sockaddr* remote, socklen_t len_remote, const sockaddr* local = nullptr, socklen_t len_local = 0) { - auto stream = create_stream(); + auto stream = create_stream(remote->sa_family); auto deleter = [&](KernelSocketStream*) { auto errno_backup = errno; delete stream; @@ -504,8 +501,8 @@ class ZeroCopySocketClient : public KernelSocketClient { public: using KernelSocketClient::KernelSocketClient; protected: - KernelSocketStream* create_stream() override { - return new ZeroCopySocketStream(m_socket_family, m_nonblocking); + KernelSocketStream* create_stream(int socket_family) override { + return new ZeroCopySocketStream(socket_family, m_nonblocking); } }; @@ -544,8 +541,8 @@ class IouringSocketClient : public KernelSocketClient { using KernelSocketClient::KernelSocketClient; protected: - KernelSocketStream* create_stream() override { - return new IouringSocketStream(m_socket_family, m_nonblocking); + KernelSocketStream* create_stream(int socket_family) override { + return new IouringSocketStream(socket_family, m_nonblocking); } int fd_connect(int fd, const sockaddr* remote, socklen_t addrlen) override { @@ -614,8 +611,8 @@ class IouringFixedFileSocketClient : public IouringSocketClient { protected: using IouringSocketClient::IouringSocketClient; - KernelSocketStream* create_stream() override { - return new IouringFixedFileSocketStream(m_socket_family, m_nonblocking); + KernelSocketStream* create_stream(int socket_family) override { + return new IouringFixedFileSocketStream(socket_family, m_nonblocking); } }; @@ -683,8 +680,8 @@ class FstackDpdkSocketClient : public KernelSocketClient { protected: using KernelSocketClient::KernelSocketClient; - KernelSocketStream* create_stream() override { - return new FstackDpdkSocketStream(m_socket_family, m_nonblocking); + KernelSocketStream* create_stream(int socket_family) override { + return new FstackDpdkSocketStream(socket_family, m_nonblocking); } int fd_connect(int fd, const sockaddr* remote, socklen_t addrlen) override { @@ -939,8 +936,8 @@ class ETKernelSocketClient : public KernelSocketClient { using KernelSocketClient::KernelSocketClient; protected: - KernelSocketStream* create_stream() override { - return new ETKernelSocketStream(m_socket_family, m_nonblocking); + KernelSocketStream* create_stream(int socket_family) override { + return new ETKernelSocketStream(socket_family, m_nonblocking); } }; @@ -974,10 +971,10 @@ class ETKernelSocketServer : public KernelSocketServer, public NotifyContext { /* ET Socket - End */ extern "C" ISocketClient* new_tcp_socket_client() { - return new KernelSocketClient(AF_INET, true); + return new KernelSocketClient(true); } extern "C" ISocketClient* new_tcp_socket_client_ipv6() { - return new KernelSocketClient(AF_INET6, true); + return new KernelSocketClient(true); } extern "C" ISocketServer* new_tcp_socket_server() { return NewObj(AF_INET, false, true)->init(); @@ -986,7 +983,7 @@ extern "C" ISocketServer* new_tcp_socket_server_ipv6() { return NewObj(AF_INET6, false, true)->init(); } extern "C" ISocketClient* new_uds_client() { - return new KernelSocketClient(AF_UNIX, true); + return new KernelSocketClient(true); } extern "C" ISocketServer* new_uds_server(bool autoremove) { return NewObj(AF_UNIX, autoremove, true)->init(); @@ -996,14 +993,14 @@ extern "C" ISocketServer* new_zerocopy_tcp_server() { return NewObj(AF_INET, false, true)->init(); } extern "C" ISocketClient* new_zerocopy_tcp_client() { - return new ZeroCopySocketClient(AF_INET, true); + return new ZeroCopySocketClient(true); } #ifdef PHOTON_URING extern "C" ISocketClient* new_iouring_tcp_client() { if (photon::iouring_register_files_enabled()) - return new IouringFixedFileSocketClient(AF_INET, false); + return new IouringFixedFileSocketClient(false); else - return new IouringSocketClient(AF_INET, false); + return new IouringSocketClient(false); } extern "C" ISocketServer* new_iouring_tcp_server() { if (photon::iouring_register_files_enabled()) @@ -1013,23 +1010,23 @@ extern "C" ISocketServer* new_iouring_tcp_server() { } #endif // PHOTON_URING extern "C" ISocketClient* new_et_tcp_socket_client() { - return new ETKernelSocketClient(AF_INET, true); + return new ETKernelSocketClient(true); } extern "C" ISocketServer* new_et_tcp_socket_server() { return NewObj(AF_INET, false, true)->init(); } extern "C" ISocketClient* new_smc_socket_client() { - return new KernelSocketClient(AF_SMC, true); + return new KernelSocketClient(true); } extern "C" ISocketServer* new_smc_socket_server() { return NewObj(AF_SMC, false, true)->init(); } #ifdef ENABLE_FSTACK_DPDK extern "C" ISocketClient* new_fstack_dpdk_socket_client() { - return new FstackDpdkSocketClient(AF_INET, true); + return new FstackDpdkSocketClient(true); } extern "C" ISocketServer* new_fstack_dpdk_socket_server() { - return NewObj(AF_INET, false, true)->init(); + return NewObj(false, true)->init(); } #endif // ENABLE_FSTACK_DPDK #endif // __linux__ diff --git a/net/test/test.cpp b/net/test/test.cpp index a61d6f72..ad182a06 100644 --- a/net/test/test.cpp +++ b/net/test/test.cpp @@ -714,9 +714,9 @@ TEST(utils, resolver) { DEFER(delete resolver); net::IPAddr localhost("127.0.0.1"); net::IPAddr addr = resolver->resolve("localhost"); - EXPECT_EQ(localhost.to_nl(), addr.to_nl()); + if (addr.is_ipv4()) EXPECT_EQ(localhost.to_nl(), addr.to_nl()); auto func = [&](net::IPAddr addr_){ - EXPECT_EQ(localhost.to_nl(), addr_.to_nl()); + if (addr_.is_ipv4()) EXPECT_EQ(localhost.to_nl(), addr_.to_nl()); }; resolver->resolve("localhost", func); resolver->discard_cache("non-exist-host.com"); diff --git a/net/utils.cpp b/net/utils.cpp index 82985ffb..ad9209d8 100644 --- a/net/utils.cpp +++ b/net/utils.cpp @@ -21,6 +21,8 @@ limitations under the License. #include #include +#include +#include #include #include @@ -249,49 +251,68 @@ bool Base64Decode(std::string_view in, std::string &out) { } class DefaultResolver : public Resolver { +protected: + struct IPAddrNode : public intrusive_list_node { + IPAddr addr; + IPAddrNode(IPAddr addr) : addr(addr) {} + }; + using IPAddrList = intrusive_list; public: DefaultResolver(uint64_t cache_ttl, uint64_t resolve_timeout) : dnscache_(cache_ttl), resolve_timeout_(resolve_timeout) {} - ~DefaultResolver() { dnscache_.clear(); } + ~DefaultResolver() { + for (auto it : dnscache_) { + ((IPAddrList*)it->_obj)->delete_all(); + } + dnscache_.clear(); + } IPAddr resolve(const char *host) override { - auto ctr = [&]() -> IPAddr * { - std::vector addrs; + auto ctr = [&]() -> IPAddrList* { + auto addrs = new IPAddrList(); photon::semaphore sem; std::thread([&]() { - int ret = gethostbyname(host, addrs); - if (ret < 0) { - addrs.clear(); + auto now = std::chrono::steady_clock::now(); + IPAddrList ret; + auto cb = [&](IPAddr addr) { + ret.push_back(new IPAddrNode(addr)); + return 0; + }; + _gethostbyname(host, cb); + auto time_elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - now).count(); + if ((uint64_t)time_elapsed <= resolve_timeout_) { + addrs->push_back(std::move(ret)); + sem.signal(1); } - sem.signal(1); }).detach(); - auto ret = sem.wait(1, resolve_timeout_); - if (ret < 0 && errno == ETIMEDOUT) { - LOG_WARN("Domain resolution for ` timeout!", host); - return new IPAddr; // undefined addr - } else if (addrs.empty()) { - LOG_WARN("Domain resolution for ` failed", host); - return new IPAddr; // undefined addr - } - // TODO: support ipv6 - for (auto& ip : addrs) { - if (ip.is_ipv4()) - return new IPAddr(ip); - } - return new IPAddr; // undefined addr + sem.wait(1, resolve_timeout_); + return addrs; }; - return *(dnscache_.borrow(host, ctr)); + auto ips = dnscache_.borrow(host, ctr); + if (ips->empty()) LOG_ERRNO_RETURN(0, IPAddr(), "Domain resolution for ` failed", host); + auto ret = ips->front(); + ips->node = ret->next(); // access in round robin order + return ret->addr; } void resolve(const char *host, Delegate func) override { func(resolve(host)); } - void discard_cache(const char *host) override { + void discard_cache(const char *host, IPAddr ip) override { auto ipaddr = dnscache_.borrow(host); - ipaddr.recycle(true); + if (ip.undefined() || ipaddr->empty()) ipaddr.recycle(true); + else { + for (auto itr = ipaddr->rbegin(); itr != ipaddr->rend(); itr++) { + if ((*itr)->addr == ip) { + ipaddr->erase(*itr); + break; + } + } + } } private: - ObjectCache dnscache_; + ObjectCache dnscache_; uint64_t resolve_timeout_; }; diff --git a/net/utils.h b/net/utils.h index 3f63f27c..bf61c3b0 100644 --- a/net/utils.h +++ b/net/utils.h @@ -157,11 +157,12 @@ class Resolver : public Object { // Normally dns servers return multiple ips in random order, choosing the first one should suffice. virtual IPAddr resolve(const char* host) = 0; virtual void resolve(const char* host, Delegate func) = 0; - virtual void discard_cache(const char* host) = 0; // discard current cache of host:ip + virtual void discard_cache(const char* host, IPAddr ip = IPAddr()) = 0; // discard current cache of ip }; /** * @brief A non-blocking Resolver based on gethostbyname. + * Currently, it's not thread safe. * * @param cache_ttl cache's lifetime in microseconds. * @param resolve_timeout timeout in microseconds for domain resolution. diff --git a/thread/list.h b/thread/list.h index 435f8aa8..3f42a8de 100644 --- a/thread/list.h +++ b/thread/list.h @@ -221,6 +221,48 @@ class intrusive_list_node : public __intrusive_list_node { return {nullptr, nullptr}; } + + struct reverse_iterator + { + __intrusive_list_node* ptr; + __intrusive_list_node* end; + T* operator*() + { + return static_cast(ptr); + } + reverse_iterator& operator++() + { + ptr = ptr->__prev_ptr; + if (ptr == end) + ptr = nullptr; + return *this; + } + reverse_iterator operator++(int) + { + auto rst = *this; + ptr = ptr->__prev_ptr; + if (ptr == end) + ptr = nullptr; + return rst; + } + bool operator == (const reverse_iterator& rhs) const + { + return ptr == rhs.ptr; + } + bool operator != (const reverse_iterator& rhs) const + { + return !(*this == rhs); + } + }; + + reverse_iterator rbegin() + { + return {this->__prev_ptr, this->__prev_ptr}; + } + reverse_iterator rend() + { + return {nullptr, nullptr}; + } }; @@ -321,6 +363,7 @@ class intrusive_list return node == nullptr; } typedef typename NodeType::iterator iterator; + typedef typename NodeType::reverse_iterator reverse_iterator; iterator begin() { return node ? node->begin() : end(); @@ -329,6 +372,14 @@ class intrusive_list { return {nullptr, nullptr}; } + reverse_iterator rbegin() + { + return node ? node->rbegin() : rend(); + } + reverse_iterator rend() + { + return {nullptr, nullptr}; + } intrusive_list split_front_inclusive(NodeType* ptr) { auto ret = node;