diff --git a/include/arpa/inet.h b/include/arpa/inet.h new file mode 100644 index 00000000..e69de29b diff --git a/include/netinet/in.h b/include/netinet/in.h new file mode 100644 index 00000000..e69de29b diff --git a/include/netinet/ip.h b/include/netinet/ip.h new file mode 100644 index 00000000..e69de29b diff --git a/include/sys/socket.h b/include/sys/socket.h new file mode 100644 index 00000000..e69de29b diff --git a/include/sys/uio.h b/include/sys/uio.h new file mode 100644 index 00000000..71e8d839 --- /dev/null +++ b/include/sys/uio.h @@ -0,0 +1,16 @@ +#ifndef SYS_UIO_H +#define SYS_UIO_H + +#include +#include + +struct iovec +{ + void *iov_base; /* Base address of a memory region for input or output */ + size_t iov_len; /* The size of the memory pointed to by iov_base */ +}; + + ssize_t readv(int fildes, const struct iovec *iov, int iovcnt); +ssize_t writev(int fildes, const struct iovec *iov, int iovcnt); + +#endif /* SYS_UIO_H */ diff --git a/thread/test/test.cpp b/thread/test/test.cpp index 463dd63f..afaf6164 100644 --- a/thread/test/test.cpp +++ b/thread/test/test.cpp @@ -18,7 +18,7 @@ limitations under the License. #include #include #include -#include +#include #include #include #include @@ -73,7 +73,7 @@ uint64_t isqrt(uint64_t n) if (n == 0) return 0; - uint64_t x = 1L << (sizeof(n) * 8 / 2); + uint64_t x = 1ULL << (sizeof(n) * 8 / 2); while (true) { auto y = (x + n / x) / 2; @@ -773,7 +773,7 @@ thread_local photon::rwlock rwl; void *rwlocktest(void* args) { uint64_t carg = (uint64_t) args; - auto mode = carg & ((1UL<<32) -1); + auto mode = carg & ((1ULL<<32) -1); auto id = carg >> 32; // LOG_DEBUG("locking ", VALUE(id), VALUE(mode)); rwl.lock(mode); @@ -885,8 +885,8 @@ void vcpu_start(uint32_t i) TEST(saturated, add) { uint64_t cases[][3] = { - {10UL, -1UL, -1UL}, - {10UL, -9UL, -1UL}, + {10UL, -1ULL, -1ULL}, + {10UL, -9ULL, -1ULL}, {1UL, 2UL, 3UL}, {10UL, 3UL, 13UL}, {100UL, 4UL, 104UL}, @@ -1058,7 +1058,7 @@ photon::rwlock srwl; void *smprwlocktest(void* args) { uint64_t carg = (uint64_t) args; - auto mode = carg & ((1UL<<32) -1); + auto mode = carg & ((1ULL<<32) -1); auto id = carg >> 32; // LOG_DEBUG("locking ", VALUE(id), VALUE(mode)); srwl.lock(mode); @@ -1123,7 +1123,7 @@ void* test_smp_cvar_recver(void* args_) args->recvd++; } if (args->senders == 0) break; - thread_usleep(random() % 128); + thread_usleep(rand() % 128); } args->recvers--; return 0; @@ -1145,7 +1145,7 @@ void* test_smp_cvar_sender(void* args_) // LOG_DEBUG("notify_all()ed to ` threads", n); args->sent += n; } - thread_usleep(random() % 128); + thread_usleep(rand() % 128); } args->senders--; return 0; @@ -1160,7 +1160,7 @@ void* test_smp_cvar(void* args_) if (args->senders == 0) while(args->recvers > 0) { - thread_usleep(random() % 128); + thread_usleep(rand() % 128); auto n = args->cvar.notify_all(); args->sent += n; } @@ -1202,7 +1202,7 @@ void* test_smp_semaphore(void* args_) TEST(smp, semaphore) { smp_semaphore_args args; - auto n = args.n = 18UL; + auto n = args.n = 10UL; args.sem.signal(n/2); std_threads_create_join(n, &photon_do, n*n, &test_smp_semaphore, &args); @@ -1465,12 +1465,32 @@ TEST(workpool, async_work_lambda_threadpool_append) { LOG_INFO("DONE"); } +#define SAVE_REG(R) register uint64_t R asm(#R); volatile uint64_t saved_##R = R; +#define CHECK_REG(R) asm volatile ("" : "=r"(saved_##R) : "0"(saved_##R)); if (saved_##R != R) puts("differs after context switch!" #R); void* waiter(void* arg) { auto p = (int*)arg; - LOG_INFO("Start", VALUE(*p)); + LOG_INFO("Start", VALUE(p), VALUE(*p)); + SAVE_REG(rbx); + SAVE_REG(rsi); + SAVE_REG(rdi); + SAVE_REG(r12); + SAVE_REG(r13); + SAVE_REG(r14); + SAVE_REG(r15); + SAVE_REG(rbp); + SAVE_REG(rsp); photon::thread_usleep(1UL * 1000 * 1000); + CHECK_REG(rbx); + CHECK_REG(rsi); + CHECK_REG(rdi); + CHECK_REG(r12); + CHECK_REG(r13); + CHECK_REG(r14); + CHECK_REG(r15); + CHECK_REG(rbp); + CHECK_REG(rsp); (*p)--; - LOG_INFO("Fin", VALUE(*p)); + LOG_INFO("Fin", VALUE(p), VALUE(*p)); return nullptr; } diff --git a/thread/test/x.cpp b/thread/test/x.cpp index 24507d85..0c35e8d5 100644 --- a/thread/test/x.cpp +++ b/thread/test/x.cpp @@ -20,3 +20,18 @@ int DevNull(void* x, int) } int (*pDevNull)(void*, int) = &DevNull; + +#ifdef TEST + +#include "../thread.h" +namespace photon { +int init(uint64_t event_engine, uint64_t io_engine) { + return vcpu_init(); +} + +int fini() { + return vcpu_fini(); +} +} + +#endif diff --git a/thread/thread.cpp b/thread/thread.cpp index 5c7af3e9..431b7624 100644 --- a/thread/thread.cpp +++ b/thread/thread.cpp @@ -14,6 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +#define protected public +#include +#include +#include "list.h" +#undef protected + #include #include #include @@ -30,7 +36,7 @@ limitations under the License. #include #include inline int posix_memalign(void** memptr, size_t alignment, size_t size) { - auto ok = _aligned_malloc(size, alignment); + auto ok = malloc(size); if (!ok) return ENOMEM; *memptr = ok; @@ -40,12 +46,6 @@ inline int posix_memalign(void** memptr, size_t alignment, size_t size) { #include #endif -#define protected public -#include "thread.h" -#include "timer.h" -#include "list.h" -#undef protected - #include #include #include @@ -81,7 +81,8 @@ inline int posix_memalign(void** memptr, size_t alignment, size_t size) { #name": " #elif defined(_WIN64) #define DEF_ASM_FUNC(name) ".text\n .p2align 4\n" \ - ".def "#name"; .scl 3; .type 32; .endef" + ".def "#name"; .scl 3; .type 32; .endef\n" \ + #name": " #else #define DEF_ASM_FUNC(name) ".section .text."#name",\"axG\",@progbits,"#name",comdat\n" \ ".type "#name", @function\n" \ @@ -646,6 +647,12 @@ namespace photon } }; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + static_assert(offsetof(thread, arg) == 0x40, "..."); + static_assert(offsetof(thread, start) == 0x48, "..."); +#pragma GCC diagnostic pop + inline void thread::dequeue_ready_atomic(states newstat) { assert("this is not in runq, and this->lock is locked"); @@ -673,7 +680,7 @@ namespace photon static void _photon_thread_die(thread* th) asm("_photon_thread_die"); #if defined(__x86_64__) - +#if !defined(_WIN64) asm( DEF_ASM_FUNC(_photon_switch_context) // (void** rdi_to, void** rsi_from) R"( @@ -708,12 +715,6 @@ R"( )" ); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Winvalid-offsetof" - static_assert(offsetof(thread, arg) == 0x40, "..."); - static_assert(offsetof(thread, start) == 0x48, "..."); -#pragma GCC diagnostic pop - inline void switch_context(thread* from, thread* to) { prepare_switch(from, to); auto _t_ = to->stack.pointer_ref(); @@ -741,6 +742,73 @@ R"( : "rax", "rbx", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"); } +#else // _WIN64 + asm( +DEF_ASM_FUNC(_photon_switch_context) // (void** rcx_to, void** rdx_from) +R"( + push %rbp + mov %rsp, (%rdx) + mov (%rcx), %rsp + pop %rbp + ret +)" + +DEF_ASM_FUNC(_photon_switch_context_defer) // (void* rcx_arg, void (*rdx_defer)(void*), void** r8_to, void** r9_from) +R"( + push %rbp + mov %rsp, (%r9) +)" + +DEF_ASM_FUNC(_photon_switch_context_defer_die) // (void* rcx_arg, void (*rdx_defer)(void*), void** r8_to) +R"( + mov (%r8), %rsp + pop %rbp + jmp *%rdx +)" + +DEF_ASM_FUNC(_photon_thread_stub) +R"( + mov 0x40(%rbp), %rcx + movq $0, 0x40(%rbp) + call *0x48(%rbp) + mov %rax, 0x48(%rbp) + mov %rbp, %rcx + call _photon_thread_die +)" + ); + + inline void switch_context(thread* from, thread* to) { + prepare_switch(from, to); + auto _t_ = to->stack.pointer_ref(); + register auto f asm("rdx") = from->stack.pointer_ref(); + register auto t asm("rcx") = _t_; + asm volatile("call _photon_switch_context" // (to, from) + : "+r"(t), "+r"(f) + : // "0"(t), "1"(f) + : "rax", "rbx", "rsi", "rdi", "r8", "r9", + "r10", "r11", "r12", "r13", "r14", "r15", + "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", + "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); + } + + inline void switch_context_defer(thread* from, thread* to, + void (*defer)(void*), void* arg) { + prepare_switch(from, to); + auto _t_ = to->stack.pointer_ref(); + register auto f asm("r9") = from->stack.pointer_ref(); + register auto t asm("r8") = _t_; + register auto a asm("rcx") = arg; + register auto d asm("rdx") = defer; + asm volatile( + "call _photon_switch_context_defer" // (arg, defer, to, from) + : "+r"(t), "+r"(f), "+r"(a), "+r"(d) + : // "0"(t), "1"(f), "2"(a), "3"(d) + : "rax", "rbx", "rsi", "rdi", "r10", + "r11", "r12", "r13", "r14", "r15", + "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", + "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); + } +#endif // _WIN64 #elif defined(__aarch64__) || defined(__arm64__) @@ -1653,7 +1721,7 @@ R"( mark &= ~(RLOCK | WLOCK); mark |= mode; CURRENT->rwlock_mark = mark; - uint64_t op = (mode == RLOCK) ? (1UL << 63) : -1UL; + uint64_t op = (mode == RLOCK) ? (1ULL << 63) : -1ULL; if (cvar.q.th || (op & state)) { do { int ret = cvar.wait(lock, timeout); diff --git a/thread/thread11.h b/thread/thread11.h index bf2a1e99..bb9d7b27 100644 --- a/thread/thread11.h +++ b/thread/thread11.h @@ -45,9 +45,10 @@ namespace photon { return th; } - template + template struct FunctorWrapper { typename std::decay::type _obj; + template __attribute__((always_inline)) void operator()(ARGUMENTS&&...args) { _obj(std::forward(args)...); @@ -102,7 +103,7 @@ namespace photon { template inline _ENABLE_IF((is_functor::value)) thread_create11(uint64_t stack_size, FUNCTOR&& f, ARGUMENTS&&...args) { - using Wrapper = FunctorWrapper; + using Wrapper = FunctorWrapper; using SavedArgs = std::tuple::type ...>; return __thread_create11( stack_size, Wrapper{std::forward(f)},