From 8feced33dac3e860ff15052368c53c4e9833ba8d Mon Sep 17 00:00:00 2001 From: Bob Chen Date: Sat, 28 Oct 2023 01:12:45 +0800 Subject: [PATCH 1/4] backport auto-pr --- .github/workflows/auto-pr.yml | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/auto-pr.yml diff --git a/.github/workflows/auto-pr.yml b/.github/workflows/auto-pr.yml new file mode 100644 index 00000000..58e430c1 --- /dev/null +++ b/.github/workflows/auto-pr.yml @@ -0,0 +1,38 @@ +name: Auto PR + +on: + push: + branches: + - 'release/*' + +jobs: + auto-pr: + runs-on: ubuntu-latest + + env: + GH_TOKEN: ${{ github.token }} + + steps: + - uses: actions/checkout@v3 + + - name: Create Pull Request to the next higher release version + + run: | + git fetch + RELEASE_VERSIONS=$(git branch -r | grep 'release/' | cut -d '/' -f 3 | sort -V) + CURRENT_VERSION=$(echo $GITHUB_REF | rev | cut -d '/' -f 1 | rev) + ME_AND_MY_NEXT=$(echo "$RELEASE_VERSIONS" | grep -w $CURRENT_VERSION -A 1) + NUM=$(echo "$ME_AND_MY_NEXT" | wc -l) + if (( NUM > 1 )); then + NEXT_VERSION=$(echo "$ME_AND_MY_NEXT" | tail -n 1) + set -x + gh pr create --base "release/$NEXT_VERSION" --head "release/$CURRENT_VERSION" \ + --title "Auto PR from release/$CURRENT_VERSION to release/$NEXT_VERSION" \ + --body 'Created by Github action' + else + echo "No more higher release versions, will merge to main" + set -x + gh pr create --base main --head "release/$CURRENT_VERSION" \ + --title "Auto PR from release/$CURRENT_VERSION to main" \ + --body 'Created by Github action' + fi From 248b926839b10fce26844246911df358f74f2b1b Mon Sep 17 00:00:00 2001 From: Coldwings Date: Thu, 26 Oct 2023 10:05:18 +0800 Subject: [PATCH 2/4] Fix lockfree queue size and full check (#203) Signed-off-by: Coldwings --- common/lockfree_queue.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/lockfree_queue.h b/common/lockfree_queue.h index eb19262f..8bdcaac3 100644 --- a/common/lockfree_queue.h +++ b/common/lockfree_queue.h @@ -58,6 +58,9 @@ struct Capacity_2expN<0> { template <> struct Capacity_2expN<1> : public Capacity_2expN<0> {}; +template <> +struct Capacity_2expN<2> : public Capacity_2expN<0> {}; + struct PauseBase {}; struct CPUPause : PauseBase { @@ -136,7 +139,7 @@ class LockfreeRingQueueBase { bool check_empty(size_t h, size_t t) const { return h == t; } bool check_full(size_t h, size_t t) const { - return check_mask_equal(h, t + 1); + return h != t && check_mask_equal(h, t); } size_t idx(size_t x) const { return x & mask; } From 5ce5912279adf110a58c6d7c8d13483214fcfe59 Mon Sep 17 00:00:00 2001 From: Coldwings Date: Fri, 3 Nov 2023 13:32:48 +0800 Subject: [PATCH 3/4] Fix thread-pool join threads in multiple vCPUs (#228) Signed-off-by: Coldwings --- .github/workflows/ci.linux.arm.yml | 4 +- .github/workflows/ci.linux.x86.yml | 4 +- .github/workflows/ci.macos.arm.yml | 4 +- .github/workflows/ci.macos.yml | 4 +- common/identity-pool.cpp | 2 +- thread/thread-pool.cpp | 75 +++++++++++++++++++----------- thread/thread-pool.h | 3 ++ 7 files changed, 59 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml index d352064e..c7766b4b 100644 --- a/.github/workflows/ci.linux.arm.yml +++ b/.github/workflows/ci.linux.arm.yml @@ -2,9 +2,9 @@ name: Linux ARM on: push: - branches: [ "main" ] + branches: [ "main", "release/*" ] pull_request: - branches: [ "main" ] + branches: [ "main", "release/*" ] jobs: centos8-gcc921-epoll-release: diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml index 79cd2cb6..940ac7ca 100644 --- a/.github/workflows/ci.linux.x86.yml +++ b/.github/workflows/ci.linux.x86.yml @@ -2,9 +2,9 @@ name: Linux x86 on: push: - branches: [ "main" ] + branches: [ "main", "release/*" ] pull_request: - branches: [ "main" ] + branches: [ "main", "release/*" ] jobs: centos8-gcc921-epoll-release: diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml index 391ed891..c8e17718 100644 --- a/.github/workflows/ci.macos.arm.yml +++ b/.github/workflows/ci.macos.arm.yml @@ -2,9 +2,9 @@ name: macOS ARM on: push: - branches: [ "main" ] + branches: [ "main", "release/*" ] pull_request: - branches: [ "main" ] + branches: [ "main", "release/*" ] jobs: macOS-clang-debug: diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.yml index 2b40153e..5e75d552 100644 --- a/.github/workflows/ci.macos.yml +++ b/.github/workflows/ci.macos.yml @@ -2,9 +2,9 @@ name: macOS on: push: - branches: [ "main" ] + branches: [ "main", "release/*" ] pull_request: - branches: [ "main" ] + branches: [ "main", "release/*" ] jobs: macOS-12-Monterey-debug: diff --git a/common/identity-pool.cpp b/common/identity-pool.cpp index e523869a..d942d597 100644 --- a/common/identity-pool.cpp +++ b/common/identity-pool.cpp @@ -49,8 +49,8 @@ void IdentityPoolBase::put(void* obj) m_mtx.lock(); } --m_refcnt; + m_cvar.notify_all(); } - m_cvar.notify_all(); assert(m_size <= m_capacity); } diff --git a/thread/thread-pool.cpp b/thread/thread-pool.cpp index 7c0f7698..64a3e511 100644 --- a/thread/thread-pool.cpp +++ b/thread/thread-pool.cpp @@ -29,62 +29,81 @@ namespace photon pCtrl->joining = false; pCtrl->start = start; pCtrl->arg = arg; + pCtrl->cvar.notify_one(); } - pCtrl->cvar.notify_one(); return pCtrl; } + bool ThreadPoolBase::wait_for_work(TPControl &ctrl) + { + SCOPED_LOCK(ctrl.m_mtx); + while (!ctrl.start) // wait for `create()` to give me + ctrl.cvar.wait(ctrl.m_mtx); // thread_entry and argument + + if (ctrl.start == &stub) + return false; + + ((partial_thread*) CURRENT)->tls = nullptr; + return true; + } + bool ThreadPoolBase::after_work_done(TPControl &ctrl) + { + SCOPED_LOCK(ctrl.m_mtx); + auto ret = !ctrl.joining; + if (ctrl.joining) { + assert(ctrl.joinable); + ctrl.cvar.notify_all(); + } else if (ctrl.joinable) { + ctrl.joining = true; + ctrl.cvar.wait(ctrl.m_mtx); + } + ctrl.joinable = false; + ctrl.joining = false; + ctrl.start = nullptr; + return ret; + } void* ThreadPoolBase::stub(void* arg) { TPControl ctrl; auto th = *(thread**)arg; *(TPControl**)arg = &ctrl; // tell ctor where my `ctrl` is thread_yield_to(th); - while(true) + while(wait_for_work(ctrl)) { - { - SCOPED_LOCK(ctrl.m_mtx); - while (!ctrl.start) // wait for `create()` to give me - ctrl.cvar.wait(ctrl.m_mtx); // thread_entry and argument - - if (ctrl.start == &stub) - break; - ((partial_thread*) CURRENT)->tls = nullptr; - } ctrl.start(ctrl.arg); deallocate_tls(); - { - SCOPED_LOCK(ctrl.m_mtx); - if (ctrl.joining) { - assert(ctrl.joinable); - ctrl.cvar.notify_all(); - } else if (ctrl.joinable) { - ctrl.joining = true; - ctrl.cvar.wait(ctrl.m_mtx); - } - ctrl.joinable = false; - ctrl.joining = false; - ctrl.start = nullptr; + auto should_put = after_work_done(ctrl); + if (should_put) { + // if has no other joiner waiting + // collect it into pool + ctrl.pool->put(&ctrl); } - ctrl.pool->put(&ctrl); } return nullptr; } - void ThreadPoolBase::join(TPControl* pCtrl) + bool ThreadPoolBase::do_thread_join(TPControl* pCtrl) { SCOPED_LOCK(pCtrl->m_mtx); if (!pCtrl->joinable) - LOG_ERROR_RETURN(EINVAL, , "thread is not joinable"); + LOG_ERROR_RETURN(EINVAL, false, "thread is not joinable"); if (!pCtrl->start) - LOG_ERROR_RETURN(EINVAL, , "thread is not running"); + LOG_ERROR_RETURN(EINVAL, false, "thread is not running"); if (pCtrl->start == &stub) - LOG_ERROR_RETURN(EINVAL, , "thread is dying"); + LOG_ERROR_RETURN(EINVAL, false, "thread is dying"); + auto ret = !pCtrl->joining; if (pCtrl->joining) { pCtrl->cvar.notify_one(); } else { pCtrl->joining = true; pCtrl->cvar.wait(pCtrl->m_mtx); } + return ret; + } + void ThreadPoolBase::join(TPControl* pCtrl) + { + auto should_put = do_thread_join(pCtrl); + if (should_put) + pCtrl->pool->put(pCtrl); } int ThreadPoolBase::ctor(ThreadPoolBase* pool, TPControl** out) { diff --git a/thread/thread-pool.h b/thread/thread-pool.h index a41405be..448f29c9 100644 --- a/thread/thread-pool.h +++ b/thread/thread-pool.h @@ -67,6 +67,9 @@ namespace photon static void* stub(void* arg); static int ctor(ThreadPoolBase*, TPControl**); static int dtor(ThreadPoolBase*, TPControl*); + static bool wait_for_work(TPControl &ctrl); + static bool after_work_done(TPControl &ctrl); + static bool do_thread_join(TPControl* pCtrl); void init(uint64_t stack_size) { set_ctor({this, &ctor}); From b07b72348fa909db2880b4049c31865491752555 Mon Sep 17 00:00:00 2001 From: Coldwings Date: Fri, 17 Nov 2023 16:03:15 +0800 Subject: [PATCH 4/4] Fix httpfs may return wrong received body length if http socket disconnected during body transfering (#254) Signed-off-by: Coldwings --- fs/httpfs/httpfs.cpp | 3 +-- net/curl.h | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/httpfs/httpfs.cpp b/fs/httpfs/httpfs.cpp index 5a6ff0d7..4ef3e207 100644 --- a/fs/httpfs/httpfs.cpp +++ b/fs/httpfs/httpfs.cpp @@ -229,8 +229,7 @@ class HttpFile : public fs::VirtualReadOnlyFile { VALUE(url), VALUE(offset), VALUE(ret)); } authorized = true; - headers.try_get("content-length", ret); - return ret; + return writer.written; } int fstat(struct stat* buf) override { diff --git a/net/curl.h b/net/curl.h index 65dee2f2..37fee819 100644 --- a/net/curl.h +++ b/net/curl.h @@ -145,9 +145,11 @@ class StringWriter { struct IOVWriter : public IOVector { using IOVector::IOVector; size_t drop = 0; + size_t written = 0; size_t write(const void* buf, size_t size) { auto actual_count = memcpy_from( buf, size); // copy from (buf, size) to iovec[] in *this, + written += actual_count; extract_front(actual_count); // and extract the portion just copied if (actual_count < size) // means full {