upstream pretest.yaml

name: Upstream pretest

on:
    push:
        branches-ignore:
          - 'subm-pretest/**'
    schedule:
        - cron: '0 0 * * *'
    workflow_dispatch:

permissions:
    contents: write

concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: false

# There are 2 steps of adding new upstream commits:
# 1. Add new commits from the PR head ref (and, if needed, main/custom branches): to "pretest/$pr_num" in fork
# 2. Add commit with test changes: to "subm-pretest/$org_repo/$pr_num" in POCGL repo

# To only update when source has been updated:
# - "pretest" references sha of latest PR commit as: meta.pr_head_sha
# - "pretest" references sha of latest "subm-pretest" commit as: meta.core_pretest_sha

# "pretest" is not updated when only main or custom branch of subm updates
# - Otherwise all pretest branches would update at the same time, even on unrelated changes

# "pretest" needs to be first checked by a separate "git ls-remote" before "git fetch"
# - Otherwise git fetch would fail fetching the missing branches

# "pretest" and "subm-pretest" can be update manually by push, in case anything new needs to be implemented for the new XML spec
# - In that case, there would be no commit sha reference, so it will be re-tested on the next run of this workflow

jobs:
    enmr-PRs:
        runs-on: windows-latest
        outputs:
            exec_list: ${{ steps.make_exec_list.outputs.exec_list }}
            remove_list: ${{ steps.make_exec_list.outputs.remove_list }}
        
        steps:
            -   name: git config
                run: |
                
                    git config --global gc.auto 0
                    git config --global core.autocrlf false
                    git config --global user.name "sun pack bot"
                    git config --global user.email "sunserega2@gmail.com"
                    git config --global --add url.https://github.com/.insteadOf "git@github.com:"
            
            -   name: checkout
                uses: actions/checkout@main
                with:
                    sparse-checkout-cone-mode: false
                    sparse-checkout: |
                        .gitmodules
            
            -   name: (!) make exec list
                id: make_exec_list
                run: |
                
                    function Get-GitRemoteBranches() {
                        param ($url, $pattern)
                        $reg_pattern = $pattern -replace '\*','(.*)'
                        $res = @()
                        foreach ($l in git ls-remote $url $pattern) {
                            if ($l -notmatch "(^[0-9a-f]{40})\s+($reg_pattern)$") {
                                throw "Unexpected git ls-remote output: $l"
                            }
                            $sha = $matches[1]
                            $branch = $matches[2]
                            if ($branch -notmatch "^$reg_pattern$") {
                                throw "What?"
                            }
                            $res += [PSCustomObject]@{
                                'sha' = $sha;
                                'branch' = $branch;
                                'matches' = $matches;
                            }
                        }
                        if (-not $?) { throw "git ls-remote failed" }
                        return $res
                    }
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Bulding open PR list"
                    $submodules = @()
                    $submodule_by_repo = @{}
                    foreach ($config_key in git config --file .gitmodules --name-only --get-regexp '^submodule\..+\.url$') {
                        $url = git config --file .gitmodules --get $config_key
                        Write-Host "URL: $url"
                        
                        if ($url -notmatch '^git@github\.com:([\w\-]+)/([\w\-]+)\.git$') {
                            throw "Unexpected url format: $url"
                        }
                        $owner = $matches[1]
                        $name = $matches[2]
                        $fork_url = "git@github.com:SunSerega/${name}.git"
                        
                        $mergeable_prs = @()
                        foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/merge') {
                            $pr_num = $l.matches[1]
                            $mergeable_prs += $pr_num
                        }
                        
                        $head_sha_by_pr = @{}
                        foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/head') {
                            $sha = $l.sha
                            $pr_num = $l.matches[1]
                            if ($pr_num -notin $mergeable_prs) { continue }
                            Write-Host "- Found open PR $pr_num with latest sha $sha"
                            $head_sha_by_pr.Add($pr_num, $sha)
                        }
                        
                        $pretest_prs = @()
                        foreach ($l in Get-GitRemoteBranches $fork_url 'refs/heads/pretest/*') {
                            $sha = $l.sha
                            $pr_num = $l.matches[1]
                            Write-Host "- Found pretest branch for PR $pr_num with latest sha $sha"
                            $pretest_prs += $pr_num
                        }
                        
                        $subm = [PSCustomObject]@{
                            'url'               = $url;
                            'fork_url'          = $fork_url;
                            'owner'             = $owner;
                            'name'              = $name;
                            'repo'              = "$owner/$name"
                            'head_sha_by_pr'    = $head_sha_by_pr;
                            'pretest_prs'       = $pretest_prs;
                        }
                        $submodules += $subm
                        $submodule_by_repo.Add($subm.repo, $subm)
                    }
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Fetching extra data"
                    $core_pretest_pr_nums = @{}
                    $subm_fetch_specs = @{}
                    foreach ($l in Get-GitRemoteBranches origin 'refs/heads/subm-pretest/*/*/*') {
                        $sha = $l.sha
                        $subm_owner = $l.matches[1]
                        $subm_name = $l.matches[2]
                        $pr_num = $l.matches[3]
                        $repo = "$subm_owner/$subm_name"
                        Write-Host "- Found core pretest branch for repo $repo PR $pr_num"
                        $core_pretest_pr_nums[$repo] += @{ "$pr_num" = $sha }
                        if ($pr_num -in $submodule_by_repo[$repo].pretest_prs) {
                            
                            $fork_pretest_branch = "pretest/$pr_num"
                            $subm_fetch_specs[$repo] += @("refs/heads/${fork_pretest_branch}:${fork_pretest_branch}")
                            
                        } else {
                            Write-Host "--- MISSING FORK PRETEST"
                        }
                    }
                    # To check which pretest branches are up to date and should not be tested
                    # - Only need commit messages, not files
                    foreach ($repo in $submodule_by_repo.Keys) {
                        if (-not $subm_fetch_specs[$repo]) { continue }
                        Write-Host "- Fetching branches for $repo"
                        # Write-Host git fetch --depth=1 --no-tags --filter=tree:0 $submodule_by_repo[$repo].fork_url $subm_fetch_specs[$repo]
                        git fetch --depth=1 --no-tags --filter=tree:0 $submodule_by_repo[$repo].fork_url $subm_fetch_specs[$repo]
                        if (-not $?) { throw "git fetch failed" }
                    }
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Building exec_list"
                    $exec_list = @()
                    foreach ($subm in $submodules) {
                        Write-Host "subm: $($subm.name)"
                        
                        foreach ($pr_num in $subm.head_sha_by_pr.Keys) {
                            Write-Host "- PR $pr_num"
                            
                            function Is-SkipNeeded() {
                                if ($pr_num -notin $subm.pretest_prs) { return $false }
                                if ($pr_num -notin $core_pretest_pr_nums[$subm.repo].Keys) { return $false }
                                $extra_debug = $true
                                
                                $pr_head_sha_found = $false
                                $core_pretest_sha_found = $false
                                foreach ($l in git log -1 --pretty=format:"%B" "pretest/$pr_num") {
                                    if ($extra_debug) {
                                        Write-Host "--- $l"
                                    }
                                    if ($l -notmatch '^meta\.(\w+)=([0-9a-f]{40})$') { continue }
                                    $key = $matches[1]
                                    $sha = $matches[2]
                                    if ($extra_debug) {
                                        Write-Host "----- FOUND [$key]: $sha"
                                    }
                                    
                                    if ($key -eq 'pr_head_sha') {
                                        
                                        if ($sha -eq $subm.head_sha_by_pr[$pr_num]) {
                                            $pr_head_sha_found = $true
                                        } elseif ($extra_debug) {
                                            Write-Host "----- SHA unexpected"
                                        }
                                        
                                    } elseif ($key -eq 'core_pretest_sha') {
                                        
                                        if ($sha -eq $core_pretest_pr_nums[$subm.repo][$pr_num]) {
                                            $core_pretest_sha_found = $true
                                        } elseif ($extra_debug) {
                                            Write-Host "----- SHA unexpected"
                                        }
                                        
                                    } elseif ($extra_debug) {
                                        Write-Host "----- Key unexpected"
                                    }
                                    
                                }
                                
                                return $pr_head_sha_found -and $core_pretest_sha_found
                            }
                            if (Is-SkipNeeded) { continue }
                            
                            Write-Host "--- TEST NOW"
                            $exec_list += [PSCustomObject]@{
                                '!id'       = $pr_num;
                                'subm_name' = $subm.name;
                                'org_repo'  = $subm.repo;
                                'fork_repo' = "SunSerega/$($subm.name)";
                                'pr_num'    = $pr_num;
                            }
                        }
                        
                    }
                    
                    $json = ConvertTo-Json -Compress $exec_list
                    Write-Host $json
                    echo "exec_list=$json" >> $env:GITHUB_OUTPUT
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Building remove_list"
                    $remove_list = @()
                    foreach ($subm in $submodules) {
                        Write-Host "subm: $($subm.name)"
                        
                        $subm_rem_pretest_pr_nums = @()
                        foreach ($pr_num in $subm.pretest_prs) {
                            Write-Host "- Fork pretest for PR $pr_num"
                            if ($pr_num -in $subm.head_sha_by_pr.Keys) { continue }
                            $subm_rem_pretest_pr_nums += $pr_num
                            Write-Host "--- REMOVE"
                        }
                        
                        $core_rem_pretest_pr_nums = @()
                        foreach ($pr_num in $core_pretest_pr_nums[$subm.repo].Keys) {
                            Write-Host "- Core pretest for PR $pr_num"
                            if ($pr_num -in $subm.head_sha_by_pr.Keys) { continue }
                            $core_rem_pretest_pr_nums += $pr_num
                        }
                        
                        if ($subm_rem_pretest_pr_nums -or $core_rem_pretest_pr_nums) {
                            $remove_list += [PSCustomObject]@{
                                '!id'                   = $subm.name;
                                'org_repo'              = $subm.repo;
                                'fork_repo'             = "SunSerega/$($subm.name)";
                                'subm_pretest_pr_nums'  = $subm_rem_pretest_pr_nums -join ',';
                                'core_pretest_pr_nums'  = $core_rem_pretest_pr_nums -join ',';
                            }
                        }
                        
                    }
                    
                    $json = ConvertTo-Json -Compress $remove_list
                    Write-Host $json
                    echo "remove_list=$json" >> $env:GITHUB_OUTPUT
            
    pretest-each:
        runs-on: windows-latest
        needs: enmr-PRs
        if: ${{ needs.enmr-PRs.outputs.exec_list != '[]' }}
        strategy:
            fail-fast: false
            matrix:
                exec-data: ${{ fromJson(needs.enmr-PRs.outputs.exec_list) }}
        
        steps:
            -   name: git config
                run: |
                
                    git config --global gc.auto 0
                    git config --global core.autocrlf false
                    git config --global advice.detachedHead false
                    git config --global user.name "sun pack bot"
                    git config --global user.email "sunserega2@gmail.com"
                    git config --global --add url.https://github.com/.insteadOf "git@github.com:"
            
            -   name: checkout main repo
                uses: actions/checkout@main
                with:
                    ref: ${{ github.event.repository.default_branch }}
                    path: './POCGL' # needs to have exactly this name for consistent traces in err tests
                    fetch-depth: 0
            
            -   name: checkout subm fork
                uses: actions/checkout@main
                with:
                    ref: 'custom'
                    path: './fork'
                    fetch-depth: 0
                    repository: ${{ matrix.exec-data.fork_repo }}
                    token: ${{ secrets.POCGL_pretest_upstream_PAT }}
            
            -   name: (!) pretest
                run: |
                
                    $subm_name = '${{ matrix.exec-data.subm_name }}'
                    $org_repo = '${{ matrix.exec-data.org_repo }}'
                    $fork_repo = '${{ matrix.exec-data.fork_repo }}'
                    $pr_num = "${{ matrix.exec-data.pr_num }}"
                    
                    
                    
                    $remote_official = '0_official'
                    
                    $b_fork_upstream_main = "$remote_official/main"
                    $b_fork_main = 'custom'
                    $b_fork_pretest = "pretest/$pr_num"
                    
                    $b_fork_merged_pr_head = "$pr_num/merged_pr_head" # $pr_num/head + main
                    $b_fork_virgin_merge_test = "$pr_num/virgin_merge_test" # $b_merged_pr_head + custom
                    
                    $b_core_main = '${{ github.event.repository.default_branch }}'
                    $b_core_subm_pretest = "subm-pretest/$org_repo/$pr_num"
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Updating subm fork..."
                    Push-Location './fork'
                    git remote add $remote_official "git@github.com:${org_repo}.git"
                    git config --add "remote.$remote_official.fetch" "+refs/pull/*:refs/remotes/$remote_official/pull/*"
                    git fetch --all 2>&1 | Out-Null
                    
                    Write-Host "- Creating $b_fork_merged_pr_head"
                    git checkout "remotes/$remote_official/pull/$pr_num/head"
                    if (-not $?) { throw "git checkout failed" }
                    git checkout -b $b_fork_merged_pr_head
                    if (-not $?) { throw "git checkout -b failed" }
                    
                    Write-Host "--- Merging with ${b_fork_upstream_main}:"
                    git merge $b_fork_upstream_main
                    if (-not $?) {
                        Write-Host "git merge failed"
                        Write-Host "This PR is outdated and cannot be properly tested"
                        # Write-Output "::notice::Cannot test PR $pr_num from $subm_name, because it has conflicts"
                        exit 0
                    }
                    
                    Write-Host "- Creating $b_fork_virgin_merge_test"
                    git checkout -b $b_fork_virgin_merge_test
                    if (-not $?) { throw "git checkout -b failed" }
                    $virgin_merge_test_exists = $true
                    
                    Write-Host "--- Merging with ${b_fork_main}:"
                    git merge $b_fork_main
                    if (-not $?) {
                        Write-Host "git merge $b_fork_main failed"
                        Write-Host "Cannot create $b_fork_virgin_merge_test"
                        $virgin_merge_test_exists = $false
                        git reset --hard
                        if (-not $?) { throw "git reset failed" }
                    }
                    
                    if (&{ git show-ref --verify -q "refs/remotes/origin/$b_fork_pretest"; $? }) {
                        Write-Host "- Branch for PR $pr_num exists"
                        git checkout $b_fork_pretest
                        
                        Write-Host "--- Merging with ${b_fork_main}:"
                        git merge $b_fork_main
                        if (-not $?) { throw "git merge failed" }
                        
                        Write-Host "--- Merging with PR head:"
                        git merge "remotes/$remote_official/pull/$pr_num/head"
                        if (-not $?) { throw "git merge failed" }
                        
                        Write-Host "--- Merging with ${b_fork_upstream_main}:"
                        git merge $b_fork_upstream_main
                        if (-not $?) { throw "git merge failed" }
                        
                        $need_push = $true
                        if ($virgin_merge_test_exists) {
                            Write-Host "--- Diff with ${b_fork_virgin_merge_test}:"
                            git diff HEAD $b_fork_virgin_merge_test --exit-code
                            if (-not $?) {
                                Write-Host "----- Diff found, building on top"
                            } else {
                                Write-Host "----- Diff not found, resetting branch"
                                git reset --hard $b_fork_virgin_merge_test
                                if (-not $?) { throw "git reset failed" }
                                git push -f
                                if (-not $?) { throw "git push -f failed" }
                                $need_push = $false
                            }
                        }
                        
                        if ($need_push) {
                            git push
                            if (-not $?) { throw "git push failed" }
                        }
                        
                    } else {
                        Write-Host "- Branch for PR $pr_num doesn't exist"
                        
                        if (-not $virgin_merge_test_exists) {
                            git checkout $b_fork_merged_pr_head
                            if (-not $?) { throw "git checkout failed" }
                            git checkout -b $b_fork_pretest
                            if (-not $?) { throw "git checkout -b failed" }
                            git push --set-upstream origin $b_fork_pretest
                            if (-not $?) { throw "git push failed" }
                            throw "Branch $b_fork_virgin_merge_test was not created, so merge with $b_fork_main is not possible"
                        }
                        
                        Write-Host "--- Creating:"
                        git checkout $b_fork_virgin_merge_test
                        if (-not $?) { throw "git checkout failed" }
                        git checkout -b $b_fork_pretest
                        if (-not $?) { throw "git checkout -b failed" }
                        
                        git push --set-upstream origin $b_fork_pretest
                        if (-not $?) { throw "git push failed" }
                        
                    }
                    
                    Pop-Location
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Updating main repo..."
                    Push-Location './POCGL'
                    git fetch --all 2>&1 | Out-Null
                    
                    if (&{ git show-ref --verify -q "refs/remotes/origin/$b_core_subm_pretest"; $? }) {
                        Write-Host "- Branch for PR $pr_num exists"
                        git checkout $b_core_subm_pretest
                        if (-not $?) { throw "git checkout failed" }
                        
                        $need_force_push = $false
                        Write-Host "--- Deleting the latest [trivial] commits:"
                        while ($true) {
                            ($commit_name = (git log -1 --pretty=format:"%B" HEAD) -join "`n")
                            if (-not $?) { throw "git log failed" }
                            if (-not $commit_name.Contains("[trivial]")) { break }
                            Write-Host "====="
                            git reset --hard HEAD~1
                            if (-not $?) { throw "git reset failed" }
                            $need_force_push = $true
                        }
                        if ($need_force_push) {
                            git push -f
                            if (-not $?) { throw "git push -f failed" }
                        }
                        
                        # expecting fast-forward when possible
                        Write-Host "--- Merging with ${b_core_main}:"
                        git merge $b_core_main -m "[trivial] Merge $b_core_main into subm-pretest/..."
                        if (-not $?) {
                            Write-Host "----- Simple merge failed, trying to fix submodule conflict"
                            
                            git reset "DataScraping/Reps/$subm_name"
                            if (-not $?) { throw "git reset failed" }
                            
                            git commit --no-edit
                            if (-not $?) { throw "git commit failed" }
                            
                        }
                        git push
                        if (-not $?) { throw "git push failed" }
                        
                    } else {
                        Write-Host "- Branch for PR $pr_num doesn't exist"
                        
                        Write-Host "--- Creating:"
                        git checkout -b $b_core_subm_pretest
                        if (-not $?) { throw "git checkout -b failed" }
                        git push --set-upstream origin $b_core_subm_pretest
                        if (-not $?) { throw "git push failed" }
                        
                    }
                    
                    if (-not $?) { throw "git rev-parse failed" }
                    Pop-Location
                    
                    
                    
                    & {
                        Write-Host "=============================="
                        Write-Host "Installing OpenCL driver for Intel CPU"
                        
                        # https://www.intel.com/content/www/us/en/developer/articles/technical/intel-cpu-runtime-for-opencl-applications-with-sycl-support.html
                        Invoke-WebRequest -Uri 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0e6849e6-2c56-480b-afcf-be8331d5c4f6-opencl/w_opencl_runtime_p_2024.1.0.968.exe' -OutFile 'D:\igfx.exe'
                        
                        7z x "D:\igfx.exe" -o"D:\igfx" -y
                        
                        D:\igfx\w_opencl_runtime_p_2024.1.0.968.msi /quiet
                    }
                    
                    & {
                        Write-Host "=============================="
                        Write-Host "Downloading and unpacking Pascal compiler"
                        
                        Invoke-WebRequest -Uri 'https://github.com/SunSerega/pascalabcnet/releases/download/custom-build-tag/PABCNETC.zip' -OutFile 'D:\PABCNETC.zip'
                        
                        Expand-Archive -Path 'D:\PABCNETC.zip' -DestinationPath 'D:\PABCNETC' -Force
                    }
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Testing main repo..."
                    Push-Location './POCGL'
                    
                    & .\DeleteAllTemp.bat NoPause | Out-Null
                    
                    Write-Host "- Compile: " -NoNewline
                    Start-Process -FilePath 'D:\PABCNETC\pabcnetcclear.exe' -ArgumentList '"PackAll.pas"' -Wait -NoNewWindow
                    
                    & .\PackAll.exe "Stages= PullUpstream + Reference + Dummy + OpenCL+OpenCLABC + OpenGL+OpenGLABC + Compile + Test + Release" "PullUpstreamBranch=${subm_name}:${b_fork_pretest}" "PasCompPath=D:\PABCNETC\pabcnetcclear.exe" SkipFinishedPause
                    if (-not $?) { throw "PackAll failed" }
                    
                    $need_commit = (git diff --name-only ":!LastPack.log" ":!Log/PullUpstream.log" ":!DataScraping/Reps") -or (git ls-files --others --exclude-standard)
                    if ($need_commit) {
                        Write-Host "- Found significant changes"
                        
                        git commit -a -m '[trivial] test changes'
                        if (-not $?) { throw "git commit failed" }
                        git push
                        if (-not $?) { throw "git push failed" }
                        
                    }
                    
                    $core_pretest_sha = git rev-parse HEAD
                    Pop-Location
                    
                    
                    
                    Write-Host "=============================="
                    Write-Host "Adding meta to fork repo..."
                    Push-Location './fork'
                    
                    $pr_head_sha = git rev-parse "remotes/0_official/pull/$pr_num/head"
                    if (-not $?) { throw "git rev-parse failed" }
                    
                    git commit --allow-empty -m @"
                    Add pretest meta
                    meta.pr_head_sha=$pr_head_sha
                    meta.core_pretest_sha=$core_pretest_sha
                    "@
                    if (-not $?) { throw "git commit failed" }
                    git push
                    if (-not $?) { throw "git push failed" }
                    
                    Pop-Location
                    
                    
                    
                    Write-Output "::notice::Newly tested PR $pr_num from $subm_name"
            
    remove-old-pretest:
        runs-on: windows-latest
        needs: enmr-PRs
        if: ${{ needs.enmr-PRs.outputs.remove_list != '[]' }}
        strategy:
            fail-fast: false
            matrix:
                remove-data: ${{ fromJson(needs.enmr-PRs.outputs.remove_list) }}
        
        steps:
            -   name: git config
                run: |
                
                    git config --global gc.auto 0
                    git config --global core.autocrlf false
                    git config --global user.name "sun pack bot"
                    git config --global user.email "sunserega2@gmail.com"
                    git config --global --add url.https://github.com/.insteadOf "git@github.com:"
            
            -   name: checkout main repo
                uses: actions/checkout@main
                with:
                    path: './POCGL'
                    fetch-depth: 0
                    sparse-checkout-cone-mode: false
                    sparse-checkout: '.gitmodules'
            
            -   name: checkout subm fork
                uses: actions/checkout@main
                with:
                    path: './fork'
                    sparse-checkout-cone-mode: false
                    sparse-checkout: '<NOTHING>'
                    repository: ${{ matrix.remove-data.fork_repo }}
                    token: ${{ secrets.POCGL_pretest_upstream_PAT }}
            
            -   name: (!) remove old pretest branches
                run: |
                
                    function Split-OrEmpty {
                        param ( $s )
                        if (-not $s) { return @() }
                        return $s -split ','
                    }
                    
                    $org_repo = '${{ matrix.remove-data.org_repo }}'
                    $fork_repo = '${{ matrix.remove-data.fork_repo }}'
                    $subm_pretest_pr_nums = Split-OrEmpty '${{ matrix.remove-data.subm_pretest_pr_nums }}'
                    $core_pretest_pr_nums = Split-OrEmpty '${{ matrix.remove-data.core_pretest_pr_nums }}'
                    
                    
                    
                    Push-Location './POCGL'
                    
                    $core_removed_pr_nums = @()
                    foreach ($pr_num in $core_pretest_pr_nums) {
                        Write-Host "Checking if subm-pretest branch for PR $pr_num is trivial"
                        $branch_name = "subm-pretest/$org_repo/$pr_num"
                        Write-Host $branch_name
                        $is_trivial = $true
                        foreach ($commit_name in git show --pretty=format:"%s" -s "HEAD..origin/$branch_name") {
                            Write-Host "- $commit_name"
                            if ($commit_name.StartsWith('[trivial]')) { continue }
                            Write-Host "--- NOT TRIVIAL"
                            $is_trivial = $false
                            # break
                        }
                        if (-not $?) { throw "git show failed" }
                        if ($is_trivial) {
                            Write-Host "Is trivial, removing..."
                            git push origin --delete $branch_name                                               # BRANCH DELETE
                            if (-not $?) { throw "git push --delete failed" }
                            $core_removed_pr_nums += $pr_num
                        }
                        Write-Host '~~~~~'
                    }
                    
                    Pop-Location
                    
                    
                    
                    $core_pretest_pr_nums = $core_pretest_pr_nums |
                        Where-Object { $_ -notin $core_removed_pr_nums }
                    
                    
                    
                    Push-Location './fork'
                    
                    foreach ($pr_num in $subm_pretest_pr_nums) {
                        Write-Host "PR $pr_num was closed"
                        if ($core_pretest_pr_nums -contains $pr_num) {
                            Write-Host "But main repo is still referencing it"
                        } else {
                            Write-Host "Removing..."
                            git push origin --delete "pretest/$pr_num"                                          # BRANCH DELETE
                            if (-not $?) { throw "git push --delete failed" }
                        }
                        Write-Host '~~~~~'
                    }
                    
                    Pop-Location
                    
                    
                    
                    if ($core_pretest_pr_nums) {
                        Write-Host "pretest branches for these PRs are dangling:"
                        foreach ($pr_num in $core_pretest_pr_nums) {
                            Write-Host "- $pr_num"
                        }
                        throw "Some pretest branches in main repo are dangling"
                    }