Skip to content

Upstream pretest

Upstream pretest #237

name: Upstream pretest
on:
push:
branches-ignore:
- 'subm-pretest/**'
schedule:
- cron: '0 0 * * *'
workflow_dispatch:
permissions:
contents: write
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false
# There are 2 steps of adding new upstream commits:
# 1. Add new commits from the PR head ref (and, if needed, main/custom branches): to "pretest/$pr_num" in fork
# 2. Add commit with test changes: to "subm-pretest/$org_repo/$pr_num" in POCGL repo
# To only update when source has been updated:
# - "pretest" references sha of latest PR commit as: meta.pr_head_sha
# - "pretest" references sha of latest "subm-pretest" commit as: meta.core_pretest_sha
# "pretest" is not updated when only main or custom branch of subm updates
# - Otherwise all pretest branches would update at the same time, even on unrelated changes
# "pretest" needs to be first checked by a separate "git ls-remote" before "git fetch"
# - Otherwise git fetch would fail fetching the missing branches
# "pretest" and "subm-pretest" can be update manually by push, in case anything new needs to be implemented for the new XML spec
# - In that case, there would be no commit sha reference, so it will be re-tested on the next run of this workflow
jobs:
enmr-PRs:
runs-on: windows-latest
outputs:
exec_list: ${{ steps.make_exec_list.outputs.exec_list }}
remove_list: ${{ steps.make_exec_list.outputs.remove_list }}
steps:
- name: git config
run: |
git config --global gc.auto 0
git config --global core.autocrlf false
git config --global user.name "sun pack bot"
git config --global user.email "sunserega2@gmail.com"
git config --global --add url.https://github.com/.insteadOf "git@github.com:"
- name: checkout
uses: actions/checkout@main
with:
sparse-checkout-cone-mode: false
sparse-checkout: |
.gitmodules
- name: (!) make exec list
id: make_exec_list
run: |
function Get-GitRemoteBranches() {
param ($url, $pattern)
$reg_pattern = $pattern -replace '\*','(.*)'
$res = @()
foreach ($l in git ls-remote $url $pattern) {
if ($l -notmatch "(^[0-9a-f]{40})\s+($reg_pattern)$") {
throw "Unexpected git ls-remote output: $l"
}
$sha = $matches[1]
$branch = $matches[2]
if ($branch -notmatch "^$reg_pattern$") {
throw "What?"
}
$res += [PSCustomObject]@{
'sha' = $sha;
'branch' = $branch;
'matches' = $matches;
}
}
if (-not $?) { throw "git ls-remote failed" }
return $res
}
Write-Host "=============================="
Write-Host "Bulding open PR list"
$submodules = @()
$submodule_by_repo = @{}
foreach ($config_key in git config --file .gitmodules --name-only --get-regexp '^submodule\..+\.url$') {
$url = git config --file .gitmodules --get $config_key
Write-Host "URL: $url"
if ($url -notmatch '^git@github\.com:([\w\-]+)/([\w\-]+)\.git$') {
throw "Unexpected url format: $url"
}
$owner = $matches[1]
$name = $matches[2]
$fork_url = "git@github.com:SunSerega/${name}.git"
$mergeable_prs = @()
foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/merge') {
$pr_num = $l.matches[1]
$mergeable_prs += $pr_num
}
$head_sha_by_pr = @{}
foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/head') {
$sha = $l.sha
$pr_num = $l.matches[1]
if ($pr_num -notin $mergeable_prs) { continue }
Write-Host "- Found open PR $pr_num with latest sha $sha"
$head_sha_by_pr.Add($pr_num, $sha)
}
$pretest_prs = @()
foreach ($l in Get-GitRemoteBranches $fork_url 'refs/heads/pretest/*') {
$sha = $l.sha
$pr_num = $l.matches[1]
Write-Host "- Found pretest branch for PR $pr_num with latest sha $sha"
$pretest_prs += $pr_num
}
$subm = [PSCustomObject]@{
'url' = $url;
'fork_url' = $fork_url;
'owner' = $owner;
'name' = $name;
'repo' = "$owner/$name"
'head_sha_by_pr' = $head_sha_by_pr;
'pretest_prs' = $pretest_prs;
}
$submodules += $subm
$submodule_by_repo.Add($subm.repo, $subm)
}
Write-Host "=============================="
Write-Host "Fetching extra data"
$core_pretest_pr_nums = @{}
$subm_fetch_specs = @{}
foreach ($l in Get-GitRemoteBranches origin 'refs/heads/subm-pretest/*/*/*') {
$sha = $l.sha
$subm_owner = $l.matches[1]
$subm_name = $l.matches[2]
$pr_num = $l.matches[3]
$repo = "$subm_owner/$subm_name"
Write-Host "- Found core pretest branch for repo $repo PR $pr_num"
$core_pretest_pr_nums[$repo] += @{ "$pr_num" = $sha }
if ($pr_num -in $submodule_by_repo[$repo].pretest_prs) {
$fork_pretest_branch = "pretest/$pr_num"
$subm_fetch_specs[$repo] += @("refs/heads/${fork_pretest_branch}:${fork_pretest_branch}")
} else {
Write-Host "--- MISSING FORK PRETEST"
}
}
# To check which pretest branches are up to date and should not be tested
# - Only need commit messages, not files
foreach ($repo in $submodule_by_repo.Keys) {
if (-not $subm_fetch_specs[$repo]) { continue }
Write-Host "- Fetching branches for $repo"
# Write-Host git fetch --depth=1 --no-tags --filter=tree:0 $submodule_by_repo[$repo].fork_url $subm_fetch_specs[$repo]
git fetch --depth=1 --no-tags --filter=tree:0 $submodule_by_repo[$repo].fork_url $subm_fetch_specs[$repo]
if (-not $?) { throw "git fetch failed" }
}
Write-Host "=============================="
Write-Host "Building exec_list"
$exec_list = @()
foreach ($subm in $submodules) {
Write-Host "subm: $($subm.name)"
foreach ($pr_num in $subm.head_sha_by_pr.Keys) {
Write-Host "- PR $pr_num"
function Is-SkipNeeded() {
if ($pr_num -notin $subm.pretest_prs) { return $false }
if ($pr_num -notin $core_pretest_pr_nums[$subm.repo].Keys) { return $false }
$extra_debug = $true
$pr_head_sha_found = $false
$core_pretest_sha_found = $false
foreach ($l in git log -1 --pretty=format:"%B" "pretest/$pr_num") {
if ($extra_debug) {
Write-Host "--- $l"
}
if ($l -notmatch '^meta\.(\w+)=([0-9a-f]{40})$') { continue }
$key = $matches[1]
$sha = $matches[2]
if ($extra_debug) {
Write-Host "----- FOUND [$key]: $sha"
}
if ($key -eq 'pr_head_sha') {
if ($sha -eq $subm.head_sha_by_pr[$pr_num]) {
$pr_head_sha_found = $true
} elseif ($extra_debug) {
Write-Host "----- SHA unexpected"
}
} elseif ($key -eq 'core_pretest_sha') {
if ($sha -eq $core_pretest_pr_nums[$subm.repo][$pr_num]) {
$core_pretest_sha_found = $true
} elseif ($extra_debug) {
Write-Host "----- SHA unexpected"
}
} elseif ($extra_debug) {
Write-Host "----- Key unexpected"
}
}
return $pr_head_sha_found -and $core_pretest_sha_found
}
if (Is-SkipNeeded) { continue }
Write-Host "--- TEST NOW"
$exec_list += [PSCustomObject]@{
'!id' = $pr_num;
'subm_name' = $subm.name;
'org_repo' = $subm.repo;
'fork_repo' = "SunSerega/$($subm.name)";
'pr_num' = $pr_num;
}
}
}
$json = ConvertTo-Json -Compress $exec_list
Write-Host $json
echo "exec_list=$json" >> $env:GITHUB_OUTPUT
Write-Host "=============================="
Write-Host "Building remove_list"
$remove_list = @()
foreach ($subm in $submodules) {
Write-Host "subm: $($subm.name)"
$subm_rem_pretest_pr_nums = @()
foreach ($pr_num in $subm.pretest_prs) {
Write-Host "- Fork pretest for PR $pr_num"
if ($pr_num -in $subm.head_sha_by_pr.Keys) { continue }
$subm_rem_pretest_pr_nums += $pr_num
Write-Host "--- REMOVE"
}
$core_rem_pretest_pr_nums = @()
foreach ($pr_num in $core_pretest_pr_nums[$subm.repo].Keys) {
Write-Host "- Core pretest for PR $pr_num"
if ($pr_num -in $subm.head_sha_by_pr.Keys) { continue }
$core_rem_pretest_pr_nums += $pr_num
}
if ($subm_rem_pretest_pr_nums -or $core_rem_pretest_pr_nums) {
$remove_list += [PSCustomObject]@{
'!id' = $subm.name;
'org_repo' = $subm.repo;
'fork_repo' = "SunSerega/$($subm.name)";
'subm_pretest_pr_nums' = $subm_rem_pretest_pr_nums -join ',';
'core_pretest_pr_nums' = $core_rem_pretest_pr_nums -join ',';
}
}
}
$json = ConvertTo-Json -Compress $remove_list
Write-Host $json
echo "remove_list=$json" >> $env:GITHUB_OUTPUT
pretest-each:
runs-on: windows-latest
needs: enmr-PRs
if: ${{ needs.enmr-PRs.outputs.exec_list != '[]' }}
strategy:
fail-fast: false
matrix:
exec-data: ${{ fromJson(needs.enmr-PRs.outputs.exec_list) }}
steps:
- name: git config
run: |
git config --global gc.auto 0
git config --global core.autocrlf false
git config --global advice.detachedHead false
git config --global user.name "sun pack bot"
git config --global user.email "sunserega2@gmail.com"
git config --global --add url.https://github.com/.insteadOf "git@github.com:"
- name: checkout main repo
uses: actions/checkout@main
with:
ref: ${{ github.event.repository.default_branch }}
path: './POCGL' # needs to have exactly this name for consistent traces in err tests
fetch-depth: 0
- name: checkout subm fork
uses: actions/checkout@main
with:
ref: 'custom'
path: './fork'
fetch-depth: 0
repository: ${{ matrix.exec-data.fork_repo }}
token: ${{ secrets.POCGL_pretest_upstream_PAT }}
- name: (!) pretest
run: |
$subm_name = '${{ matrix.exec-data.subm_name }}'
$org_repo = '${{ matrix.exec-data.org_repo }}'
$fork_repo = '${{ matrix.exec-data.fork_repo }}'
$pr_num = "${{ matrix.exec-data.pr_num }}"
$remote_official = '0_official'
$b_fork_upstream_main = "$remote_official/main"
$b_fork_main = 'custom'
$b_fork_pretest = "pretest/$pr_num"
$b_fork_merged_pr_head = "$pr_num/merged_pr_head" # $pr_num/head + main
$b_fork_virgin_merge_test = "$pr_num/virgin_merge_test" # $b_merged_pr_head + custom
$b_core_main = '${{ github.event.repository.default_branch }}'
$b_core_subm_pretest = "subm-pretest/$org_repo/$pr_num"
Write-Host "=============================="
Write-Host "Updating subm fork..."
Push-Location './fork'
git remote add $remote_official "git@github.com:${org_repo}.git"
git config --add "remote.$remote_official.fetch" "+refs/pull/*:refs/remotes/$remote_official/pull/*"
git fetch --all 2>&1 | Out-Null
Write-Host "- Creating $b_fork_merged_pr_head"
git checkout "remotes/$remote_official/pull/$pr_num/head"
if (-not $?) { throw "git checkout failed" }
git checkout -b $b_fork_merged_pr_head
if (-not $?) { throw "git checkout -b failed" }
Write-Host "--- Merging with ${b_fork_upstream_main}:"
git merge $b_fork_upstream_main
if (-not $?) {
Write-Host "git merge failed"
Write-Host "This PR is outdated and cannot be properly tested"
# Write-Output "::notice::Cannot test PR $pr_num from $subm_name, because it has conflicts"
exit 0
}
Write-Host "- Creating $b_fork_virgin_merge_test"
git checkout -b $b_fork_virgin_merge_test
if (-not $?) { throw "git checkout -b failed" }
$virgin_merge_test_exists = $true
Write-Host "--- Merging with ${b_fork_main}:"
git merge $b_fork_main
if (-not $?) {
Write-Host "git merge $b_fork_main failed"
Write-Host "Cannot create $b_fork_virgin_merge_test"
$virgin_merge_test_exists = $false
git reset --hard
if (-not $?) { throw "git reset failed" }
}
if (&{ git show-ref --verify -q "refs/remotes/origin/$b_fork_pretest"; $? }) {
Write-Host "- Branch for PR $pr_num exists"
git checkout $b_fork_pretest
Write-Host "--- Merging with ${b_fork_main}:"
git merge $b_fork_main
if (-not $?) { throw "git merge failed" }
Write-Host "--- Merging with PR head:"
git merge "remotes/$remote_official/pull/$pr_num/head"
if (-not $?) { throw "git merge failed" }
Write-Host "--- Merging with ${b_fork_upstream_main}:"
git merge $b_fork_upstream_main
if (-not $?) { throw "git merge failed" }
$need_push = $true
if ($virgin_merge_test_exists) {
Write-Host "--- Diff with ${b_fork_virgin_merge_test}:"
git diff HEAD $b_fork_virgin_merge_test --exit-code
if (-not $?) {
Write-Host "----- Diff found, building on top"
} else {
Write-Host "----- Diff not found, resetting branch"
git reset --hard $b_fork_virgin_merge_test
if (-not $?) { throw "git reset failed" }
git push -f
if (-not $?) { throw "git push -f failed" }
$need_push = $false
}
}
if ($need_push) {
git push
if (-not $?) { throw "git push failed" }
}
} else {
Write-Host "- Branch for PR $pr_num doesn't exist"
if (-not $virgin_merge_test_exists) {
git checkout $b_fork_merged_pr_head
if (-not $?) { throw "git checkout failed" }
git checkout -b $b_fork_pretest
if (-not $?) { throw "git checkout -b failed" }
git push --set-upstream origin $b_fork_pretest
if (-not $?) { throw "git push failed" }
throw "Branch $b_fork_virgin_merge_test was not created, so merge with $b_fork_main is not possible"
}
Write-Host "--- Creating:"
git checkout $b_fork_virgin_merge_test
if (-not $?) { throw "git checkout failed" }
git checkout -b $b_fork_pretest
if (-not $?) { throw "git checkout -b failed" }
git push --set-upstream origin $b_fork_pretest
if (-not $?) { throw "git push failed" }
}
Pop-Location
Write-Host "=============================="
Write-Host "Updating main repo..."
Push-Location './POCGL'
git fetch --all 2>&1 | Out-Null
if (&{ git show-ref --verify -q "refs/remotes/origin/$b_core_subm_pretest"; $? }) {
Write-Host "- Branch for PR $pr_num exists"
git checkout $b_core_subm_pretest
if (-not $?) { throw "git checkout failed" }
$need_force_push = $false
Write-Host "--- Deleting the latest [trivial] commits:"
while ($true) {
($commit_name = (git log -1 --pretty=format:"%B" HEAD) -join "`n")
if (-not $?) { throw "git log failed" }
if (-not $commit_name.Contains("[trivial]")) { break }
Write-Host "====="
git reset --hard HEAD~1
if (-not $?) { throw "git reset failed" }
$need_force_push = $true
}
if ($need_force_push) {
git push -f
if (-not $?) { throw "git push -f failed" }
}
# expecting fast-forward when possible
Write-Host "--- Merging with ${b_core_main}:"
$need_commit_even_minimal = $false
git merge $b_core_main -m "[trivial] Merge $b_core_main into subm-pretest/..."
if (-not $?) {
Write-Host "----- Simple merge failed, trying to fix submodule conflict"
git reset "DataScraping/Reps/$subm_name"
if (-not $?) { throw "git reset failed" }
git add *.log
if (-not $?) { throw "git add (unmerged *.log files) failed" }
git commit --no-edit
if (-not $?) { throw "git commit (after merge) failed" }
$need_commit_even_minimal = $true
}
git push
if (-not $?) { throw "git push failed" }
} else {
Write-Host "- Branch for PR $pr_num doesn't exist"
Write-Host "--- Creating:"
git checkout -b $b_core_subm_pretest
if (-not $?) { throw "git checkout -b failed" }
git push --set-upstream origin $b_core_subm_pretest
if (-not $?) { throw "git push failed" }
}
if (-not $?) { throw "git rev-parse failed" }
Pop-Location
& {
Write-Host "=============================="
Write-Host "Installing OpenCL driver for Intel CPU"
# https://www.intel.com/content/www/us/en/developer/articles/technical/intel-cpu-runtime-for-opencl-applications-with-sycl-support.html
Invoke-WebRequest -Uri 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0e6849e6-2c56-480b-afcf-be8331d5c4f6-opencl/w_opencl_runtime_p_2024.1.0.968.exe' -OutFile 'D:\igfx.exe'
7z x "D:\igfx.exe" -o"D:\igfx" -y
D:\igfx\w_opencl_runtime_p_2024.1.0.968.msi /quiet
}
& {
Write-Host "=============================="
Write-Host "Downloading and unpacking Pascal compiler"
Invoke-WebRequest -Uri 'https://github.com/SunSerega/pascalabcnet/releases/download/custom-build-tag/PABCNETC.zip' -OutFile 'D:\PABCNETC.zip'
Expand-Archive -Path 'D:\PABCNETC.zip' -DestinationPath 'D:\PABCNETC' -Force
}
Write-Host "=============================="
Write-Host "Testing main repo..."
Push-Location './POCGL'
& .\DeleteAllTemp.bat NoPause | Out-Null
Write-Host "- Compile: " -NoNewline
Start-Process -FilePath 'D:\PABCNETC\pabcnetcclear.exe' -ArgumentList '"PackAll.pas"' -Wait -NoNewWindow
& .\PackAll.exe "Stages= PullUpstream + Reference + Dummy + OpenCL+OpenCLABC + OpenGL+OpenGLABC + Compile + Test + Release" "PullUpstreamBranch=${subm_name}:${b_fork_pretest}" "PasCompPath=D:\PABCNETC\pabcnetcclear.exe" SkipFinishedPause
if (-not $?) { throw "PackAll failed" }
$need_commit = (git diff --name-only ":!LastPack.log" ":!Log/PullUpstream.log" ":!DataScraping/Reps") -or (git ls-files --others --exclude-standard)
if ($need_commit) {
Write-Host "- Found significant changes"
git commit -a -m '[trivial] test changes'
if (-not $?) { throw "git commit failed" }
git push
if (-not $?) { throw "git push failed" }
} elseif ($need_commit_even_minimal) {
Write-Host "- Found insignificant changes after merge conflict"
git diff
git commit -a --amend --no-edit
if (-not $?) { throw "git commit failed" }
git push --force-with-lease
if (-not $?) { throw "git push failed" }
} else {
Write-Host "- Didn't find any significant changes"
}
$core_pretest_sha = git rev-parse HEAD
Pop-Location
Write-Host "=============================="
Write-Host "Adding meta to fork repo..."
Push-Location './fork'
$pr_head_sha = git rev-parse "remotes/0_official/pull/$pr_num/head"
if (-not $?) { throw "git rev-parse failed" }
git commit --allow-empty -m @"
Add pretest meta
meta.pr_head_sha=$pr_head_sha
meta.core_pretest_sha=$core_pretest_sha
"@
if (-not $?) { throw "git commit failed" }
git push
if (-not $?) { throw "git push failed" }
Pop-Location
Write-Output "::notice::Newly tested PR $pr_num from $subm_name"
remove-old-pretest:
runs-on: windows-latest
needs: enmr-PRs
if: ${{ needs.enmr-PRs.outputs.remove_list != '[]' }}
strategy:
fail-fast: false
matrix:
remove-data: ${{ fromJson(needs.enmr-PRs.outputs.remove_list) }}
steps:
- name: git config
run: |
git config --global gc.auto 0
git config --global core.autocrlf false
git config --global user.name "sun pack bot"
git config --global user.email "sunserega2@gmail.com"
git config --global --add url.https://github.com/.insteadOf "git@github.com:"
- name: checkout main repo
uses: actions/checkout@main
with:
path: './POCGL'
fetch-depth: 0
sparse-checkout-cone-mode: false
sparse-checkout: '.gitmodules'
- name: checkout subm fork
uses: actions/checkout@main
with:
path: './fork'
sparse-checkout-cone-mode: false
sparse-checkout: '<NOTHING>'
repository: ${{ matrix.remove-data.fork_repo }}
token: ${{ secrets.POCGL_pretest_upstream_PAT }}
- name: (!) remove old pretest branches
run: |
function Split-OrEmpty {
param ( $s )
if (-not $s) { return @() }
return $s -split ','
}
$org_repo = '${{ matrix.remove-data.org_repo }}'
$fork_repo = '${{ matrix.remove-data.fork_repo }}'
$subm_pretest_pr_nums = Split-OrEmpty '${{ matrix.remove-data.subm_pretest_pr_nums }}'
$core_pretest_pr_nums = Split-OrEmpty '${{ matrix.remove-data.core_pretest_pr_nums }}'
Push-Location './POCGL'
$core_removed_pr_nums = @()
foreach ($pr_num in $core_pretest_pr_nums) {
Write-Host "Checking if subm-pretest branch for PR $pr_num is trivial"
$branch_name = "subm-pretest/$org_repo/$pr_num"
Write-Host $branch_name
$is_trivial = $true
foreach ($commit_name in git show --pretty=format:"%s" -s "HEAD..origin/$branch_name") {
Write-Host "- $commit_name"
if ($commit_name.StartsWith('[trivial]')) { continue }
Write-Host "--- NOT TRIVIAL"
$is_trivial = $false
# break
}
if (-not $?) { throw "git show failed" }
if ($is_trivial) {
Write-Host "Is trivial, removing..."
git push origin --delete $branch_name # BRANCH DELETE
if (-not $?) { throw "git push --delete failed" }
$core_removed_pr_nums += $pr_num
}
Write-Host '~~~~~'
}
Pop-Location
$core_pretest_pr_nums = $core_pretest_pr_nums |
Where-Object { $_ -notin $core_removed_pr_nums }
Push-Location './fork'
foreach ($pr_num in $subm_pretest_pr_nums) {
Write-Host "PR $pr_num was closed"
if ($core_pretest_pr_nums -contains $pr_num) {
Write-Host "But main repo is still referencing it"
} else {
Write-Host "Removing..."
git push origin --delete "pretest/$pr_num" # BRANCH DELETE
if (-not $?) { throw "git push --delete failed" }
}
Write-Host '~~~~~'
}
Pop-Location
if ($core_pretest_pr_nums) {
Write-Host "pretest branches for these PRs are dangling:"
foreach ($pr_num in $core_pretest_pr_nums) {
Write-Host "- $pr_num"
}
throw "Some pretest branches in main repo are dangling"
}