diff --git a/.github/workflows/upstream pretest.yaml b/.github/workflows/upstream pretest.yaml index 2b7b10b3..1a1e09fb 100644 --- a/.github/workflows/upstream pretest.yaml +++ b/.github/workflows/upstream pretest.yaml @@ -9,6 +9,23 @@ on: permissions: contents: write +# There are 2 steps of adding new upstream commits: +# 1. Add new commits from the PR merge ref (and possibly custom branch): to "pretest/$pr_num" in fork +# 2. Add commit with submodule ref move + test changes, with this ref move: to "subm-pretest/$org_repo/$pr_num" in POCGL repo + +# To only update when source has been updated: +# - "pretest" references sha of latest merge commit for PR as: meta.pr_merge_sha +# - "subm-pretest" references sha of latest pretest commit in fork as: meta.fork_pretest_sha + +# "pretest" is not updated when only main or custom branch of subm updates +# - Otherwise all pretest branches would update at the same time, even on unrelated changes + +# "pretest" and "subm-pretest" need to be first checked by a separate "git ls-remote" before "git fetch" +# - Otherwise git fetch would fail fetching them + +# "pretest" and "subm-pretest" can be update manually by push, in case anything new needs to be implemented for the new XML spec +# - In that case, there would be no commit sha reference, so it will be re-tested on the next run of this workflow + jobs: enmr-PRs: runs-on: windows-latest @@ -30,81 +47,158 @@ jobs: with: sparse-checkout-cone-mode: false sparse-checkout: | - .gitmodules + .gitmodules - name: make exec list id: make_exec_list run: | - $exec_list = @() - $remove_list = @() - - foreach ($subm_key in git config --file .gitmodules --name-only --get-regexp '^submodule\..+\.url$') { - $subm_url = git config --file .gitmodules --get $subm_key - Write-Host "URL: $subm_url" - - if ($subm_url -notmatch '^git@github\.com:([\w\-]+)/([\w\-]+)\.git$') { - throw "Unexpected subm_url format: $subm_url" - } - $subm_owner = $Matches[1] - $subm_name = $Matches[2] - - $subm_merge_pr_nums = @() - foreach ($ref_l in git ls-remote $subm_url 'refs/pull/*/merge') { - if ($ref_l -notmatch '^[0-9a-f]+\s+refs/pull/(\d+)/merge$') { - throw "Unexpected merge ref format: $ref_l" + function Get-GitRemoteBranches() { + param ($url, $pattern) + $reg_pattern = $pattern -replace '\*','(.*)' + $res = @() + foreach ($l in git ls-remote $url $pattern) { + if ($l -notmatch "(^[0-9a-f]{40})\s+($reg_pattern)$") { + throw "Unexpected git ls-remote output: $l" } - $pr_num = $Matches[1] - - Write-Host "- Found PR $pr_num" - - $subm_merge_pr_nums += $pr_num - $exec_list += @{ - 'subm_name' = $subm_name; - 'org_repo' = "$subm_owner/$subm_name"; - 'fork_repo' = "SunSerega/$subm_name"; - 'pr_num' = $pr_num; + $sha = $matches[1] + $branch = $matches[2] + if ($branch -notmatch "^$reg_pattern$") { + throw "What?" + } + $res += [PSCustomObject]@{ + 'sha' = $sha; + 'branch' = $branch; + 'matches' = $matches; } } + if (-not $?) { throw "git ls-remote failed" } + return $res + } + + Write-Host "Bulding open PR list" + $submodules = @() + $submodule_by_repo = @{} + foreach ($config_key in git config --file .gitmodules --name-only --get-regexp '^submodule\..+\.url$') { + $url = git config --file .gitmodules --get $config_key + Write-Host "URL: $url" - - - if ($subm_owner -ne 'KhronosGroup') { - throw 'Unexpected original owner of submodule: $subm_owner' + if ($url -notmatch '^git@github\.com:([\w\-]+)/([\w\-]+)\.git$') { + throw "Unexpected url format: $url" } - $subm_fork_url = "git@github.com:SunSerega/${subm_name}.git" - if ($subm_fork_url -eq $subm_url) { - throw "Fork url [$subm_fork_url] = subm_url [$subm_url]" + $owner = $matches[1] + $name = $matches[2] + $fork_url = "git@github.com:SunSerega/${name}.git" + + $merge_sha_by_pr = @{} + foreach ($l in Get-GitRemoteBranches $url 'refs/pull/*/merge') { + $sha = $l.sha + $pr_num = $l.matches[1] + Write-Host "- Found open PR $pr_num with latest sha $sha" + $merge_sha_by_pr.Add($pr_num, $sha) } - $subm_pretest_pr_nums = @() - foreach ($ref_l in git ls-remote $subm_fork_url 'refs/heads/pretest/*') { - if ($ref_l -notmatch '^[0-9a-f]+\s+refs/heads/pretest/(.+)$') { - throw "Unexpected subm pretest ref format: $ref_l" - } - $pr_num = $Matches[1] - Write-Host "- Found subm pretest branch for PR $pr_num" - if ($subm_merge_pr_nums -contains $pr_num) { continue } - $subm_pretest_pr_nums += $pr_num + $pretest_sha_by_pr = @{} + foreach ($l in Get-GitRemoteBranches $fork_url 'refs/heads/pretest/*') { + $sha = $l.sha + $pr_num = $l.matches[1] + Write-Host "- Found pretest branch for PR $pr_num with latest sha $sha" + $pretest_sha_by_pr.Add($pr_num, $sha) } - $core_pretest_pr_nums = @() - foreach ($ref_l in git ls-remote origin "refs/heads/subm-pretest/$subm_owner/$subm_name/*") { - if ($ref_l -notmatch "^[0-9a-f]+\s+refs/heads/subm-pretest/$subm_owner/$subm_name/(.+)$") { - throw "Unexpected core pretest ref format: $ref_l" - } - $pr_num = $Matches[1] - Write-Host "- Found core pretest branch for PR $pr_num" - if ($subm_merge_pr_nums -contains $pr_num) { continue } - $core_pretest_pr_nums += $pr_num + $subm = [PSCustomObject]@{ + 'url' = $url; + 'fork_url' = $fork_url; + 'owner' = $owner; + 'name' = $name; + 'repo' = "$owner/$name" + 'merge_sha_by_pr' = $merge_sha_by_pr; + 'pretest_sha_by_pr' = $pretest_sha_by_pr; + } + $submodules += $subm + $submodule_by_repo.Add($subm.repo, $subm) + } + + + + Write-Host "Fetching extra data" + $core_pretest_pr_nums = @{} + $core_fetch_specs = @() + $subm_fetch_specs = @{} + foreach ($l in Get-GitRemoteBranches origin 'refs/heads/subm-pretest/*/*/*') { + $subm_owner = $l.matches[1] + $subm_name = $l.matches[2] + $pr_num = $l.matches[3] + $repo = "$subm_owner/$subm_name" + Write-Host "- Found core pretest branch for repo $repo PR $pr_num" + $core_pretest_pr_nums[$repo] += @($pr_num) + if ($pr_num -in $submodule_by_repo[$repo].pretest_sha_by_pr.Keys) { + + $core_pretest_branch = "subm-pretest/$repo/$pr_num" + $core_fetch_specs += "refs/heads/$core_pretest_branch:$core_pretest_branch" + + $fork_pretest_branch = "pretest/$pr_num" + $subm_fetch_specs[$repo] += @("refs/heads/$fork_pretest_branch:$fork_pretest_branch") + + } else { + Write-Host "--- MISSING FORK PRETEST" } + } + # To check which pretest branches are up to date and should not be tested + # - Only need commit messages, not files + git fetch --depth=1 --no-tags --filter=tree:0 origin $core_fetch_specs + if (-not $?) { throw "git fetch failed" } + foreach ($repo in $submodule_by_repo.Keys) { + git fetch --depth=1 --no-tags --filter=tree:0 $subm_fetch_specs[$repo].fork_url $subm_fetch_specs[$repo] + if (-not $?) { throw "git fetch failed" } + } + + + + Write-Host "==============================" + Write-Host "Building exec_list" + $exec_list = @() + foreach ($subm in $submodules) { + Write-Host "subm: $($subm.name)" - if ($subm_pretest_pr_nums -or $core_pretest_pr_nums) { - $remove_list += @{ - 'org_repo' = "$subm_owner/$subm_name"; - 'fork_repo' = "SunSerega/$subm_name"; - 'subm_pretest_pr_nums' = $subm_pretest_pr_nums -join ','; - 'core_pretest_pr_nums' = $core_pretest_pr_nums -join ','; + foreach ($pr_num in $subm.merge_sha_by_pr.Keys) { + Write-Host "- PR $pr_num" + + function Is-SkipNeeded() { + if ($pr_num -notin $subm.pretest_sha_by_pr.Keys) { return $false } + if ($pr_num -notin $core_pretest_pr_nums[$subm.repo]) { return $false } + + function Find-MetaSHA() { + param ($ref, $key) + $pattern = "^meta\.$key=([0-9a-f]{40})$" + + Write-Host "--- Looking for commit message lines matching [$pattern]" + $res = @() + foreach ($l in git log -1 --pretty=format:"%B" $ref) { + Write-Host "----- $l" + if ($l -notmatch $pattern) { continue } + $sha = $matches[1] + Write-Host "------- FOUND: $sha" + $res += $sha + } + if (-not $?) { throw "git log failed" } + + return $res + } + + if ($subm.merge_sha_by_pr[$pr_num] -notin Find-MetaSHA "pretest/$pr_num" 'pr_merge_sha') { return $false } + if ($subm.pretest_sha_by_pr[$pr_num] -notin Find-MetaSHA "subm-pretest/$($subm.repo)/$pr_num" 'fork_pretest_sha') { return $false } + + return $true + } + if (Is-SkipNeeded) { continue } + + Write-Host "--- TEST NOW" + $exec_list += [PSCustomObject]@{ + 'subm_name' = $subm.name; + 'org_repo' = $subm.repo; + 'fork_repo' = "SunSerega/$($subm.name)"; + 'pr_num' = $pr_num; } } @@ -114,6 +208,40 @@ jobs: Write-Host $json echo "exec_list=$json" >> $env:GITHUB_OUTPUT + + + Write-Host "==============================" + Write-Host "Building remove_list" + $remove_list = @() + foreach ($subm in $submodules) { + Write-Host "subm: $($subm.name)" + + $subm_rem_pretest_pr_nums = @() + foreach ($pr_num in $subm.pretest_sha_by_pr.Keys) { + Write-Host "- Fork pretest for PR $pr_num" + if ($pr_num -in $subm.merge_sha_by_pr.Keys) { continue } + $subm_rem_pretest_pr_nums += $pr_num + Write-Host "--- REMOVE" + } + + $core_rem_pretest_pr_nums = @() + foreach ($pr_num in $core_pretest_pr_nums[$subm.repo]) { + Write-Host "- Core pretest for PR $pr_num" + if ($pr_num -in $subm.merge_sha_by_pr.Keys) { continue } + $core_rem_pretest_pr_nums += $pr_num + } + + if ($subm_rem_pretest_pr_nums -or $core_rem_pretest_pr_nums) { + $remove_list += [PSCustomObject]@{ + 'org_repo' = $subm.repo; + 'fork_repo' = "SunSerega/$($subm.name)"; + 'subm_pretest_pr_nums' = $subm_rem_pretest_pr_nums -join ','; + 'core_pretest_pr_nums' = $core_rem_pretest_pr_nums -join ','; + } + } + + } + $json = ConvertTo-Json -Compress $remove_list Write-Host $json echo "remove_list=$json" >> $env:GITHUB_OUTPUT @@ -191,28 +319,35 @@ jobs: git checkout $fork_branch_name Write-Host "Merging latest commits from PR $pr_num" - git merge "remotes/0_official/pull/$pr_num/head" + git merge "remotes/0_official/pull/$pr_num/merge" if (-not $?) { throw "git merge failed" } git push if (-not $?) { throw "git push failed" } } else { Write-Host "Branch for PR $pr_num doesn't exist" - git checkout "remotes/0_official/pull/$pr_num/head" + git checkout "remotes/0_official/pull/$pr_num/merge" + if (-not $?) { throw "git checkout failed" } Write-Host "Creating:" git checkout -b $fork_branch_name + if (-not $?) { throw "git checkout -b failed" } git push --set-upstream origin $fork_branch_name if (-not $?) { throw "git push failed" } } Write-Host "Merging with ${fork_main_branch_name}:" - git merge $fork_main_branch_name + $pr_merge_sha = git rev-parse remotes/0_official/pull/$pr_num/merge + git merge $fork_main_branch_name --no-ff --commit -m @" + Merge $fork_main_branch_name into $fork_branch_name + meta.pr_merge_sha=$pr_merge_sha + "@ if (-not $?) { throw "git merge failed" } git push if (-not $?) { throw "git push failed" } + $fork_pretest_sha = git rev-parse HEAD Pop-Location @@ -226,13 +361,17 @@ jobs: if (&{ git show-ref --verify -q "refs/remotes/origin/$core_branch_name"; $? }) { Write-Host "Branch for PR $pr_num exists. Merging with ${core_main_branch_name}:" git checkout $core_branch_name + if (-not $?) { throw "git checkout failed" } git merge $core_main_branch_name -m "[auto pretest] Merge $core_branch_name into subm-pretest/... branch" if (-not $?) { throw "git merge failed" } git push + if (-not $?) { throw "git push failed" } } else { Write-Host "Branch for PR $pr_num doesn't exist. Creating:" git checkout -b $core_branch_name + if (-not $?) { throw "git checkout -b failed" } git push --set-upstream origin $core_branch_name + if (-not $?) { throw "git push failed" } } & .\DeleteAllTemp.bat NoPause | Out-Null @@ -243,8 +382,11 @@ jobs: & .\PackAll.exe "Stages= PullUpstream + Reference + Dummy + OpenCL+OpenCLABC + OpenGL+OpenGLABC + Compile + Test + Release" "PullUpstreamBranch=${subm_name}:${fork_branch_name}" "PasCompPath=D:\PABCNETC\pabcnetcclear.exe" SkipFinishedPause if (-not $?) { throw "PackAll failed" } - git add -A - git commit -m "[auto pretest] test changes" + git commit -a --allow-empty -m @" + [auto pretest] test changes + meta.fork_pretest_sha=$fork_pretest_sha + "@ + if (-not $?) { throw "git commit failed" } git push if (-not $?) { throw "git push failed" } @@ -320,6 +462,7 @@ jobs: if ($is_trivial) { Write-Host "Is trivial, removing..." git push origin --delete $branch_name # BRANCH DELETE + if (-not $?) { throw "git push --delete failed" } $core_removed_pr_nums += $pr_num } Write-Host '~~~~~' @@ -339,6 +482,7 @@ jobs: } else { Write-Host "Removing..." git push origin --delete "pretest/$pr_num" # BRANCH DELETE + if (-not $?) { throw "git push --delete failed" } } Write-Host '~~~~~' } diff --git a/Delete subm pretest branches.ps1 b/Delete subm pretest branches.ps1 new file mode 100644 index 00000000..3c8327e9 --- /dev/null +++ b/Delete subm pretest branches.ps1 @@ -0,0 +1,40 @@ +try { + + function Delete-Branches { + param ( + [string]$folder, + [string]$remote, + [string]$pattern + ) + + Push-Location $folder + + git fetch $remote + + $branch_del = @() + foreach ($branch in git branch -r --list "$remote/$pattern") { + $branch = $branch.Trim().SubString($remote.Length+1) + Write-Host "Deleting branch: $branch" + $branch_del += $branch + } + git push $remote --delete $branch_del + + Pop-Location + } + + + + Delete-Branches '.\DataScraping\Reps\OpenCL-Docs\' 'SunSerega' 'pretest/*' + Delete-Branches '.\DataScraping\Reps\OpenGL-Registry\' 'SunSerega' 'pretest/*' + Delete-Branches '.' 'origin' 'subm-pretest/*' + + + +} +catch { + Write-Host "An error occurred:" + Write-Host $_ + #pause + exit 1 +} +pause \ No newline at end of file