From 5035079a96a41a42972547dff04c8128cbeefc0a Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Wed, 6 May 2026 17:54:42 -0700 Subject: [PATCH] run PR builds in fork and mirror status to apache repo --- .asf.yaml | 4 +- .../automatic-email-notif-on-ddl-change.yml | 72 +++- .github/workflows/check-header.yml | 1 - .github/workflows/fork-ci.yml | 99 +++++ .github/workflows/notify_test_workflow.yml | 350 ++++++++++++++++++ .github/workflows/required-checks.yml | 19 +- .github/workflows/update_build_status.yml | 186 ++++++++++ 7 files changed, 705 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/fork-ci.yml create mode 100644 .github/workflows/notify_test_workflow.yml create mode 100644 .github/workflows/update_build_status.yml diff --git a/.asf.yaml b/.asf.yaml index 14e9e9f4c17..0d7ba4bfaac 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -67,10 +67,8 @@ github: required_status_checks: # strict means "Require branches to be up to date before merging". strict: true - # contexts are the names of checks that must pass contexts: - - Required Checks - - Check License Headers + - Build - Validate PR title required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/workflows/automatic-email-notif-on-ddl-change.yml b/.github/workflows/automatic-email-notif-on-ddl-change.yml index 1c5f96c4403..35069b72566 100644 --- a/.github/workflows/automatic-email-notif-on-ddl-change.yml +++ b/.github/workflows/automatic-email-notif-on-ddl-change.yml @@ -16,33 +16,75 @@ name: Automatic email notification on DDL change +# Triggered post-merge on push to main when sql/updates/** changes. Was +# previously `pull_request: closed`, which queued for first-time-contributor +# approval on every fork PR even though the job condition (`merged == true`) +# meant it never actually ran on PR open. Push trigger fires only on actual +# main-branch updates and needs no approval. on: - pull_request: - types: - - closed + push: + branches: [main] + paths: + - 'sql/updates/**' + +permissions: + contents: read + pull-requests: read jobs: notify: - if: >- - github.event.pull_request.merged == true && - contains(github.event.pull_request.labels.*.name, 'ddl-change') runs-on: ubuntu-latest steps: + - name: Resolve PR for this commit + id: pr + uses: actions/github-script@v8 + with: + script: | + const pulls = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: context.sha, + }) + const pr = pulls.data[0] + if (!pr) { + console.log('No PR associated with ' + context.sha + '; skipping email') + core.setOutput('skip', 'true') + return + } + const hasLabel = pr.labels.some(l => l.name === 'ddl-change') + if (!hasLabel) { + console.log('PR #' + pr.number + ' has no ddl-change label; skipping email') + core.setOutput('skip', 'true') + return + } + core.setOutput('skip', 'false') + core.setOutput('number', String(pr.number)) + core.setOutput('title', pr.title) + core.setOutput('html_url', pr.html_url) + core.setOutput('user', pr.user.login) + - name: Checkout + if: steps.pr.outputs.skip == 'false' uses: actions/checkout@v5 with: - fetch-depth: 0 + fetch-depth: 2 sparse-checkout: sql/updates/ - - name: Get added file in sql/updates/ + - name: Find added SQL update file + if: steps.pr.outputs.skip == 'false' id: get_sql_file run: | - FILE=$(git diff --name-only --diff-filter=A \ - ${{ github.event.pull_request.base.sha }} \ - ${{ github.event.pull_request.merge_commit_sha }} \ - -- 'sql/updates/') + FILE=$(git diff --name-only --diff-filter=A HEAD~1 HEAD -- 'sql/updates/') echo "sql_file=$FILE" >> $GITHUB_OUTPUT + - name: Send email + if: steps.pr.outputs.skip == 'false' + env: + PR_NUMBER: ${{ steps.pr.outputs.number }} + PR_TITLE: ${{ steps.pr.outputs.title }} + PR_URL: ${{ steps.pr.outputs.html_url }} + PR_USER: ${{ steps.pr.outputs.user }} + SQL_FILE: ${{ steps.get_sql_file.outputs.sql_file }} run: | curl --ssl-reqd \ --url "smtps://smtp.gmail.com:465" \ @@ -57,6 +99,6 @@ jobs: Content-Type: text/html

Hi all,

-

We have merged PR #${{ github.event.pull_request.number }} (${{ github.event.pull_request.html_url }}): ${{ github.event.pull_request.title }}. To incorporate the change, please apply ${{ steps.get_sql_file.outputs.sql_file }} to your local Postgres instance and run sbt jooqGenerate to generate jooq classes.

-

Best,
${{ github.event.pull_request.user.login }}

- EOF \ No newline at end of file +

We have merged PR #${PR_NUMBER} (${PR_URL}): ${PR_TITLE}. To incorporate the change, please apply ${SQL_FILE} to your local Postgres instance and run sbt jooqGenerate to generate jooq classes.

+

Best,
${PR_USER}

+ EOF diff --git a/.github/workflows/check-header.yml b/.github/workflows/check-header.yml index 5557bc5c781..285ccf289d6 100644 --- a/.github/workflows/check-header.yml +++ b/.github/workflows/check-header.yml @@ -21,7 +21,6 @@ on: branches: - 'ci-enable/**' - 'main' - pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/fork-ci.yml b/.github/workflows/fork-ci.yml new file mode 100644 index 00000000000..0fedb3fd445 --- /dev/null +++ b/.github/workflows/fork-ci.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fork CI — Apache Spark model +# +# Runs the full PR-gating suite in the contributor's fork on every branch +# push. Mirrors what required-checks.yml runs in apache/texera on a +# ci-enable/** push: build matrix + license headers, aggregated into a +# single "fork-ci" result that notify_test_workflow.yml surfaces on the PR +# as the "Build" commit status. +# +# This workflow is a no-op in the canonical apache/texera repository — +# every job is guarded by (github.repository_owner != 'apache') so it +# never consumes main-repo runner quota. +# +# Secrets (CODECOV_TOKEN, NX_CLOUD_ACCESS_TOKEN) are unavailable in forks +# and degrade gracefully: Codecov uploads are skipped (fail_ci_if_error: +# false) and NX Cloud remote caching is disabled; builds still pass. + +name: Fork CI + +on: + push: + branches-ignore: + - main + - 'release/**' + - 'ci-enable/**' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + if: github.repository_owner != 'apache' + uses: ./.github/workflows/build.yml + with: + run_frontend: true + run_amber: true + run_amber_integration: true + run_platform: true + run_python: true + run_agent_service: true + secrets: inherit + + check-headers: + name: Check License Headers + if: github.repository_owner != 'apache' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: apache/skywalking-eyes@5c5b974209f0de5d905f37deb69369068ebfc15c # v0.7.0 + + fork-ci: + # Aggregator job — its conclusion is what notify_test_workflow.yml + # mirrors as the PR's "Build" status. Mirrors required-checks.yml's + # final required-checks job. Runs even if upstream jobs failed so the + # aggregate result is always reported. + name: Fork CI + needs: [build, check-headers] + if: always() && github.repository_owner != 'apache' + runs-on: ubuntu-latest + steps: + - name: Verify all fork CI jobs succeeded or were skipped + run: | + declare -A results=( + [build]="${{ needs.build.result }}" + [check-headers]="${{ needs.check-headers.result }}" + ) + failed=0 + for job in "${!results[@]}"; do + r="${results[$job]}" + echo "${job}: ${r}" + if [[ "$r" != "success" && "$r" != "skipped" ]]; then + failed=1 + fi + done + if (( failed )); then + echo "::error::One or more fork CI jobs did not succeed." + exit 1 + fi + echo "All fork CI jobs succeeded or were skipped." diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml new file mode 100644 index 00000000000..01dff7f4e90 --- /dev/null +++ b/.github/workflows/notify_test_workflow.yml @@ -0,0 +1,350 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: On pull request update +on: + pull_request_target: + types: [opened, reopened, synchronize] + +jobs: + notify: + name: Notify test workflow + runs-on: ubuntu-latest + permissions: + actions: read + statuses: write + pull-requests: write + steps: + - name: "Notify test workflow" + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const statusContext = 'Build' + const pr = context.payload.pull_request + const head_sha = pr.head.sha + const head_ref = pr.head.ref + const headRepo = pr.head.repo ? pr.head.repo.full_name : null + const baseRepo = context.repo.owner + '/' + context.repo.repo + + // Used as target_url for failure scenarios where there's no fork + // run to link to — the notify run's logs are the next-best + // explanation. Commit statuses honor target_url (unlike check_runs + // created via GITHUB_TOKEN, which silently overrides details_url). + const notifyRunUrl = context.serverUrl + '/' + context.repo.owner + '/' + context.repo.repo + '/actions/runs/' + context.runId + + console.log('=== Fork CI detection ===') + console.log('PR #' + pr.number + ': ' + (headRepo || '') + ':' + head_ref + ' -> ' + baseRepo + ':' + pr.base.ref) + console.log(' head sha: ' + head_sha) + console.log(' event: ' + context.payload.action) + console.log(' notify run: ' + notifyRunUrl) + + // Post the initial Build status as `pending` so the required + // status appears on the PR the moment notify starts. Every + // scenario below routes through setBuildStatus to update it + // in place. + async function setBuildStatus(state, description, target_url) { + const desc = (description || '').slice(0, 140) + for (let attempt = 1; attempt <= 3; attempt++) { + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: head_sha, + state: state, + context: statusContext, + description: desc, + target_url: target_url, + }) + console.log(' -> Build status set: state=' + state + ' target_url=' + target_url) + return + } catch (error) { + if (attempt < 3) { + const delay = 1000 * Math.pow(2, attempt - 1) + console.error(' -> setBuildStatus attempt ' + attempt + '/3 failed: ' + error.message + '; retrying in ' + (delay / 1000) + 's') + await new Promise(r => setTimeout(r, delay)) + } else { + console.error(' -> FAILED to set Build status after 3 attempts: ' + error.message) + core.setFailed('Could not set Build status after retries: ' + error.message) + } + } + } + } + + await setBuildStatus('pending', 'Detecting fork CI run for ' + head_sha.substring(0, 8), notifyRunUrl) + + // Post a comment once per unique marker. Idempotent across re-runs + // of this workflow on the same PR (opened/reopened/synchronize). + async function postOnceComment(marker, body) { + try { + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + per_page: 100, + }) + if (comments.some(c => c.body && c.body.includes(marker))) { + console.log(' comment "' + marker + '" already on PR; not reposting') + return + } + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body: marker + '\n' + body, + }) + console.log(' posted comment "' + marker + '"') + } catch (error) { + console.error(' failed to post comment "' + marker + '": ' + error.message) + } + } + + // Failure scenarios: red ✗ Build status, with target_url pointing + // somewhere useful for the committer to dig in. The detailed + // explanation lives in the postOnceComment on the PR conversation. + async function failCheck(title, target_url) { + await setBuildStatus('failure', title, target_url || notifyRunUrl) + core.setFailed(title) + } + + // Success scenario (non-fork PR): leaves the workflow green. + async function passCheck(title, target_url) { + await setBuildStatus('success', title, target_url || notifyRunUrl) + } + + // Map the fork run's actual status onto the Build commit status: + // completed/success -> success + // completed/cancelled -> error (yellow !) + // completed/ -> failure (red x) + // queued / in_progress -> pending (yellow circle) + // target_url always goes to the fork CI run's actions page so the + // PR's Details link is one-click to the fork run. + async function linkCheck(run, actions_url) { + let state, description + if (run.status === 'completed') { + state = run.conclusion === 'success' ? 'success' + : run.conclusion === 'cancelled' ? 'error' + : 'failure' + description = 'Fork CI run #' + run.id + ' ' + run.conclusion + } else { + state = 'pending' + description = 'Fork CI run #' + run.id + ' ' + run.status + } + await setBuildStatus(state, description, actions_url) + } + + // ----- scenario 1: fork repo gone ---------------------------------- + if (!pr.head.repo) { + console.log('SCENARIO: head repo is null (fork deleted, made private, or otherwise inaccessible)') + await postOnceComment( + '', + ':no_entry: **Fork CI cannot run — your fork repository is not accessible.**\n' + + '\n' + + 'GitHub returned `null` for this PR\'s head repository. That usually means one of:\n' + + '\n' + + '- The fork has been **deleted**\n' + + '- The fork has been **made private** (Fork CI only works on public forks)\n' + + '- The branch has been **force-removed**\n' + + '\n' + + 'Without access to your fork, this workflow cannot detect any `fork-ci.yml` runs and the required `Build` status will stay failed.\n' + + '\n' + + '**To fix:** restore (or recreate) the public fork, push your branch back, then close and reopen this PR (or push a new commit) to retrigger detection.\n' + ) + await failCheck('Fork repository not accessible', pr.html_url) + return + } + + // ----- scenario 2: non-fork PR (branch in base repo) --------------- + if (headRepo === baseRepo) { + console.log('SCENARIO: non-fork PR (head repo == base repo)') + await postOnceComment( + '', + ':information_source: **Fork CI is not applicable to this PR.**\n' + + '\n' + + 'This PR is opened from a branch inside `' + baseRepo + '` itself, not from a fork. ' + + 'Fork CI is the system that runs the full build matrix in *contributor forks* and surfaces the result on PRs to `apache/texera`. ' + + 'Since there\'s no fork involved here, the `Build` status has been auto-passed.\n' + + '\n' + + 'In-tree branches like this one are gated by the **Required Checks** status check (which runs builds directly in `' + baseRepo + '`).\n' + ) + await passCheck( + 'Fork CI not applicable (in-tree PR)', + context.serverUrl + '/' + baseRepo + '/actions/workflows/required-checks.yml' + ) + return + } + + // ----- scenario 3: blocked branch in fork -------------------------- + // These branch patterns are listed in fork-ci.yml's `branches-ignore`, + // so no run will ever exist for them in the fork. We fail the status + // immediately with a specific, branch-aware explanation rather than + // burning 60s polling for a run that can't appear. + const BLOCKED_PATTERNS = [ + { + label: 'main', + matches: ref => ref === 'main', + marker: '', + why: + 'Fork CI **deliberately ignores pushes to `main` in forks** (see `branches-ignore` in `.github/workflows/fork-ci.yml`).\n' + + '\n' + + 'The reason: every time you sync your fork from `apache/texera` upstream, the sync pushes new commits to your fork\'s `main`. ' + + 'Without this exclusion, every sync would burn your fork\'s GitHub Actions minutes rebuilding commits that already passed CI upstream — hundreds of minutes per week for active contributors.', + fix: + '**Open this PR from a feature branch instead:**\n' + + '\n' + + '```bash\n' + + 'git checkout -b feat/your-change\n' + + 'git push origin feat/your-change\n' + + '# close this PR and open a new one from feat/your-change -> ' + baseRepo + ':' + pr.base.ref + '\n' + + '```\n' + + '\n' + + 'See `AGENTS.md` (Branch and commit naming) for the conventional branch prefixes (`feat/`, `fix/`, `chore/`, `ci/`, `test/`).', + }, + { + label: 'release/**', + matches: ref => ref.startsWith('release/'), + marker: '', + why: + 'Fork CI ignores pushes to `release/**` in forks (see `branches-ignore` in `.github/workflows/fork-ci.yml`).\n' + + '\n' + + '`release/**` branches are reserved for `apache/texera`\'s backport coordination workflow — backports are driven by `release/` *labels* on PRs, not by a fork branch with that name. Running fork CI on such a branch would conflict with the canonical release pipeline.', + fix: + '**Open this PR from a normal feature branch.** If you need the change backported to a release line, after the PR is open add the `release/` label — `required-checks.yml` will then run the backport matrix in `apache/texera`. See `AGENTS.md` (CI labels & gating) for details.', + }, + { + label: 'ci-enable/**', + matches: ref => ref.startsWith('ci-enable/'), + marker: '', + why: + 'Fork CI ignores pushes to `ci-enable/**` in forks (see `branches-ignore` in `.github/workflows/fork-ci.yml`).\n' + + '\n' + + '`ci-enable/**` is a special branch namespace **inside `apache/texera`** that committers use to run `required-checks.yml` directly on a push (without opening a PR). It\'s not a contributor branch — using it from a fork has no effect.', + fix: + '**Open this PR from a normal feature branch** (`feat/...`, `fix/...`, etc.). If you\'re a committer who actually wants the `ci-enable/**` behavior, push the branch directly to `apache/texera` instead of going through a fork.', + }, + ] + + const blocked = BLOCKED_PATTERNS.find(p => p.matches(head_ref)) + if (blocked) { + console.log('SCENARIO: head ref "' + head_ref + '" matches blocked pattern "' + blocked.label + '"') + await postOnceComment( + blocked.marker, + ':no_entry: **Fork CI cannot run on `' + head_ref + '` in your fork.**\n' + + '\n' + + '**Why:**\n' + + '\n' + + blocked.why + '\n' + + '\n' + + '**How to fix:**\n' + + '\n' + + blocked.fix + '\n' + + '\n' + + '---\n' + + '\n' + + 'The required `Build` status will stay failed on this PR until you reopen it from an unblocked branch. The full `branches-ignore` list lives in `.github/workflows/fork-ci.yml`.\n' + ) + await failCheck('Fork CI not allowed on ' + head_ref, pr.html_url) + return + } + + // ----- scenario 4: detection --------------------------------------- + // Poll the fork for a fork-ci.yml run matching this PR's head SHA. + // GitHub Actions can take 10–30s+ to register a queued run, so a + // single 3s wait misses most runs. Poll up to ~60s and search the + // 10 most recent runs (not just the first) so a stale earlier run + // on the same branch can't shadow the right one. + console.log('SCENARIO: detecting fork-ci.yml run on ' + headRepo + ':' + head_ref + ' @ ' + head_sha) + const maxAttempts = 7 + const delays = [3000, 7000, 10000, 10000, 10000, 10000, 10000] + let matchingRun = null + let lastSeenRuns = [] + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + await new Promise(r => setTimeout(r, delays[attempt - 1])) + + let runs + try { + runs = await github.request( + 'GET /repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs', + { + owner: pr.head.repo.owner.login, + repo: pr.head.repo.name, + workflow_id: 'fork-ci.yml', + branch: head_ref, + per_page: 10, + } + ) + } catch (error) { + console.log(' attempt ' + attempt + '/' + maxAttempts + ': API error: ' + error.message) + continue + } + + lastSeenRuns = runs.data.workflow_runs + console.log(' attempt ' + attempt + '/' + maxAttempts + ': ' + lastSeenRuns.length + ' fork-ci runs on ' + head_ref) + for (const r of lastSeenRuns) { + console.log(' run id=' + r.id + ' sha=' + r.head_sha + ' status=' + r.status + ' conclusion=' + r.conclusion) + } + + matchingRun = lastSeenRuns.find(r => r.head_sha === head_sha) + if (matchingRun) { + console.log(' match: run id=' + matchingRun.id) + break + } + console.log(' no run matches PR head sha ' + head_sha + ' yet') + } + + if (!matchingRun) { + console.log('SCENARIO: detection failed after ' + maxAttempts + ' attempts (' + lastSeenRuns.length + ' unrelated runs visible)') + const seenSummary = lastSeenRuns.length === 0 + ? 'No `fork-ci.yml` runs on `' + head_ref + '` were visible at all — most likely cause is **GitHub Actions is disabled in your fork**.' + : '`fork-ci.yml` runs were visible on `' + head_ref + '`, but **none of them were for commit `' + head_sha + '`**. The most likely cause is that the run for the commit you pushed hasn\'t started yet, or the workflow file in your fork is out of date.' + await postOnceComment( + '', + ':warning: **Fork CI detection failed for `' + head_sha.substring(0, 8) + '`.**\n' + + '\n' + + seenSummary + '\n' + + '\n' + + '**Things to check:**\n' + + '\n' + + '1. **GitHub Actions enabled in your fork.** Visit the [Actions tab on `' + headRepo + '`](https://github.com/' + headRepo + '/actions) and click "I understand my workflows, go ahead and enable them" if Actions are disabled.\n' + + '2. **Your fork\'s `fork-ci.yml` is up to date.** If your fork was created before fork CI was added to `apache/texera`, sync your fork from upstream:\n' + + ' ```bash\n' + + ' git fetch upstream\n' + + ' git rebase upstream/main\n' + + ' git push origin ' + head_ref + ' --force-with-lease\n' + + ' ```\n' + + '3. **Retrigger detection.** After fixing the above, push an empty commit:\n' + + ' ```bash\n' + + ' git commit --allow-empty -m "retrigger fork CI"\n' + + ' git push\n' + + ' ```\n' + + ' The `synchronize` event will re-run this detection and `update_build_status.yml` will pick up the run on its next 5-minute poll.\n' + ) + const forkActionsUrl = 'https://github.com/' + headRepo + '/actions' + await failCheck('Fork CI run not detected for ' + head_sha.substring(0, 8), forkActionsUrl) + return + } + + // ----- scenario 5: detection success ------------------------------- + // Found a matching fork CI run — set the Build status to mirror + // its current state with target_url pointing at the fork run. + // update_build_status.yml will keep updating it as the run + // progresses (workflow_run trigger fires immediately after this + // workflow completes; cron polls every 5 minutes after that). + const actions_url = 'https://github.com/' + headRepo + '/actions/runs/' + matchingRun.id + console.log('SCENARIO: detection success — fork run id=' + matchingRun.id + ' status=' + matchingRun.status + ' conclusion=' + matchingRun.conclusion) + await linkCheck(matchingRun, actions_url) diff --git a/.github/workflows/required-checks.yml b/.github/workflows/required-checks.yml index 54c86006381..48760feb397 100644 --- a/.github/workflows/required-checks.yml +++ b/.github/workflows/required-checks.yml @@ -23,13 +23,6 @@ on: - 'ci-enable/**' - 'main' - 'release/**' - pull_request: - types: - - opened - - reopened - - synchronize - - labeled - - unlabeled workflow_dispatch: permissions: @@ -233,6 +226,18 @@ jobs: build: needs: precheck + # PR builds are owned by Fork CI: fork-ci.yml runs the full build matrix + # in the contributor's fork, and notify_test_workflow.yml / + # update_build_status.yml surface the result on the PR as the "Build" + # status check. Running build.yml here on pull_request would duplicate + # those builds (same code, same matrix) and double the project's runner + # minutes for no extra coverage. + # + # The base repo still runs the full build matrix for: + # - push events to main / release/** (post-merge validation) + # - push events to ci-enable/** (committers' escape hatch) + # - workflow_dispatch (manual reruns) + if: github.event_name != 'pull_request' uses: ./.github/workflows/build.yml with: run_frontend: ${{ needs.precheck.outputs.run_frontend == 'true' }} diff --git a/.github/workflows/update_build_status.yml b/.github/workflows/update_build_status.yml new file mode 100644 index 00000000000..4718dae9614 --- /dev/null +++ b/.github/workflows/update_build_status.yml @@ -0,0 +1,186 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Update build status workflow + +on: + workflow_run: + workflows: ["On pull request update"] + types: [completed] + schedule: + - cron: "*/5 * * * *" + workflow_dispatch: + +jobs: + update: + name: Update build status + runs-on: ubuntu-latest + # workflow_run mode polls up to 60 min waiting for fork CI completion. + # Other modes (cron, workflow_dispatch) finish in seconds. + timeout-minutes: 65 + permissions: + actions: read + statuses: write + steps: + - name: "Update build status" + uses: actions/github-script@v8 + env: + TRIGGER_EVENT: ${{ github.event_name }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const isLongPoll = process.env.TRIGGER_EVENT === 'workflow_run' + const maxIterations = isLongPoll ? 120 : 1 // 120 * 30s = 60 min + const sleepMs = 30000 + const statusContext = 'Build' + + console.log('=== update_build_status: ' + new Date().toISOString() + ' ===') + console.log('trigger=' + process.env.TRIGGER_EVENT + ' mode=' + (isLongPoll ? 'long-poll up to ' + maxIterations + ' iters' : 'single pass')) + + // Map fork run state -> commit status state. + function runToStatusState(run) { + if (run.status !== 'completed') return 'pending' + if (run.conclusion === 'success') return 'success' + if (run.conclusion === 'cancelled') return 'error' + return 'failure' + } + + // Find the most recent Build status for a SHA. /statuses/{sha} + // returns statuses in reverse chronological order; the first + // matching context is current. + async function getCurrentBuildStatus(sha) { + try { + const resp = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/statuses', { + owner: context.repo.owner, + repo: context.repo.repo, + ref: sha, + per_page: 100, + }) + return resp.data.find(s => s.context === statusContext) || null + } catch (error) { + console.log(' -> failed to read current Build status: ' + error.message) + return null + } + } + + // One pass over all open PRs. Returns { pendingCount, updateCount } + // — pendingCount is how many PRs are still in non-terminal state + // (used by the long-poll loop to decide whether to keep going). + async function syncOnce() { + const prsIter = github.paginate.iterator( + 'GET /repos/{owner}/{repo}/pulls', + { owner: context.repo.owner, repo: context.repo.repo, state: 'open', per_page: 100 } + ) + let prCount = 0 + let updateCount = 0 + let pendingCount = 0 + + for await (const prs of prsIter) { + for (const pr of prs.data) { + prCount++ + if (!pr.head.repo) continue + if (pr.head.repo.full_name === context.repo.owner + '/' + context.repo.repo) continue + + const current = await getCurrentBuildStatus(pr.head.sha) + // Don't fight notify's deliberate non-fork failure decisions + // (blocked branch, fork inaccessible). target_url for those + // points at the PR or notify run — never at the fork run. + if (current && current.state === 'failure') { + const isForkTarget = current.target_url && current.target_url.includes('/' + pr.head.repo.full_name + '/actions/runs/') + if (!isForkTarget) continue + } + if (current && current.state === 'success') { + const isForkTarget = current.target_url && current.target_url.includes('/' + pr.head.repo.full_name + '/actions/runs/') + if (isForkTarget) continue + } + + let runs + try { + runs = await github.request( + 'GET /repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs', + { + owner: pr.head.repo.owner.login, + repo: pr.head.repo.name, + workflow_id: 'fork-ci.yml', + branch: pr.head.ref, + per_page: 10, + } + ) + } catch (error) { + console.log(' PR #' + pr.number + ': fork API error ' + error.message) + pendingCount++ + continue + } + + const matching = runs.data.workflow_runs.find(r => r.head_sha === pr.head.sha) + if (!matching) { + pendingCount++ + continue + } + + const actions_url = 'https://github.com/' + pr.head.repo.full_name + '/actions/runs/' + matching.id + const newState = runToStatusState(matching) + const description = matching.status === 'completed' + ? 'Fork CI run #' + matching.id + ' ' + matching.conclusion + : 'Fork CI run #' + matching.id + ' ' + matching.status + + if (newState === 'pending') pendingCount++ + + // Idempotency: only POST if state, target_url, or description + // actually changed. Avoids history spam. + if (current + && current.state === newState + && current.target_url === actions_url + && current.description === description.slice(0, 140)) { + continue + } + + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: pr.head.sha, + state: newState, + context: statusContext, + description: description.slice(0, 140), + target_url: actions_url, + }) + updateCount++ + console.log(' PR #' + pr.number + ' (' + pr.head.sha.substring(0, 8) + '): synced state=' + newState + ' target=' + actions_url) + } catch (error) { + console.error(' PR #' + pr.number + ': POST status FAILED ' + (error.status || '?') + ' ' + error.message) + } + } + } + return { prCount, updateCount, pendingCount } + } + + for (let iter = 0; iter < maxIterations; iter++) { + if (iter > 0) { + console.log('--- iter ' + (iter + 1) + '/' + maxIterations + ' (sleeping ' + (sleepMs / 1000) + 's) ---') + await new Promise(r => setTimeout(r, sleepMs)) + } else { + console.log('--- iter 1/' + maxIterations + ' ---') + } + const { prCount, updateCount, pendingCount } = await syncOnce() + console.log(' scanned=' + prCount + ' updated=' + updateCount + ' pending=' + pendingCount) + if (pendingCount === 0) { + console.log('All PRs resolved; exiting after ' + (iter + 1) + ' iter(s)') + break + } + } + console.log('=== done ===')