diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..2f6f3275 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,373 @@ +name: Benchmark + +on: + # TODO: Remove pull_request trigger after testing; keep only issue_comment and workflow_dispatch. + pull_request: + issue_comment: + types: [created] + workflow_dispatch: + inputs: + base-ref: + description: 'Base ref to compare against (branch, tag, or SHA)' + required: true + default: 'v2' + +permissions: + contents: read + pull-requests: write + issues: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number || github.ref }} + cancel-in-progress: true + +jobs: + setup: + if: >- + github.event_name == 'workflow_dispatch' || + github.event_name == 'pull_request' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + contains(github.event.comment.body, '/bench')) + runs-on: ubuntu-latest + outputs: + head-ref: ${{ steps.resolve.outputs.head-ref }} + base-ref: ${{ steps.resolve.outputs.base-ref }} + pr-number: ${{ steps.resolve.outputs.pr-number }} + steps: + - name: React to comment + if: github.event_name == 'issue_comment' + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes', + }); + + - name: Resolve refs + id: resolve + uses: actions/github-script@v7 + with: + script: | + if (context.eventName === 'issue_comment') { + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + core.setOutput('head-ref', pr.data.head.sha); + core.setOutput('base-ref', pr.data.base.ref); + core.setOutput('pr-number', String(context.issue.number)); + } else if (context.eventName === 'pull_request') { + core.setOutput('head-ref', context.payload.pull_request.head.sha); + core.setOutput('base-ref', context.payload.pull_request.base.ref); + core.setOutput('pr-number', String(context.payload.pull_request.number)); + } else { + core.setOutput('head-ref', context.sha); + core.setOutput('base-ref', '${{ inputs.base-ref }}'); + core.setOutput('pr-number', ''); + } + + bench-ubuntu: + needs: setup + strategy: + fail-fast: false + matrix: + go-version: [1.25.x, 1.26.x] + openssl-version: [3.0.1, 3.0.13, 3.1.5, 3.2.1, 3.3.0, 3.3.1, 3.4.0, 3.5.0] + host: [ubuntu-22.04, ubuntu-24.04-arm] + runs-on: ${{ matrix.host }} + steps: + - name: Install build tools + run: sudo apt-get update && sudo apt-get install -y build-essential + + - name: Install OpenSSL + run: | + # Checkout just the install script first + git clone --depth 1 --filter=blob:none --sparse ${{ github.server_url }}/${{ github.repository }} _openssl_setup + cd _openssl_setup && git sparse-checkout set scripts + cd .. + sudo sh ./_openssl_setup/scripts/openssl.sh ${{ matrix.openssl-version }} + rm -rf _openssl_setup + + - name: Checkout HEAD + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.setup.outputs.head-ref }} + path: head + + - name: Checkout BASE + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.setup.outputs.base-ref }} + path: base + + - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: ${{ matrix.go-version }} + cache-dependency-path: head/go.mod + + - name: Install benchstat + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Run benchmarks (base) + working-directory: base + run: | + export GO_OPENSSL_VERSION_OVERRIDE=${{ matrix.openssl-version }} + export CGO_ENABLED=1 + go test -run='^$' -bench=. -count=7 -benchmem -timeout 30m ./... 2>&1 | tee ../base.txt || true + + - name: Run benchmarks (head) + working-directory: head + run: | + export GO_OPENSSL_VERSION_OVERRIDE=${{ matrix.openssl-version }} + export CGO_ENABLED=1 + go test -run='^$' -bench=. -count=7 -benchmem -timeout 30m ./... 2>&1 | tee ../head.txt || true + + - name: Check for test failures + id: test-failures + run: | + { + grep -E '^(FAIL|---\s*FAIL)' base.txt | sed 's/^/base: /' || true + grep -E '^(FAIL|---\s*FAIL)' head.txt | sed 's/^/head: /' || true + } > failures.txt + if [ -s failures.txt ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Compare benchmarks + id: benchstat + run: | + benchstat base.txt head.txt | tee benchstat.txt + THRESHOLD="5" + if awk -v threshold="$THRESHOLD" ' + /^[[:space:]]*│.*(sec\/op|B\/op|allocs\/op)/ { header=1; next } + /^$/ { header=0 } + header && match($0, /\+([0-9]+\.[0-9]+)% \(p=/, m) { if (m[1]+0 >= threshold) found=1 } + END { exit !found } + ' benchstat.txt; then + echo "regression=true" >> "$GITHUB_OUTPUT" + else + echo "regression=false" >> "$GITHUB_OUTPUT" + fi + + - name: Save status + if: always() + run: | + echo 'regression=${{ steps.benchstat.outputs.regression }}' > status.txt + echo 'test_failures=${{ steps.test-failures.outputs.found }}' >> status.txt + + - name: Upload benchstat results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchstat-ubuntu-${{ matrix.host }}-ossl${{ matrix.openssl-version }}-go${{ matrix.go-version }} + path: | + benchstat.txt + failures.txt + status.txt + + - name: Fail on regression or test failure + if: steps.benchstat.outputs.regression == 'true' || steps.test-failures.outputs.found == 'true' + run: | + echo "Benchmark regression or test failure detected." + exit 1 + + bench-azl3: + needs: setup + strategy: + fail-fast: false + matrix: + go-version: [1.25.x, 1.26.x] + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + steps: + - name: Install build tools + run: | + tdnf install -y --nogpgcheck azurelinux-repos + tdnf install -y gcc glibc-devel openssl-devel git tar + + - name: Checkout HEAD + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.setup.outputs.head-ref }} + path: head + + - name: Checkout BASE + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.setup.outputs.base-ref }} + path: base + + - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: ${{ matrix.go-version }} + cache-dependency-path: head/go.mod + + - name: Install benchstat + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Run benchmarks (base) + working-directory: base + run: | + export CGO_ENABLED=1 + go test -run='^$' -bench=. -count=7 -benchmem -timeout 30m ./... 2>&1 | tee ../base.txt || true + + - name: Run benchmarks (head) + working-directory: head + run: | + export CGO_ENABLED=1 + go test -run='^$' -bench=. -count=7 -benchmem -timeout 30m ./... 2>&1 | tee ../head.txt || true + + - name: Check for test failures + id: test-failures + run: | + { + grep -E '^(FAIL|---\s*FAIL)' base.txt | sed 's/^/base: /' || true + grep -E '^(FAIL|---\s*FAIL)' head.txt | sed 's/^/head: /' || true + } > failures.txt + if [ -s failures.txt ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Compare benchmarks + id: benchstat + run: | + benchstat base.txt head.txt | tee benchstat.txt + THRESHOLD="5" + if awk -v threshold="$THRESHOLD" ' + /^[[:space:]]*│.*(sec\/op|B\/op|allocs\/op)/ { header=1; next } + /^$/ { header=0 } + header && match($0, /\+([0-9]+\.[0-9]+)% \(p=/, m) { if (m[1]+0 >= threshold) found=1 } + END { exit !found } + ' benchstat.txt; then + echo "regression=true" >> "$GITHUB_OUTPUT" + else + echo "regression=false" >> "$GITHUB_OUTPUT" + fi + + - name: Save status + if: always() + run: | + echo 'regression=${{ steps.benchstat.outputs.regression }}' > status.txt + echo 'test_failures=${{ steps.test-failures.outputs.found }}' >> status.txt + + - name: Upload benchstat results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchstat-azl3-go${{ matrix.go-version }} + path: | + benchstat.txt + failures.txt + status.txt + + - name: Fail on regression or test failure + if: steps.benchstat.outputs.regression == 'true' || steps.test-failures.outputs.found == 'true' + run: | + echo "Benchmark regression or test failure detected." + exit 1 + + conclusion: + needs: [setup, bench-ubuntu, bench-azl3] + runs-on: ubuntu-latest + if: always() && needs.setup.result == 'success' + steps: + - name: Download all benchstat artifacts + uses: actions/download-artifact@v4 + with: + path: results + pattern: benchstat-* + + - name: Build report + run: | + { + echo "## Benchmark Results" + echo "" + if [[ "${{ needs.bench-ubuntu.result }}" == "failure" || "${{ needs.bench-azl3.result }}" == "failure" ]]; then + echo ":warning: **Regressions detected** (>= 5%, statistically significant)" + else + echo ":white_check_mark: **No significant regressions detected**" + fi + echo "" + for dir in results/benchstat-*; do + [ -d "$dir" ] || continue + label="${dir#results/benchstat-}" + icon=":white_check_mark:" + if [ -f "$dir/status.txt" ]; then + grep -q 'regression=true' "$dir/status.txt" && icon=":x:" + grep -q 'test_failures=true' "$dir/status.txt" && icon=":x:" + fi + echo "
" + echo "${icon} $label" + echo "" + if [ -f "$dir/failures.txt" ] && [ -s "$dir/failures.txt" ]; then + echo "**Test failures:**" + echo '```' + cat "$dir/failures.txt" + echo '```' + echo "" + fi + if [ -f "$dir/benchstat.txt" ]; then + echo "**Benchstat:**" + echo '```' + cat "$dir/benchstat.txt" + echo '```' + fi + echo "" + echo "
" + echo "" + done + } > report.md + + - name: Post PR comment + if: needs.setup.outputs.pr-number != '' + uses: actions/github-script@v7 + env: + PR_NUMBER: ${{ needs.setup.outputs.pr-number }} + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('report.md', 'utf8'); + const prNumber = parseInt(process.env.PR_NUMBER, 10); + const marker = ''; + const fullBody = marker + '\n' + body; + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + }); + const existing = comments.find(c => c.body.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: fullBody, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: fullBody, + }); + } + + - name: Check result + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more benchmark jobs detected regressions or were cancelled." + exit 1 + fi + echo "All benchmark jobs passed." \ No newline at end of file