Add ./branches.sh to generate the build matrix

PhrozenByte · PhrozenByte · commit bf955e2ea579 · 2026-02-18T12:59:57.000+01:00
`./branches.sh` will print an error when a new milestone is released, but wasn't added to `container.env` yet. We won't attempt to build that milestone, but rather let the new "Generate jobs" job fail. This failure will cause GitHub to send us a notification. However, we still proceed with building the configured milestones. The milestones to build are now configured in \`container.env\`, not in the GitHub workflow manifest.
diff --git a/.github/workflows/container-auto-update.yml b/.github/workflows/container-auto-update.yml
@@ -15,29 +15,60 @@ defaults:
     shell: bash -eu -o pipefail {0}
 
 jobs:
+  generate-jobs:
+    name: Generate jobs
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    outputs:
+      BUILD_MATRIX: ${{ steps.generate-jobs.outputs.BUILD_MATRIX }}
+
+    steps:
+      - name: Setup CI tools
+        run: |
+          . <(curl -fsS -L "$CI_TOOLS_SETUP" | bash -s ~/ci-tools)
+          echo "CI_TOOLS=$CI_TOOLS" | tee -a "$GITHUB_ENV"
+          echo "CI_TOOLS_PATH=$CI_TOOLS_PATH" | tee -a "$GITHUB_ENV"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Generate jobs
+        id: generate-jobs
+        run: |
+          MILESTONES="$(./branches.sh)" || EXIT_CODE=$?
+          
+          echo ::group::Jobs list
+          echo "$MILESTONES"
+          echo ::endgroup::
+          
+          echo "BUILD_MATRIX=$([ -z "$MILESTONES" ] || jq -Rcn '{"MILESTONE": [inputs]}' <<< "$MILESTONES")" >> $GITHUB_OUTPUT
+          exit ${EXIT_CODE:-0}
+
   build:
+    name: Build & publish container
+
+    needs: generate-jobs
+    if: ${{ always() && needs.generate-jobs.outputs.BUILD_MATRIX }}
+
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
 
+    concurrency: build-${{ matrix.MILESTONE }}
+
     strategy:
-      matrix:
-        BUILD_REF:
-          - refs/heads/main
-        BASE_IMAGE:
-          - docker.io/debian:bookworm
-          - docker.io/debian:bullseye
-          - docker.io/debian:trixie
+      matrix: ${{ fromJSON(needs.generate-jobs.outputs.BUILD_MATRIX) }}
       fail-fast: false
-      max-parallel: 1
 
     env:
       REGISTRY: ghcr.io
       OWNER: sgsgermany
       IMAGE: debian
-      BUILD_REF: ${{ matrix.BUILD_REF }}
-      BASE_IMAGE: ${{ matrix.BASE_IMAGE }}
+      MILESTONE: ${{ matrix.MILESTONE }}
 
     steps:
       - name: Setup CI tools
@@ -48,8 +79,6 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v4
-        with:
-          ref: ${{ env.BUILD_REF }}
 
       - name: Log into container registry ${{ env.REGISTRY }}
         uses: redhat-actions/podman-login@v1
diff --git a/.github/workflows/container-publish.yml b/.github/workflows/container-publish.yml
@@ -3,7 +3,12 @@ name: Build & publish container
 on:
   push:
     branches: [ main ]
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: Debian version branch
+        required: false
+        type: string
 
 concurrency: build
 
@@ -15,24 +20,62 @@ defaults:
     shell: bash -eu -o pipefail {0}
 
 jobs:
+  generate-jobs:
+    name: Generate jobs
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    outputs:
+      BUILD_MATRIX: ${{ steps.generate-jobs.outputs.BUILD_MATRIX }}
+
+    steps:
+      - name: Setup CI tools
+        run: |
+          . <(curl -fsS -L "$CI_TOOLS_SETUP" | bash -s ~/ci-tools)
+          echo "CI_TOOLS=$CI_TOOLS" | tee -a "$GITHUB_ENV"
+          echo "CI_TOOLS_PATH=$CI_TOOLS_PATH" | tee -a "$GITHUB_ENV"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Generate jobs
+        id: generate-jobs
+        env:
+          MILESTONES: ${{ inputs.branch }}
+        run: |
+          [ -n "$MILESTONES" ] || { MILESTONES="$(./branches.sh)" || EXIT_CODE=$?; }
+          
+          echo ::group::Jobs list
+          echo "$MILESTONES"
+          echo ::endgroup::
+          
+          echo "BUILD_MATRIX=$([ -z "$MILESTONES" ] || jq -Rcn '{"MILESTONE": [inputs]}' <<< "$MILESTONES")" >> $GITHUB_OUTPUT
+          exit ${EXIT_CODE:-0}
+
   build:
+    name: Build & publish container
+
+    needs: generate-jobs
+    if: ${{ always() && needs.generate-jobs.outputs.BUILD_MATRIX }}
+
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
 
+    concurrency: build-${{ matrix.MILESTONE }}
+
     strategy:
-      matrix:
-        BASE_IMAGE:
-          - docker.io/debian:bookworm
-          - docker.io/debian:bullseye
-          - docker.io/debian:trixie
+      matrix: ${{ fromJSON(needs.generate-jobs.outputs.BUILD_MATRIX) }}
+      fail-fast: false
 
     env:
       REGISTRY: ghcr.io
       OWNER: sgsgermany
       IMAGE: debian
-      BASE_IMAGE: ${{ matrix.BASE_IMAGE }}
+      MILESTONE: ${{ matrix.MILESTONE }}
 
     steps:
       - name: Setup CI tools
diff --git a/branches.sh b/branches.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+# Debian GNU/Linux
+# @SGSGermany's base image for containers based on Debian GNU/Linux.
+#
+# Copyright (c) 2022  SGS Serious Gaming & Simulations GmbH
+#
+# This work is licensed under the terms of the MIT license.
+# For a copy, see LICENSE file or <https://opensource.org/licenses/MIT>.
+#
+# SPDX-License-Identifier: MIT
+# License-Filename: LICENSE
+
+set -eu -o pipefail
+export LC_ALL=C.UTF-8
+
+[ -v CI_TOOLS ] && [ "$CI_TOOLS" == "SGSGermany" ] \
+    || { echo "Invalid build environment: Environment variable 'CI_TOOLS' not set or invalid" >&2; exit 1; }
+
+[ -v CI_TOOLS_PATH ] && [ -d "$CI_TOOLS_PATH" ] \
+    || { echo "Invalid build environment: Environment variable 'CI_TOOLS_PATH' not set or invalid" >&2; exit 1; }
+
+[ -x "$(type -P skopeo 2>/dev/null)" ] \
+    || { echo "Missing script dependency: skopeo" >&2; exit 1; }
+
+[ -x "$(type -P jq 2>/dev/null)" ] \
+    || { echo "Missing script dependency: jq" >&2; exit 1; }
+
+source "$CI_TOOLS_PATH/helper/common.sh.inc"
+
+BUILD_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+source "$BUILD_DIR/container.env"
+
+EXIT_CODE=0
+
+echo + "BRANCHES_LOCAL=\"\$(printf '%s\n' \"\${MILESTONES[@]}\" | sort_semver)\"" >&2
+BRANCHES_LOCAL="$(printf '%s\n' "${MILESTONES[@]}" | sort_semver)"
+
+echo + "LATEST_BRANCH=\"\$(head -n 1 <<< \"\$BRANCHES_LOCAL\")\"" >&2
+LATEST_BRANCH="$(head -n 1 <<< "$BRANCHES_LOCAL")"
+
+echo + "BRANCHES_GLOBAL_JSON=\"\$(skopeo list-tags $(quote "docker://${BASE_IMAGE%:*}"))\"" >&2
+BRANCHES_GLOBAL_JSON="$(skopeo list-tags "docker://${BASE_IMAGE%:*}" || true)"
+
+if ! jq -e '.Tags[]' &> /dev/null <<< "$BRANCHES_GLOBAL_JSON"; then
+    echo "Unable to read image tags from container repository 'docker://${BASE_IMAGE%:*}'" >&2
+    exit 2
+fi
+
+echo + "BRANCHES_GLOBAL=\"\$(jq -re '.Tags[]|select(test(\"^[0-9]+$\"))' <<< \"\$BRANCHES_GLOBAL_JSON\" | sort_semver)\"" >&2
+BRANCHES_GLOBAL="$(jq -re '.Tags[]|select(test("^[0-9]+$"))' <<< "$BRANCHES_GLOBAL_JSON" | sort_semver)"
+
+echo + "BRANCHES_MISSING=\"\$(awk -v latest=$(quote "$LATEST_BRANCH") '\$0 == latest { found=1; next } found { print }'\" < <(tac <<< \"\$BRANCHES_GLOBAL\")" >&2
+BRANCHES_MISSING="$(awk -v latest="$LATEST_BRANCH" '$0 == latest { found=1; next } found { print }' < <(tac <<< "$BRANCHES_GLOBAL"))"
+
+if [ -n "$BRANCHES_MISSING" ]; then
+    echo "Explicit build instructions for the following Debian branches are missing" >&2
+    sed -e 's/^/- /' <<< "$BRANCHES_MISSING" >&2
+    EXIT_CODE=1
+fi
+
+echo + "echo \"\$BRANCHES_LOCAL\"" >&2
+[ -z "$BRANCHES_LOCAL" ] || echo "$BRANCHES_LOCAL"
+
+exit $EXIT_CODE
diff --git a/container.env b/container.env
@@ -1,6 +1,13 @@
+MILESTONE="${MILESTONE:-latest}"
+MILESTONES=(
+    13    # trixie
+    12    # bookworm
+    11    # bullseye
+)
+
 REGISTRY="${REGISTRY:-ghcr.io}"
 OWNER="${OWNER:-sgsgermany}"
 IMAGE="${IMAGE:-debian}"
 TAGS="${TAGS:-latest}"
 
-BASE_IMAGE="${BASE_IMAGE:-docker.io/debian:latest}"
+BASE_IMAGE="${BASE_IMAGE:-docker.io/debian:$MILESTONE}"
diff --git a/tags.sh b/tags.sh
@@ -19,6 +19,15 @@ export LC_ALL=C.UTF-8
 [ -v CI_TOOLS_PATH ] && [ -d "$CI_TOOLS_PATH" ] \
     || { echo "Invalid build environment: Environment variable 'CI_TOOLS_PATH' not set or invalid" >&2; exit 1; }
 
+[ -x "$(type -P podman 2>/dev/null)" ] \
+    || { echo "Missing script dependency: podman" >&2; exit 1; }
+
+[ -x "$(type -P skopeo 2>/dev/null)" ] \
+    || { echo "Missing script dependency: skopeo" >&2; exit 1; }
+
+[ -x "$(type -P jq 2>/dev/null)" ] \
+    || { echo "Missing script dependency: jq" >&2; exit 1; }
+
 source "$CI_TOOLS_PATH/helper/common.sh.inc"
 
 BUILD_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
@@ -65,14 +74,14 @@ fi
 ls_versions() {
     jq -re --arg "VERSION" "$1" \
         '.Tags[]|select(test("^[0-9]+\\.[0-9]+$") and startswith($VERSION + "."))' \
-        <<<"$BASE_IMAGE_REPO_TAGS" | sort_semver
+        <<< "$BASE_IMAGE_REPO_TAGS" | sort_semver
 }
 
 echo + "BASE_IMAGE_REPO_TAGS=\"\$(skopeo list-tags $(quote "docker://${BASE_IMAGE%:*}"))\"" >&2
 BASE_IMAGE_REPO_TAGS="$(skopeo list-tags "docker://${BASE_IMAGE%:*}" || true)"
 
-if [ -z "$BASE_IMAGE_REPO_TAGS" ]; then
-    echo "Unable to read tags from container repository 'docker://${BASE_IMAGE%:*}'" >&2
+if ! jq -e '.Tags[]' &> /dev/null <<< "$BASE_IMAGE_REPO_TAGS"; then
+    echo "Unable to read image tags from container repository 'docker://${BASE_IMAGE%:*}'" >&2
     exit 1
 fi
 
