Updates the build/release workflows

danschmidt5189 · danschmidt5189 · commit c94140836e11 · 2025-10-06T23:09:50.000-07:00
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -6,18 +6,30 @@ on:
       - '**'
   workflow_dispatch:
 
+env:
+  DOCKER_METADATA_SET_OUTPUT_ENV: 'true'
+
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     outputs:
-      build-image: ${{ steps.build-meta.outputs.tags }}
+      image-app-arm64: ${{ steps.gen-output.outputs.image-app-arm64 }}
+      image-app-x64: ${{ steps.gen-output.outputs.image-app-x64 }}
+      image-dev-arm64: ${{ steps.gen-output.outputs.image-development-arm64 }}
+      image-dev-x64: ${{ steps.gen-output.outputs.image-development-x64 }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner:
+          - ubuntu-24.04
+          - ubuntu-24.04-arm
+        target:
+          - app
+          - development
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -28,48 +40,58 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Produce the build image tag
-        id: build-meta
+      - name: Docker meta
+        id: meta
         uses: docker/metadata-action@v5
         with:
           images: ghcr.io/${{ github.repository }}
-          tags: type=sha,suffix=-build-${{ github.run_id }}_${{ github.run_attempt }}
+          # note Specifies a single tag to ensure the default doesn't add more than one.
+          #      The actual tag is not used, this is just used to sanitize the registry name
+          #      and produce labels.
+          tags: type=sha
+
+      - name: Sanitize registry repository name
+        id: get-reg
+        run: |
+          echo "registry=$(echo '${{ steps.meta.outputs.tags }}' | cut -f1 -d:)" | tee -a "$GITHUB_OUTPUT"
 
-      - name: Build and push the untested image
+      - name: Build/push the arch-specific image
+        id: build
         uses: docker/build-push-action@v6
         with:
-          push: true
-          labels: ${{ steps.build-meta.outputs.labels }}
-          tags: ${{ steps.build-meta.outputs.tags }}
-          provenance: true
-          sbom: true
-          target: development
+          # @todo GHA caching needs tuning, these tend not to hit. Perhaps switch to type=registry?
           cache-from: type=gha
-          cache-to: type=gha
+          cache-to: type=gha,mode=max
+          labels: ${{ steps.meta.outputs.labels }}
+          provenance: mode=max
+          sbom: true
+          tags: ${{ steps.get-reg.outputs.registry }}
+          outputs: type=image,push-by-digest=true,push=true
+          target: ${{ matrix.target }}
+
+      - name: Write arch-specific image digest to outputs
+        id: gen-output
+        run: |
+          echo "image-${{ matrix.target }}-${RUNNER_ARCH,,}=${{ steps.get-reg.outputs.registry }}@${{ steps.build.outputs.digest }}" | tee -a "$GITHUB_OUTPUT"
 
-  test:
-    runs-on: ubuntu-latest
+  merge:
+    runs-on: ubuntu-24.04
     needs:
       - build
-    strategy:
-      fail-fast: false
-      matrix:
-        test:
-          - mypy .
-          - pydoclint .
-          # pylint returns error codes if the checks fail
-          # https://pylint.readthedocs.io/en/latest/user_guide/usage/run.html#exit-codes
-          - pylint -v .
-          - python -m unittest -v
     env:
-      COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml
-      DOCKER_APP_IMAGE: ${{ needs.build.outputs.build-image }}
+      DOCKER_APP_IMAGE_ARM64: ${{ needs.build.outputs.image-app-arm64 }}
+      DOCKER_APP_IMAGE_X64: ${{ needs.build.outputs.image-app-x64 }}
+      DOCKER_DEV_IMAGE_ARM64: ${{ needs.build.outputs.image-dev-arm64 }}
+      DOCKER_DEV_IMAGE_X64: ${{ needs.build.outputs.image-dev-x64 }}
+    outputs:
+      app-image: ${{ steps.app-meta.outputs.tags }}
+      dev-image: ${{ steps.dev-meta.outputs.tags }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up Docker Compose
-        uses: docker/setup-compose-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
@@ -78,35 +100,120 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Copy environment file
-        run: cp env.example .env
-
-      - name: Set ARTIFACTS_DIR
-        run: echo "ARTIFACTS_DIR=${RUNNER_TEMP}/artifacts" >> $GITHUB_ENV
+      - name: Generate tag for the dev image
+        id: dev-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=sha,suffix=-build-${{ github.run_id }}_${{ github.run_attempt }}-dev
 
-      - name: Create the artifacts directory
-        run: mkdir -p "$ARTIFACTS_DIR"
+      - name: Push the multi-platform dev image
+        run: |
+          docker buildx imagetools create \
+            --tag "$DOCKER_METADATA_OUTPUT_TAGS" \
+            "$DOCKER_DEV_IMAGE_ARM64" "$DOCKER_DEV_IMAGE_X64"
 
-      - name: Record start time
-        run: TEST_START=`date +%s` >> $GITHUB_ENV
+      - name: Generate tag for the app image
+        id: app-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=sha,suffix=-build-${{ github.run_id }}_${{ github.run_attempt }}
 
-      - name: Run the test command
+      - name: Push the multi-platform app image
+        run: |
+          docker buildx imagetools create \
+            --tag "$DOCKER_METADATA_OUTPUT_TAGS" \
+            "$DOCKER_APP_IMAGE_ARM64" "$DOCKER_APP_IMAGE_X64"
+
+  test-mypy:
+    runs-on: ubuntu-24.04
+    needs: merge
+    container:
+      image: ${{ needs.merge.outputs.dev-image }}
+    defaults:
+      run:
+        working-directory: /app
+    steps:
+      - name: Run mypy
+        run: mypy .
+
+  test-pydoclint:
+    runs-on: ubuntu-24.04
+    needs: merge
+    container:
+      image: ${{ needs.merge.outputs.dev-image }}
+    defaults:
+      run:
+        working-directory: /app
+    steps:
+      - name: Run pydoclint
+        run: pydoclint .
+
+  # pylint returns error codes if the checks fail
+  # @see https://pylint.readthedocs.io/en/latest/user_guide/usage/run.html#exit-codes
+  test-pylint:
+    runs-on: ubuntu-24.04
+    needs: merge
+    container:
+      image: ${{ needs.merge.outputs.dev-image }}
+    defaults:
+      run:
+        working-directory: /app
+    steps:
+      - name: Run pylint
+        run: pylint -v .
+
+  test-unittest:
+    runs-on: ubuntu-24.04
+    needs: merge
+    container:
+      image: ${{ needs.merge.outputs.dev-image }}
+    defaults:
+      run:
+        working-directory: /app
+    steps:
+      - name: Run unit tests
         run: |
-          docker compose run --no-deps --rm app ${{ matrix.test }}
+          python -m unittest .
+
+  test-startup:
+    runs-on: ubuntu-24.04
+    needs: merge
+    env:
+      COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml
+      DOCKER_APP_IMAGE: ${{ needs.merge.outputs.app-image }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@v1
 
-      - name: Upload test report
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v4
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
         with:
-          name: Test Report - ${{ matrix.test }} (${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }})
-          path: ${{ env.ARTIFACTS_DIR }}
-          if-no-files-found: warn
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run the test script
+        run: |
+          docker compose up --detach --wait
 
   push:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs:
-      - build
-      - test
+      - merge
+      - test-mypy
+      - test-pydoclint
+      - test-pylint
+      - test-startup
+      - test-unittest
+    env:
+      DOCKER_APP_IMAGE: ${{ needs.merge.outputs.app-image }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -129,10 +236,7 @@ jobs:
             type=raw,value=latest,enable={{is_default_branch}}
 
       - name: Retag and push the image
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          labels: ${{ steps.branch-meta.outputs.labels }}
-          tags: ${{ steps.branch-meta.outputs.tags }}
-          cache-from: type=registry,ref=${{ needs.build.outputs.build-image }}
-          target: app
+        run: |
+          docker pull "$DOCKER_APP_IMAGE"
+          echo "$DOCKER_METADATA_OUTPUT_TAGS" | tr ' ' '\n' | xargs -n1 docker tag "$DOCKER_APP_IMAGE"
+          docker push --all-tags "$(echo "$DOCKER_APP_IMAGE" | cut -f1 -d:)"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,62 @@
+name: Push Release Tags
+
+on:
+  push:
+    tags:
+      - '**'
+  workflow_dispatch:
+
+env:
+  DOCKER_METADATA_SET_OUTPUT_ENV: 'true'
+
+jobs:
+  retag:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Determine the sha-based image tag to retag
+        id: get-base-image
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: type=sha
+
+      - name: Verify that the image was previously built
+        env:
+          BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }}
+        run: |
+          docker pull "$BASE_IMAGE"
+
+      - name: Produce release tags
+        id: tag-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          flavor: latest=false
+          tags: |
+            type=ref,event=tag
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{version}}
+
+      - name: Retag the pulled image
+        env:
+          BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }}
+        run: |
+          echo "$DOCKER_METADATA_OUTPUT_TAGS" | tr ' ' '\n' | xargs -n1 docker tag "$BASE_IMAGE"
+          docker push --all-tags "$(echo "$BASE_IMAGE" | cut -f1 -d:)"
diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
@@ -1,14 +1,16 @@
 services:
-  db: !reset
+  db:
+    volumes: !reset
+
   app:
     build: !reset
     depends_on: !reset
     ports: !reset
-    environment:
-      CHAT_BACKEND: ollama
-      DEFAULT_STORAGE_DIR: /tmp
-      EMBED_BACKEND: ollama
+    env_file: !override env.example
     volumes: !reset
+
   ollama: !reset
+
   prisma: !reset
+
   localstack: !reset
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,15 +4,15 @@ services:
     restart: always
     environment: &dbconfig
       POSTGRES_DB: ${POSTGRES_DB:-willa}
-      POSTGRES_USER: ${POSTGRES_USER:-root}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-root}
-      DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB}}
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+      DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db/${POSTGRES_DB:-willa}}
     ports:
       - 5432:5432
     volumes:
       - ./.data/postgres:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-willa}"]
       interval: 10s
       timeout: 5s
       retries: 5
