Container CI #361
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ============================================================================= | |
| # Container CI - World-class container testing after CI passes | |
| # ============================================================================= | |
| name: Container CI | |
| on: | |
| # Triggered after CI workflow completes successfully | |
| workflow_run: | |
| workflows: ["CI"] | |
| types: [completed] | |
| branches: [main, develop] | |
| # Manual trigger for testing specific containers | |
| workflow_dispatch: | |
| inputs: | |
| containers: | |
| description: 'Containers to test (comma-separated, or "all")' | |
| required: false | |
| default: "all" | |
| concurrency: | |
| group: container-ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_PREFIX: ${{ github.repository_owner }}/allsource | |
| jobs: | |
| # =========================================================================== | |
| # Check CI status and get change detection from CI workflow | |
| # =========================================================================== | |
| check-ci: | |
| name: Check CI Status | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_run' | |
| outputs: | |
| should_run: ${{ steps.check.outputs.should_run }} | |
| steps: | |
| - name: Check CI workflow conclusion | |
| id: check | |
| run: | | |
| if [ "${{ github.event.workflow_run.conclusion }}" != "success" ]; then | |
| echo "CI workflow failed with: ${{ github.event.workflow_run.conclusion }}" | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| # =========================================================================== | |
| # Determine which containers to build based on CI change detection | |
| # =========================================================================== | |
| changes: | |
| name: Detect Changes | |
| runs-on: ubuntu-latest | |
| needs: check-ci | |
| if: | | |
| always() && | |
| (github.event_name == 'workflow_dispatch' || needs.check-ci.outputs.should_run == 'true') | |
| outputs: | |
| core: ${{ steps.set-matrix.outputs.core }} | |
| query-service: ${{ steps.set-matrix.outputs.query-service }} | |
| control-plane: ${{ steps.set-matrix.outputs.control-plane }} | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set Matrix | |
| id: set-matrix | |
| run: | | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| if [[ "${{ github.event.inputs.containers }}" == "all" ]]; then | |
| echo 'matrix=["core","query-service","control-plane"]' >> "$GITHUB_OUTPUT" | |
| echo "core=true" >> "$GITHUB_OUTPUT" | |
| echo "query-service=true" >> "$GITHUB_OUTPUT" | |
| echo "control-plane=true" >> "$GITHUB_OUTPUT" | |
| else | |
| # Convert comma-separated to JSON array | |
| containers=$(echo '${{ github.event.inputs.containers }}' | tr ',' '\n' | jq -R . | jq -sc .) | |
| echo "matrix=$containers" >> "$GITHUB_OUTPUT" | |
| # Set individual outputs | |
| echo '${{ github.event.inputs.containers }}' | tr ',' '\n' | while read -r c; do | |
| echo "$c=true" >> "$GITHUB_OUTPUT" | |
| done | |
| fi | |
| else | |
| # Use paths-filter for workflow_run trigger to detect changes | |
| # This is more efficient than duplicating CI's detection | |
| matrix='[]' | |
| # Check for changes using git diff against base | |
| git fetch origin "${{ github.event.workflow_run.head_branch || 'main' }}" --depth=2 | |
| # Detect changes per service | |
| CORE_CHANGED="false" | |
| QS_CHANGED="false" | |
| CP_CHANGED="false" | |
| if git diff --name-only HEAD~1 | grep -q "apps/core/"; then | |
| CORE_CHANGED="true" | |
| matrix=$(echo "$matrix" | jq -c '. + ["core"]') | |
| fi | |
| if git diff --name-only HEAD~1 | grep -q "apps/query-service/"; then | |
| QS_CHANGED="true" | |
| matrix=$(echo "$matrix" | jq -c '. + ["query-service"]') | |
| fi | |
| if git diff --name-only HEAD~1 | grep -q "apps/control-plane/"; then | |
| CP_CHANGED="true" | |
| matrix=$(echo "$matrix" | jq -c '. + ["control-plane"]') | |
| fi | |
| # If no changes detected, skip container builds | |
| if [[ "$matrix" == "[]" ]]; then | |
| echo "No container changes detected" | |
| fi | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| echo "core=$CORE_CHANGED" >> "$GITHUB_OUTPUT" | |
| echo "query-service=$QS_CHANGED" >> "$GITHUB_OUTPUT" | |
| echo "control-plane=$CP_CHANGED" >> "$GITHUB_OUTPUT" | |
| fi | |
| # =========================================================================== | |
| # Stage 1: Build containers (no push) - validates Dockerfiles compile | |
| # Skip on main branch since Docker Publish will handle the build there | |
| # =========================================================================== | |
| build: | |
| name: Build ${{ matrix.container }} | |
| needs: changes | |
| runs-on: ubuntu-latest | |
| if: | | |
| needs.changes.outputs.matrix != '[]' && | |
| needs.changes.outputs.matrix != '' && | |
| github.event.workflow_run.head_branch != 'main' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| container: ${{ fromJson(needs.changes.outputs.matrix) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Set build context and dockerfile | |
| id: config | |
| run: | | |
| case "${{ matrix.container }}" in | |
| core) | |
| echo "context=." >> "$GITHUB_OUTPUT" | |
| echo "dockerfile=apps/core/Dockerfile" >> "$GITHUB_OUTPUT" | |
| echo "image=allsource-core" >> "$GITHUB_OUTPUT" | |
| echo "port=3900" >> "$GITHUB_OUTPUT" | |
| ;; | |
| query-service) | |
| echo "context=apps/query-service" >> "$GITHUB_OUTPUT" | |
| echo "dockerfile=apps/query-service/Dockerfile" >> "$GITHUB_OUTPUT" | |
| echo "image=allsource-query-service" >> "$GITHUB_OUTPUT" | |
| echo "port=4000" >> "$GITHUB_OUTPUT" | |
| ;; | |
| control-plane) | |
| echo "context=apps/control-plane" >> "$GITHUB_OUTPUT" | |
| echo "dockerfile=apps/control-plane/Dockerfile" >> "$GITHUB_OUTPUT" | |
| echo "image=allsource-control-plane" >> "$GITHUB_OUTPUT" | |
| echo "port=8080" >> "$GITHUB_OUTPUT" | |
| ;; | |
| esac | |
| - name: Build image (validation only) | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: ${{ steps.config.outputs.context }} | |
| file: ${{ steps.config.outputs.dockerfile }} | |
| push: false | |
| load: true | |
| tags: ${{ steps.config.outputs.image }}:test | |
| cache-from: type=gha,scope=${{ matrix.container }} | |
| cache-to: type=gha,mode=max,scope=${{ matrix.container }} | |
| build-args: | | |
| VERSION=${{ github.sha }} | |
| REVISION=${{ github.sha }} | |
| BUILDTIME=${{ github.event.workflow_run.head_commit.timestamp || github.event.head_commit.timestamp }} | |
| - name: Check image size | |
| run: | | |
| SIZE=$(docker images --format "{{.Size}}" ${{ steps.config.outputs.image }}:test) | |
| echo "### 📦 Image Size: $SIZE" >> "$GITHUB_STEP_SUMMARY" | |
| # Convert to MB for comparison | |
| SIZE_MB=$(docker images --format "{{.Size}}" ${{ steps.config.outputs.image }}:test | \ | |
| awk '{ | |
| if (index($0, "GB") > 0) { gsub("GB", "", $0); print $0 * 1024 } | |
| else if (index($0, "MB") > 0) { gsub("MB", "", $0); print $0 } | |
| else if (index($0, "KB") > 0) { gsub("KB", "", $0); print $0 / 1024 } | |
| else { print 0 } | |
| }') | |
| if (( $(echo "$SIZE_MB > 500" | bc -l) )); then | |
| echo "⚠️ Warning: Image size ($SIZE) exceeds 500MB threshold" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Export image for integration tests | |
| if: matrix.container == 'core' || matrix.container == 'query-service' | |
| run: | | |
| docker save ${{ steps.config.outputs.image }}:test | gzip > ${{ matrix.container }}-image.tar.gz | |
| - name: Upload image artifact | |
| if: matrix.container == 'core' || matrix.container == 'query-service' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ matrix.container }}-image | |
| path: ${{ matrix.container }}-image.tar.gz | |
| retention-days: 1 | |
| - name: Test container startup | |
| run: | | |
| # Set minimal env vars for services that need them | |
| ENV_ARGS="" | |
| if [ "${{ matrix.container }}" == "query-service" ]; then | |
| # Query service needs these to start (but can run without database) | |
| ENV_ARGS="-e SECRET_KEY_BASE=$(openssl rand -hex 64)" | |
| fi | |
| # Start container | |
| CONTAINER_ID=$(docker run -d $ENV_ARGS -p ${{ steps.config.outputs.port }}:${{ steps.config.outputs.port }} \ | |
| ${{ steps.config.outputs.image }}:test) | |
| # Wait for container to be ready | |
| MAX_WAIT=60 | |
| WAITED=0 | |
| while [ $WAITED -lt $MAX_WAIT ]; do | |
| STATUS=$(docker inspect --format='{{.State.Status}}' $CONTAINER_ID) | |
| if [ "$STATUS" = "exited" ]; then | |
| EXIT_CODE=$(docker inspect --format='{{.State.ExitCode}}' $CONTAINER_ID) | |
| echo "❌ Container exited with code $EXIT_CODE" >> "$GITHUB_STEP_SUMMARY" | |
| docker logs $CONTAINER_ID | |
| docker rm -f $CONTAINER_ID | |
| exit 1 | |
| fi | |
| # Check if container has healthcheck | |
| HEALTH=$(docker inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' $CONTAINER_ID) | |
| if [ "$HEALTH" = "healthy" ] || [ "$HEALTH" = "no-healthcheck" ]; then | |
| if [ "$STATUS" = "running" ]; then | |
| echo "✅ Container started successfully in ${WAITED}s" >> "$GITHUB_STEP_SUMMARY" | |
| docker rm -f $CONTAINER_ID | |
| exit 0 | |
| fi | |
| fi | |
| sleep 1 | |
| WAITED=$((WAITED + 1)) | |
| done | |
| echo "❌ Container failed to start within ${MAX_WAIT}s" >> "$GITHUB_STEP_SUMMARY" | |
| docker logs $CONTAINER_ID | |
| docker rm -f $CONTAINER_ID | |
| exit 1 | |
| # =========================================================================== | |
| # Docker Compose Integration Test | |
| # Skip on main branch since Docker Publish will handle the full validation there | |
| # =========================================================================== | |
| integration: | |
| name: Integration Test | |
| needs: [changes, build] | |
| runs-on: ubuntu-latest | |
| if: | | |
| always() && | |
| github.event.workflow_run.head_branch != 'main' && | |
| needs.build.result == 'success' && | |
| needs.changes.outputs.matrix != '[]' && | |
| needs.changes.outputs.matrix != '' && | |
| (contains(fromJson(needs.changes.outputs.matrix), 'core') || | |
| contains(fromJson(needs.changes.outputs.matrix), 'query-service')) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download core image | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: core-image | |
| path: /tmp/images | |
| - name: Download query-service image | |
| if: contains(fromJson(needs.changes.outputs.matrix), 'query-service') | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: query-service-image | |
| path: /tmp/images | |
| - name: Load images from artifacts | |
| run: | | |
| # Load core image and tag for docker-compose | |
| gunzip -c /tmp/images/core-image.tar.gz | docker load | |
| docker tag allsource-core:test ghcr.io/all-source-os/allsource-core:latest | |
| # Load query-service if it was built | |
| if [ -f /tmp/images/query-service-image.tar.gz ]; then | |
| gunzip -c /tmp/images/query-service-image.tar.gz | docker load | |
| docker tag allsource-query-service:test ghcr.io/all-source-os/allsource-query-service:latest | |
| fi | |
| echo "### 📦 Loaded images from build job:" >> "$GITHUB_STEP_SUMMARY" | |
| docker images | grep -E "allsource-(core|query-service)" >> "$GITHUB_STEP_SUMMARY" || true | |
| - name: Start services | |
| run: | | |
| docker compose up -d core | |
| # Wait for core to be healthy | |
| timeout 60 bash -c 'until docker compose exec -T core wget --spider -q http://localhost:3900/health; do sleep 2; done' | |
| echo "✅ Core service is healthy" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Run Core integration tests | |
| run: | | |
| echo "### Core Integration Tests" >> "$GITHUB_STEP_SUMMARY" | |
| # Basic health check | |
| curl -sf http://localhost:3900/health || exit 1 | |
| echo "✅ Core health check passed" >> "$GITHUB_STEP_SUMMARY" | |
| # Test event creation | |
| RESPONSE=$(curl -sf -X POST http://localhost:3900/api/v1/events \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"stream_id":"test-stream","event_type":"TestEvent","data":{"message":"hello"}}') | |
| if echo "$RESPONSE" | jq -e '.event_id' > /dev/null; then | |
| echo "✅ Event creation test passed" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "❌ Event creation test failed" >> "$GITHUB_STEP_SUMMARY" | |
| exit 1 | |
| fi | |
| # Test WebSocket endpoint is accessible | |
| WS_RESPONSE=$(curl -sf -i \ | |
| -H "Connection: Upgrade" \ | |
| -H "Upgrade: websocket" \ | |
| -H "Sec-WebSocket-Version: 13" \ | |
| -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" \ | |
| http://localhost:3900/api/v1/events/stream 2>&1 || true) | |
| if echo "$WS_RESPONSE" | grep -q "101\|Upgrade"; then | |
| echo "✅ WebSocket endpoint accessible" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "⚠️ WebSocket endpoint check inconclusive" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Start Query Service for WebSocket tests | |
| if: contains(fromJson(needs.changes.outputs.matrix), 'query-service') | |
| run: | | |
| # Start postgres first | |
| docker compose up -d postgres | |
| timeout 30 bash -c 'until docker compose exec -T postgres pg_isready -U postgres; do sleep 1; done' | |
| # Start query-service with WebSocket enabled | |
| docker compose up -d query-service | |
| # Wait for query-service to be ready | |
| timeout 60 bash -c 'until curl -sf http://localhost:3902/api/health/live; do sleep 2; done' | |
| echo "✅ Query Service started" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Run WebSocket integration tests | |
| if: contains(fromJson(needs.changes.outputs.matrix), 'query-service') | |
| run: | | |
| echo "### WebSocket Integration Tests" >> "$GITHUB_STEP_SUMMARY" | |
| # Test 1: Health endpoint includes WebSocket status | |
| HEALTH_RESPONSE=$(curl -sf http://localhost:3902/api/health) | |
| echo "Health response: $HEALTH_RESPONSE" | |
| if echo "$HEALTH_RESPONSE" | jq -e '.checks.websocket' > /dev/null; then | |
| WS_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.checks.websocket') | |
| echo "✅ Health check includes WebSocket status: $WS_STATUS" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "❌ Health check missing WebSocket status" >> "$GITHUB_STEP_SUMMARY" | |
| exit 1 | |
| fi | |
| # Test 2: Verify health shows all required checks | |
| if echo "$HEALTH_RESPONSE" | jq -e '.checks.database and .checks.backend and .checks.websocket' > /dev/null; then | |
| echo "✅ Health check includes all required components" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "❌ Health check missing required components" >> "$GITHUB_STEP_SUMMARY" | |
| exit 1 | |
| fi | |
| # Test 3: Real-time event propagation (simplified) | |
| # Create an event and verify Query Service can fetch it | |
| EVENT_ID=$(curl -sf -X POST http://localhost:3900/api/v1/events \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"stream_id":"ws-test-stream","event_type":"WebSocketTest","entity_id":"ws-test-entity","data":{"ws_test":true}}' \ | |
| | jq -r '.event_id') | |
| if [ -n "$EVENT_ID" ] && [ "$EVENT_ID" != "null" ]; then | |
| echo "✅ Test event created: $EVENT_ID" >> "$GITHUB_STEP_SUMMARY" | |
| # Give WebSocket time to propagate | |
| sleep 2 | |
| # Verify event exists (via Core API since Query Service may not have HTTP query) | |
| QUERY_RESPONSE=$(curl -sf "http://localhost:3900/api/v1/events/query?entity_id=ws-test-entity" || echo "{}") | |
| if echo "$QUERY_RESPONSE" | jq -e '.events | length > 0' > /dev/null 2>&1; then | |
| echo "✅ Event propagation verified" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "⚠️ Event query returned no results (may be expected)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| else | |
| echo "❌ Failed to create test event" >> "$GITHUB_STEP_SUMMARY" | |
| exit 1 | |
| fi | |
| # Test 4: Service degradation behavior | |
| # Stop Core and verify Query Service enters degraded mode | |
| echo "Testing graceful degradation..." | |
| docker compose stop core | |
| sleep 3 | |
| DEGRADED_HEALTH=$(curl -sf http://localhost:3902/api/health || echo '{"status":"error"}') | |
| DEGRADED_STATUS=$(echo "$DEGRADED_HEALTH" | jq -r '.status') | |
| if [ "$DEGRADED_STATUS" = "degraded" ] || [ "$DEGRADED_STATUS" = "unhealthy" ]; then | |
| echo "✅ Service correctly reports degraded/unhealthy when Core is down" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "⚠️ Service status: $DEGRADED_STATUS (expected degraded/unhealthy)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| # Liveness should still pass even when degraded | |
| LIVE_RESPONSE=$(curl -sf http://localhost:3902/api/health/live) | |
| if echo "$LIVE_RESPONSE" | jq -e '.status == "alive"' > /dev/null; then | |
| echo "✅ Liveness probe passes when backend unavailable" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "❌ Liveness probe failed unexpectedly" >> "$GITHUB_STEP_SUMMARY" | |
| exit 1 | |
| fi | |
| # Restart Core for cleanup | |
| docker compose start core | |
| timeout 30 bash -c 'until curl -sf http://localhost:3900/health; do sleep 1; done' | |
| echo "✅ Core restarted successfully" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Cleanup | |
| if: always() | |
| run: docker compose down -v | |
| # =========================================================================== | |
| # Summary Report | |
| # =========================================================================== | |
| summary: | |
| name: Test Summary | |
| needs: [changes, build, integration] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Generate Summary | |
| run: | | |
| echo "# 🐳 Container CI Summary" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| # Check if we're on main branch (skipping for Docker Publish) | |
| if [ "${{ github.event.workflow_run.head_branch }}" == "main" ]; then | |
| echo "**Main branch detected - container validation deferred to Docker Publish workflow**" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "This avoids redundant builds since Docker Publish will build and push the same images." >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| # Check if there were any containers to build | |
| if [ "${{ needs.changes.outputs.matrix }}" == "[]" ] || [ -z "${{ needs.changes.outputs.matrix }}" ]; then | |
| echo "**No container changes detected - skipped container builds**" >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| echo "| Container | Build | Startup |" >> "$GITHUB_STEP_SUMMARY" | |
| echo "|-----------|-------|---------|" >> "$GITHUB_STEP_SUMMARY" | |
| # This is a simplified summary - actual results come from build job | |
| if [ "${{ needs.build.result }}" == "success" ]; then | |
| echo "| All | ✅ | ✅ |" >> "$GITHUB_STEP_SUMMARY" | |
| elif [ "${{ needs.build.result }}" == "skipped" ]; then | |
| echo "| All | ⏭️ | ⏭️ |" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "| All | ❌ | - |" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| if [ "${{ needs.integration.result }}" == "success" ]; then | |
| echo "**Integration Tests:** ✅ Passed" >> "$GITHUB_STEP_SUMMARY" | |
| elif [ "${{ needs.integration.result }}" == "skipped" ]; then | |
| echo "**Integration Tests:** ⏭️ Skipped" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "**Integration Tests:** ❌ Failed" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Check overall status | |
| if: needs.build.result == 'failure' | |
| run: exit 1 |