mvp-0.0.1 #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ main, develop, "feature/*", "hotfix/*" ] | |
| pull_request: | |
| branches: [ main, develop ] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| JAVA_OPTS: -Xmx3G -XX:+UseG1GC -XX:MaxGCPauseMillis=100 | |
| SBT_OPTS: -Xmx3G -XX:+UseG1GC | |
| jobs: | |
| changes: | |
| name: Detect Changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| core: ${{ steps.changes.outputs.core }} | |
| connectors: ${{ steps.changes.outputs.connectors }} | |
| engines: ${{ steps.changes.outputs.engines }} | |
| quality: ${{ steps.changes.outputs.quality }} | |
| examples: ${{ steps.changes.outputs.examples }} | |
| docs: ${{ steps.changes.outputs.docs }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v2 | |
| id: changes | |
| with: | |
| filters: | | |
| core: | |
| - 'modules/core/**' | |
| - 'modules/safety/**' | |
| - 'build.sbt' | |
| - 'project/**' | |
| connectors: | |
| - 'modules/connectors*/**' | |
| engines: | |
| - 'modules/engines*/**' | |
| quality: | |
| - 'modules/quality*/**' | |
| examples: | |
| - 'examples/**' | |
| docs: | |
| - 'docs/**' | |
| - 'flowforge-docs/**' | |
| - '*.md' | |
| code-quality: | |
| name: Code Quality | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Cache SBT | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.coursier/cache | |
| target | |
| project/target | |
| project/project/target | |
| key: sbt-cache-${{ runner.os }}-${{ hashFiles('**/*.sbt', 'project/build.properties', 'project/plugins.sbt') }} | |
| restore-keys: | | |
| sbt-cache-${{ runner.os }}- | |
| - name: Compile | |
| run: sbt compile Test/compile | |
| - name: Check formatting | |
| run: sbt scalafmtCheckAll | |
| - name: Check scalafix | |
| run: sbt "scalafixAll --check" | |
| - name: Check unused dependencies | |
| run: sbt unusedCompileDependenciesTest | |
| - name: Wartremover checks | |
| run: sbt wartremoverErrors | |
| test: | |
| name: Test | |
| needs: [changes, code-quality] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| java: ['11', '17', '21'] | |
| scala: ['2.13.12', '3.3.1'] | |
| exclude: | |
| # Reduce matrix size - only test all combos on ubuntu with Java 17 | |
| - os: windows-latest | |
| java: '11' | |
| - os: windows-latest | |
| java: '21' | |
| - os: macos-latest | |
| java: '11' | |
| - os: macos-latest | |
| java: '21' | |
| - os: windows-latest | |
| scala: '3.3.1' | |
| - os: macos-latest | |
| scala: '3.3.1' | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| cache: 'sbt' | |
| - name: Cache SBT | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.coursier/cache | |
| target | |
| project/target | |
| key: sbt-cache-${{ runner.os }}-${{ matrix.java }}-${{ matrix.scala }}-${{ hashFiles('**/*.sbt') }} | |
| restore-keys: | | |
| sbt-cache-${{ runner.os }}-${{ matrix.java }}-${{ matrix.scala }}- | |
| sbt-cache-${{ runner.os }}-${{ matrix.java }}- | |
| sbt-cache-${{ runner.os }}- | |
| - name: Run tests | |
| run: sbt ++${{ matrix.scala }} test | |
| env: | |
| JAVA_OPTS: ${{ env.JAVA_OPTS }} | |
| - name: Upload test results | |
| uses: dorny/test-reporter@v1 | |
| if: success() || failure() | |
| with: | |
| name: Test Results (${{ matrix.os }}, Java ${{ matrix.java }}, Scala ${{ matrix.scala }}) | |
| path: '**/target/test-reports/TEST-*.xml' | |
| reporter: java-junit | |
| integration-test: | |
| name: Integration Tests | |
| needs: [changes, code-quality] | |
| if: needs.changes.outputs.core == 'true' || needs.changes.outputs.connectors == 'true' || needs.changes.outputs.engines == 'true' | |
| runs-on: ubuntu-latest | |
| services: | |
| postgres: | |
| image: postgres:15 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: flowforge_test | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| redis: | |
| image: redis:7-alpine | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Cache SBT | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.coursier/cache | |
| target | |
| project/target | |
| key: sbt-it-cache-${{ runner.os }}-${{ hashFiles('**/*.sbt') }} | |
| - name: Start test containers | |
| run: | | |
| docker-compose -f docker/docker-compose.test.yml up -d | |
| sleep 30 | |
| - name: Run integration tests | |
| run: sbt it:test | |
| env: | |
| POSTGRES_URL: jdbc:postgresql://localhost:5432/flowforge_test | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| REDIS_URL: redis://localhost:6379 | |
| - name: Stop test containers | |
| if: always() | |
| run: docker-compose -f docker/docker-compose.test.yml down | |
| coverage: | |
| name: Coverage | |
| needs: [test] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Cache SBT | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.coursier/cache | |
| target | |
| project/target | |
| key: sbt-coverage-cache-${{ runner.os }}-${{ hashFiles('**/*.sbt') }} | |
| - name: Run tests with coverage | |
| run: sbt coverage test coverageReport | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| files: ./target/scala-*/scoverage-report/scoverage.xml | |
| fail_ci_if_error: false | |
| - name: Upload coverage to Codacy | |
| uses: codacy/codacy-coverage-reporter-action@v1 | |
| with: | |
| project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} | |
| coverage-reports: target/scala-*/scoverage-report/scoverage.xml | |
| property-based-test: | |
| name: Property-Based Tests | |
| needs: [changes, code-quality] | |
| if: needs.changes.outputs.core == 'true' || needs.changes.outputs.quality == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Run property-based tests | |
| run: sbt "testOnly -- -l \"org.scalatest.tags.Slow\" -n \"PropertyBasedTest\"" | |
| timeout-minutes: 30 | |
| benchmark: | |
| name: Performance Benchmarks | |
| needs: [changes, code-quality] | |
| if: needs.changes.outputs.core == 'true' || needs.changes.outputs.engines == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Run benchmarks | |
| run: sbt "benchmarks/Jmh/run -i 3 -wi 3 -f 1 -t 1" | |
| timeout-minutes: 60 | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: benchmark-results | |
| path: benchmarks/target/jmh-result.json | |
| security-scan: | |
| name: Security Scan | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Dependency vulnerability scan | |
| run: sbt dependencyCheck | |
| - name: Upload dependency check reports | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: dependency-check-report | |
| path: target/scala-*/dependency-check-report.html | |
| - name: SAST scan with Semgrep | |
| uses: returntocorp/semgrep-action@v1 | |
| with: | |
| config: >- | |
| p/security-audit | |
| p/secrets | |
| p/scala | |
| docs: | |
| name: Documentation | |
| needs: [changes] | |
| if: needs.changes.outputs.docs == 'true' || needs.changes.outputs.core == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| cache: 'npm' | |
| cache-dependency-path: flowforge-docs/website/package-lock.json | |
| - name: Generate API documentation | |
| run: sbt unidoc | |
| - name: Build documentation site | |
| run: | | |
| cd flowforge-docs/website | |
| npm ci | |
| npm run build | |
| - name: Upload documentation artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: documentation | |
| path: | | |
| target/scala-*/unidoc/ | |
| flowforge-docs/website/build/ | |
| validate-examples: | |
| name: Validate Examples | |
| needs: [changes, test] | |
| if: needs.changes.outputs.examples == 'true' || needs.changes.outputs.core == 'true' | |
| strategy: | |
| matrix: | |
| example: [customer-analytics, fraud-detection, ml-pipeline, data-migration] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Compile example | |
| run: sbt "examples/Test/compile" | |
| - name: Run example tests | |
| run: sbt "examples/testOnly *${{ matrix.example }}*" | |
| build-docker: | |
| name: Build Docker Image | |
| needs: [test, integration-test] | |
| if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'sbt' | |
| - name: Build application | |
| run: sbt universal:packageBin | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| push: true | |
| tags: | | |
| flowforge/flowforge:latest | |
| flowforge/flowforge:${{ github.sha }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| notification: | |
| name: Notification | |
| needs: [test, integration-test, coverage, security-scan, docs] | |
| if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Notify success | |
| if: needs.test.result == 'success' && needs.integration-test.result == 'success' | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: success | |
| text: "✅ FlowForge CI passed successfully!" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| - name: Notify failure | |
| if: needs.test.result == 'failure' || needs.integration-test.result == 'failure' | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: failure | |
| text: "❌ FlowForge CI failed!" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} |