ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  schedule:
    - cron: "0 2 * * *"
  workflow_dispatch:
    inputs:
      run_bench_smoke:
        description: "Run benchmark smoke (1 episode per task)"
        required: false
        default: false
        type: boolean
      run_coordination_smoke:
        description: "Run coordination smoke (validate-policy, coordination tests, coord_scale/coord_risk)"
        required: false
        default: false
        type: boolean
      run_external_reviewer_checks:
        description: "Run external reviewer checks (coordination study + summary + COORDINATION_LLM_CARD)"
        required: false
        default: false
        type: boolean

env:
  LABTRUST_BENCH_SMOKE: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_bench_smoke == 'true')) && '1' || '0' }}
  LABTRUST_COORDINATION_SMOKE: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_coordination_smoke == 'true')) && '1' || '0' }}
  LABTRUST_EXTERNAL_REVIEWER_CHECKS: ${{ (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_external_reviewer_checks == 'true')) && '1' || '0' }}

jobs:
  lint-format:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install ruff
        run: pip install ruff
      - name: Ruff format check
        run: ruff format --check .
      - name: Ruff check
        run: ruff check .

  typecheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install deps
        run: pip install -e '.[dev]'
      - name: Mypy
        run: mypy src/

  test:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
        python-version: ["3.11", "3.12"]
    defaults:
      run:
        # Use bash on all runners so pip/pytest commands and brackets in '.[dev,env]' work (Windows defaults to PowerShell).
        shell: bash
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install package (dev + env so env-dependent tests run)
        run: pip install -e '.[dev,env]'
      - name: Pytest
        run: python -m pytest -q -m "not slow"
      - name: Metamorphic properties (smoke)
        run: python -m pytest tests/test_metamorphic_properties.py -q
      - name: Install env deps and run security + CLI smoke
        run: |
          pip install -e '.[dev,env]'
          python -m pytest tests/test_security_property_fuzz.py tests/test_prompt_injection_defense.py tests/test_llm_prompt_injection_golden.py -q
          python -m pytest tests/test_cli_smoke_matrix.py -q -k "not train_ppo and not eval_ppo" -m "not slow"
      - name: Run security suite (smoke, includes coord_pack_ref when env available)
        run: python -m labtrust_gym.cli.main run-security-suite --out ./security_suite_out --smoke --seed 42
      - name: Extension example (install plugin and run example_task)
        run: |
          pip install -e examples/extension_example
          python -m labtrust_gym.cli.main --profile example run-benchmark --task example_task --episodes 1 --out ./extension_example_out.json

  policy-validate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package (dev + env for policy/schema code that may touch env)
        run: pip install -e '.[dev,env]'
      - name: Validate policy
        run: labtrust validate-policy
      - name: Validate policy (partner overlay hsl_like)
        run: labtrust validate-policy --partner hsl_like

  no-placeholders:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: No placeholders / stubs gate
        run: python tools/no_placeholders.py

  risk-register-gate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package (dev + env so export/lab_design and pytest collection succeed)
        run: pip install -e '.[dev,env]'
      - name: Verify ui_fixtures evidence bundle
        run: labtrust verify-bundle --bundle tests/fixtures/ui_fixtures/evidence_bundle/EvidenceBundle.v0.1
      - name: Export bundle from ui_fixtures (for artifact inspection; tests build bundle in memory)
        run: labtrust export-risk-register --out ./risk_register_out --runs tests/fixtures/ui_fixtures
      - name: Risk register contract gate (schema, snapshot, crosswalk, coverage)
        run: pytest tests/test_risk_register_contract_gate.py -v

  risk-coverage-every-pr:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package (dev + env for export-risk-register)
        run: pip install -e '.[dev,env]'
      # Plan completeness checked on every PR so required_bench_plan is runnable.
      - name: Required bench plan completeness
        run: python scripts/required_bench_plan_runs.py > /dev/null
      - name: Export risk register from fixtures (ui + coord pack)
        run: labtrust export-risk-register --out ./risk_register_out $(python scripts/risk_coverage_fixture_dirs.py)
      - name: Validate coverage (strict)
        run: labtrust validate-coverage --bundle ./risk_register_out/RISK_REGISTER_BUNDLE.v0.1.json --strict

  risk-coverage-strict:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package
        run: pip install -e '.[dev]'
      - name: Generate bundle from fixtures (ui + coord pack)
        run: labtrust export-risk-register --out ./risk_register_out $(python scripts/risk_coverage_fixture_dirs.py)
      - name: Validate coverage (strict)
        run: labtrust validate-coverage --bundle ./risk_register_out/RISK_REGISTER_BUNDLE.v0.1.json --strict

  coordination-interface-contract:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Coordination interface contract (all methods instantiate and run 5 steps coord_scale)
        run: pytest tests/test_coordination_interface_contract.py -v

  bench-smoke:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_bench_smoke == 'true')
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Benchmark smoke (1 episode per task)
        run: labtrust bench-smoke --seed 42

  baseline-regression:
    runs-on: ubuntu-latest
    env:
      LABTRUST_CHECK_BASELINES: "1"
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Baseline regression guard
        run: pytest tests/test_official_baselines_regression.py -v
        env:
          LABTRUST_CHECK_BASELINES: "1"

  golden:
    runs-on: ubuntu-latest
    timeout-minutes: 35
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Determinism report (throughput_sla, 3 episodes, seed 42)
        run: |
          labtrust determinism-report --task throughput_sla --episodes 3 --seed 42 --out ./det_report
          test -f det_report/determinism_report.json
          test -f det_report/determinism_report.md
          python -c "import json; d=json.load(open('det_report/determinism_report.json')); assert d.get('passed') is True, d.get('errors', ['unknown'])"
      - name: Determinism report (coord_risk, 2 episodes, seed 42)
        run: |
          labtrust determinism-report --task coord_risk --coord-method kernel_centralized_edf --episodes 2 --seed 42 --out ./det_report_coord
          test -f det_report_coord/determinism_report.json
          python -c "import json; d=json.load(open('det_report_coord/determinism_report.json')); assert d.get('passed') is True, d.get('errors', ['unknown'])"
      - name: Full golden suite
        run: LABTRUST_RUN_GOLDEN=1 pytest tests/test_golden_suite.py -q
        env:
          LABTRUST_RUN_GOLDEN: "1"

  determinism-golden:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Determinism report (throughput_sla, 3 episodes, seed 42)
        run: |
          labtrust determinism-report --task throughput_sla --episodes 3 --seed 42 --out ./det_report
          test -f det_report/determinism_report.json
          test -f det_report/determinism_report.md
          python -c "import json; d=json.load(open('det_report/determinism_report.json')); assert d.get('passed') is True, d.get('errors', ['unknown'])"
      - name: Determinism report (coord_risk, 2 episodes, seed 42)
        run: |
          labtrust determinism-report --task coord_risk --coord-method kernel_centralized_edf --episodes 2 --seed 42 --out ./det_report_coord
          test -f det_report_coord/determinism_report.json
          python -c "import json; d=json.load(open('det_report_coord/determinism_report.json')); assert d.get('passed') is True, d.get('errors', ['unknown'])"
      - name: Full golden suite
        run: LABTRUST_RUN_GOLDEN=1 pytest tests/test_golden_suite.py -q

  coordination-smoke:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_coordination_smoke == 'true')
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Validate policy
        run: labtrust validate-policy
      # Coordinator guardrails and multi-LLM: test_coordinator_guardrails*, test_coord_llm_*; see docs/coordination/coordination_methods.md for -k subset
      - name: Coordination tests
        run: pytest -q tests/test_coordination_*
      - name: coord_scale (centralized_planner, 1 episode)
        run: |
          labtrust run-benchmark --task coord_scale --episodes 1 --seed 42 \
            --coord-method centralized_planner --out ./taskg_smoke.json
      - name: coord_risk (market_auction, INJ-COLLUSION-001, 1 episode)
        run: |
          labtrust run-benchmark --task coord_risk --episodes 1 --seed 42 \
            --coord-method market_auction --injection INJ-COLLUSION-001 --out ./taskh_smoke.json
      - name: Coordination security pack smoke (fixed methods, critical injections)
        run: labtrust run-coordination-security-pack --out ./pack_smoke --methods-from fixed --injections-from critical --seed 42

  external-reviewer-checks:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_external_reviewer_checks == 'true')
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: External reviewer checks (coordination study + summary + COORDINATION_LLM_CARD)
        run: |
          bash scripts/run_external_reviewer_checks.sh ./external_reviewer_out tests/fixtures/coordination_study_llm_smoke_spec.yaml

  quick-eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install labtrust-gym[env,plots]
        run: pip install -e '.[env,plots]'
      - name: Quick-eval (throughput_sla, adversarial_disruption, multi_site_stat)
        run: labtrust quick-eval --seed 42 --out-dir ./labtrust_runs

  transparency-log:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package (dev + env for CLI imports)
        run: pip install -e '.[dev,env]'
      - name: Create minimal artifact and run transparency-log
        run: |
          mkdir -p artifact/_repr/throughput_sla
          echo '{"schema_version":"0.2","task":"throughput_sla","seeds":[42],"episodes":[{"seed":42,"metrics":{"throughput":5,"steps":100}}],"agent_baseline_id":"scripted_ops_v1"}' > artifact/_repr/throughput_sla/results.json
          echo '{"action_type":"CREATE_ACCESSION","t_s":10}' > artifact/_repr/throughput_sla/episodes.jsonl
          labtrust transparency-log --in artifact --out out
          test -f out/TRANSPARENCY_LOG/root.txt
          test -f out/TRANSPARENCY_LOG/log.json
          test -d out/TRANSPARENCY_LOG/proofs

  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package (docs + env so mkdocstrings can import labtrust_gym)
        run: pip install -e '.[docs,env]'
      - name: Build MkDocs
        run: mkdocs build --strict

  paper-claims-regression:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Paper claims regression (smoke)
        run: LABTRUST_PAPER_SMOKE=1 pytest tests/test_paper_claims_regression.py -v
        env:
          LABTRUST_PAPER_SMOKE: "1"

  security-fuzz-stress:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    timeout-minutes: 25
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: Security fuzz stress (hypothesis 500 examples)
        run: pytest tests/test_security_property_fuzz.py -m security_fuzz_stress -v --hypothesis-max-examples=500

  llm-attacker-baseline-regression:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package with env deps
        run: pip install -e '.[dev,env]'
      - name: LLM attacker baseline regression (when baseline present)
        run: |
          BASELINE="tests/fixtures/llm_attacker_baseline.json"
          if [ -f "$BASELINE" ]; then
            python scripts/check_llm_attacker_regression.py --baseline "$BASELINE" --policy-root .
          else
            echo "No llm_attacker_baseline.json at $BASELINE; skipping (pass). To enable: run security suite with --llm-attacker, then commit SECURITY/llm_attacker_baseline.json to $BASELINE."
          fi

  coverage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install package (dev + env for gymnasium so env-dependent tests run)
        run: pip install -e '.[dev,env]'
      - name: Run tests with coverage
        run: pytest -q -m "not slow" --cov=src/labtrust_gym --cov-report=xml --cov-report=term --cov-fail-under=54
      - name: Upload coverage report
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report
          path: coverage.xml