test-software.eessi.io.yml

# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: Check for missing software installations in software.eessi.io
on:
  push:
    branches: [ "main" ]
  pull_request:
  workflow_dispatch:
permissions:
  contents: read # to fetch code (actions/checkout)
env:
  # list below should correspond with table @ https://eessi.io/docs/software_layer/gpu_targets/
  EESSI_ACCELERATOR_TARGETS: |
    2023.06:
      # Provide a default set of compute capabilities
      default:
        - nvidia/cc70
        - nvidia/cc80
        - nvidia/cc90
      # and then allow for special cases for specific architectures
      x86_64/amd/zen2:
        - nvidia/cc70
        - nvidia/cc80
        - nvidia/cc90
      aarch64/a64fx: []
    2025.06:
      # Provide a default set of compute capabilities
      default:
        - nvidia/cc70
        - nvidia/cc80
        - nvidia/cc90
        - nvidia/cc100
        - nvidia/cc120
      # and then allow for special cases for specific architectures
      aarch64/a64fx: []
jobs:
  # Checks whether this PR modifies any easystack files and, if so,
  # determines which EESSI versions are affected.
  # It then stores those versions as a space-separated list in an
  # environment variable.
  # Finally, it writes that value to GITHUB_OUTPUT using the correct
  # heredoc format.
  check_EESSI_version_changed_files:
    runs-on: ubuntu-24.04
    outputs:
      EESSI_VERSIONS: ${{ steps.detect.outputs.EESSI_VERSIONS }}
    steps:
      - name: Check out software-layer repository
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0  # Fetch all history for all branches and tags

      - name: Detect EESSI version in modified easystack files
        id: detect
        run: |
          # Fetch base branch explicitly to ensure it's available
          git fetch origin ${{ github.base_ref }}

          changed_files=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
          echo "Files changed: $changed_files"

          # Default to not checking missing software for any version
          EESSI_VERSIONS=""
          
          # Check for specific versions in the changed easystack files
          # The regex pattern matches a prefix, but only returns the part after \K, so that we get only the version
          # The sort ensures predictable ordering, and with -u only keeps unique items (neither are probably essential, but both are nice)
          # Finally, since the grep returns multiple lines if there are multiple versions that were touched,
          # the tr replaces newlines with space, to make this space-separated.          
          EESSI_VERSIONS=$(git diff --name-only origin/${{ github.base_ref }}...HEAD \
            | grep -oP 'easystacks/software\.eessi\.io/\K[0-9]+\.[0-9]+' \
            | sort -u \
            | tr '\n' ' ')
          echo "PR easystack changes related to EESSI VERSION: $EESSI_VERSIONS"
          
          # Use GITHUB_OUTPUT heredoc correctly
          echo "EESSI_VERSIONS<<EOF" >> "$GITHUB_OUTPUT"
          echo "$EESSI_VERSIONS" >> "$GITHUB_OUTPUT"
          echo "EOF" >> "$GITHUB_OUTPUT"

  check_missing:
    needs: check_EESSI_version_changed_files
    strategy:
      fail-fast: false
      matrix:
        include:
          # Arm CPU targets (EESSI 2023.06)
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_n1
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_v1
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/nvidia/grace
          # Arm CPU targets (EESSI 2025.06)
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/a64fx
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_n1
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_v1
          - runs_on: ubuntu-24.04-arm
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/nvidia/grace
          # x86_64 CPU targets (EESSI 2023.06)
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen2
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen3
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen4
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/haswell
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/sapphirerapids
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/skylake_avx512
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/icelake
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/cascadelake
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2023.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic
          # x86_64 CPU targets (EESSI 2025.06)
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen2
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen3
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen4
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen5
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/haswell
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/sapphirerapids
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/skylake_avx512
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/icelake
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/cascadelake
          - runs_on: ubuntu-24.04
            EESSI_VERSION: 2025.06
            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic
    runs-on: ${{ matrix.runs_on }}
    steps:
        - name: Check out software-layer repository
          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
          with:
              fetch-depth: 0  # Fetch all history for all branches and tags

        - name: Clone EESSI/software-layer-scripts repository
          run: |
            git clone https://github.com/EESSI/software-layer-scripts

        - name: Show host system info
          run: |
            echo "/proc/cpuinfo:"
            cat /proc/cpuinfo
            echo
            echo "lscpu:"
            lscpu

        - name: Mount EESSI CernVM-FS pilot repository
          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
          with:
              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
              cvmfs_http_proxy: DIRECT
              cvmfs_repositories: software.eessi.io

        - name: Check for missing installlations
          if: contains(needs.check_EESSI_version_changed_files.outputs.EESSI_VERSIONS, matrix.EESSI_VERSION)
          run: |
              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
              # to prevent issues with checks in the Easybuild configuration that use this variable
              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
              export EESSI_OS_TYPE=linux
              env | grep ^EESSI | sort

              # first check the CPU-only builds for this CPU target
              echo "first run check_missing_installations.sh for CPU-only builds"
              for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
                  eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
                  echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
                  module purge
                  module load EasyBuild/${eb_version}
                  which eb
                  eb --version
                  software-layer-scripts/check_missing_installations.sh ${easystack_file}
                  ec=$?
                  if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
              done

              # now check the accelerator builds for this CPU target
              accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".\"${{matrix.EESSI_VERSION}}\".\"${EESSI_SOFTWARE_SUBDIR_OVERRIDE}\" // .\"${{matrix.EESSI_VERSION}}\".default | .[]")
              if [ -z "${accelerators}" ]; then
                  echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
              else
                  for accel in ${accelerators}; do
                      module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
                      echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
                      for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
                          eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
                          echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
                          module purge
                          module load EasyBuild/${eb_version}
                          which eb
                          eb --version
                          software-layer-scripts/check_missing_installations.sh ${easystack_file}
                          ec=$?
                          if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
                      done
                      module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
                  done
              fi

              # make sure that Lmod cache file is present
              ls -l ${EESSI_SOFTWARE_PATH}/.lmod/cache/spiderT.lua

        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
          run: |
              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
              # to prevent issues with checks in the Easybuild configuration that use this variable
              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
              module load EasyBuild
              which eb
              eb --version
              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
              export EESSI_OS_TYPE=linux
              env | grep ^EESSI | sort
              # create dummy easystack file with a single entry (something that is not installed in EESSI)
              easystack_file="test.yml"
              echo "easyconfigs:" > ${easystack_file}
              echo "  - GCC-8.3.0:" >> ${easystack_file}
              echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):"
              cat ${easystack_file}
              # note, check_missing_installations.sh exits 1 if a package was
              # missing, which is intepreted as false (exit code based, not
              # boolean logic), hence when the script exits 0 if no package was
              # missing it is interpreted as true, thus the test did not capture
              # the missing package
              if software-layer-scripts/check_missing_installations.sh ${easystack_file}; then
                echo "did NOT capture missing package; test FAILED"
                exit 1
              else
                echo "captured missing package; test PASSED"
                exit 0
              fi