Add tests with dranet

tests/scenarios/k8s/files/dranet-ds.yaml

@@ -0,0 +1,182 @@
+# Copyright The Kubernetes Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: dranet
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+  - apiGroups:
+      - "resource.k8s.io"
+    resources:
+      - resourceslices
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - "resource.k8s.io"
+    resources:
+      - resourceclaims
+      - deviceclasses
+    verbs:
+      - get
+  - apiGroups:
+      - "resource.k8s.io"
+    resources:
+      - resourceclaims/status
+    verbs:
+      - patch
+      - update
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: dranet
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: dranet
+subjects:
+- kind: ServiceAccount
+  name: dranet
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dranet
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: dranet
+  namespace: kube-system
+  labels:
+    tier: node
+    app: dranet
+    k8s-app: dranet
+spec:
+  selector:
+    matchLabels:
+      app: dranet
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: dranet
+        k8s-app: dranet
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: dranet
+      hostPID: true
+      initContainers:
+      - name: enable-nri
+        image: busybox:stable
+        volumeMounts:
+        - mountPath: /etc
+          name: etc
+        securityContext:
+          privileged: true
+        command:
+        - /bin/sh
+        - -c
+        - |
+          set -o errexit
+          set -o pipefail
+          set -o nounset
+          set -x
+          if grep -q "io.containerd.nri.v1.nri" /etc/containerd/config.toml
+          then
+             echo "containerd config contains NRI reference already; taking no action"
+          else
+             echo "containerd config does not mention NRI, thus enabling it";
+             printf '%s\n' "[plugins.\"io.containerd.nri.v1.nri\"]" "  disable = false" "  disable_connections = false" "  plugin_config_path = \"/etc/nri/conf.d\"" "  plugin_path = \"/opt/nri/plugins\"" "  plugin_registration_timeout = \"5s\"" "  plugin_request_timeout = \"5s\"" "  socket_path = \"/var/run/nri/nri.sock\"" >> /etc/containerd/config.toml
+             echo "restarting containerd"
+             nsenter -t 1 -m -u -i -n -p -- systemctl restart containerd
+          fi
+      containers:
+      - name: dranet
+        args:
+        - /dranet
+        - --v=4
+        - --hostname-override=$(NODE_NAME)
+        image: acnpublic.azurecr.io/dranet:dev8
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 9177
+        volumeMounts:
+        - name: device-plugin
+          mountPath: /var/lib/kubelet/plugins
+        - name: plugin-registry
+          mountPath: /var/lib/kubelet/plugins_registry
+        - name: nri-plugin
+          mountPath: /var/run/nri
+        - name: netns
+          mountPath: /var/run/netns
+          mountPropagation: HostToContainer
+        - name: infiniband
+          mountPath: /dev/infiniband
+          mountPropagation: HostToContainer
+        - name: bpf-programs
+          mountPath: /sys/fs/bpf
+          mountPropagation: HostToContainer
+      volumes:
+      - name: device-plugin
+        hostPath:
+          path: /var/lib/kubelet/plugins
+      - name: plugin-registry
+        hostPath:
+          path: /var/lib/kubelet/plugins_registry
+      - name: nri-plugin
+        hostPath:
+          path: /var/run/nri
+      - name: netns
+        hostPath:
+          path: /var/run/netns
+      - name: infiniband
+        hostPath:
+          path: /dev/infiniband
+      - name: etc
+        hostPath:
+          path: /etc
+      - name: bpf-programs
+        hostPath:
+          path: /sys/fs/bpf
+---

tests/scenarios/k8s/templates/dranet-deviceclass.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.dranet.enabled }}
+# DeviceClass for DRANET - selects RDMA-capable NICs managed by DRANET
+apiVersion: resource.k8s.io/v1
+kind: DeviceClass
+metadata:
+  name: {{ .Values.dranet.deviceClassName | default "dranet-rdma" }}
+spec:
+  selectors:
+    - cel:
+        expression: device.driver == "dra.net"
+    - cel:
+        expression: device.attributes["dra.net"].rdma == true
+        expression: device.attributes["dra.net"].encapsulation == "infiniband"
+{{- end }}

tests/scenarios/k8s/templates/dranet-resourceclaimtemplate.yaml

@@ -0,0 +1,30 @@
+{{- if .Values.dranet.enabled }}
+# ResourceClaimTemplate for requesting RDMA NICs via DRA
+apiVersion: resource.k8s.io/v1
+kind: ResourceClaimTemplate
+metadata:
+  name: {{ .Values.dranet.resourceClaimTemplateName | default "rdma-nic-template" }}
+spec:
+  spec:
+    devices:
+      requests:
+      - name: rdma-nic
+        exactly:
+          deviceClassName: {{ .Values.dranet.deviceClassName | default "dranet-rdma" }}
+          count: {{ .Values.dranet.nicCount | default 1 }}
+          {{- if .Values.dranet.selectors }}
+          selectors:
+          {{- range .Values.dranet.selectors }}
+          - cel:
+              expression: {{ . | quote }}
+          {{- end }}
+          {{- end }}
+      {{- if .Values.dranet.interfaceConfig }}
+      config:
+      - opaque:
+          driver: dra.net
+          parameters:
+            interface:
+              {{- toYaml .Values.dranet.interfaceConfig | nindent 14 }}
+      {{- end }}
+{{- end }}

tests/scenarios/k8s/templates/leader-job.yaml

@@ -17,6 +17,11 @@ spec:
       labels:
         role: leader
     spec:
+      {{- if .Values.dranet.enabled }}
+      resourceClaims:
+      - name: rdma-nic
+        resourceClaimTemplateName: {{ .Values.dranet.resourceClaimTemplateName | default "rdma-nic-template" }}
+      {{- end }}
       containers:
       - name: runner
         image: {{ .Values.job.image }}

tests/scenarios/k8s/templates/mpi-job.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   slotsPerWorker: 8
   runPolicy:
-    cleanPodPolicy: Running
+    cleanPodPolicy: {{ .Values.mpiJob.cleanPodPolicy | default "Running" }}
   mpiReplicaSpecs:
     Launcher:
       replicas: 1
@@ -76,6 +76,11 @@ spec:
           labels:
             task: test
         spec:
+          {{- if .Values.dranet.enabled }}
+          resourceClaims:
+          - name: rdma-nic
+            resourceClaimTemplateName: {{ .Values.dranet.resourceClaimTemplateName | default "rdma-nic-template" }}
+          {{- end }}
           containers:
           - image: {{ .Values.mpiJob.image }}
             name: nccl
@@ -118,3 +123,4 @@ spec:
           enableServiceLinks: false
           automountServiceAccountToken: false
 {{- end }}
+

tests/scenarios/k8s/templates/worker-job.yaml

@@ -17,6 +17,11 @@ spec:
       labels:
         role: worker
     spec:
+      {{- if .Values.dranet.enabled }}
+      resourceClaims:
+      - name: rdma-nic
+        resourceClaimTemplateName: {{ .Values.dranet.resourceClaimTemplateName | default "rdma-nic-template" }}
+      {{- end }}
       containers:
       - name: runner
         image: {{ .Values.job.image }}

tests/scenarios/k8s/values.yaml

@@ -2,6 +2,7 @@ mpiJob:
   enabled: false
   image: ghcr.io/azure/aks-rdma-infiniband/nccl-tests
   numberOfProcesses: 0
+  cleanPodPolicy: Running  # Options: Running, All, None
 
 job:
   enabled: false
@@ -21,11 +22,22 @@ job:
 
 ipoib: false
 
+# DRANET (DRA) Configuration for RDMA NICs via Resource Claims
+# Enable this to use Dynamic Resource Allocation instead of device plugin
+dranet:
+  enabled: false
+  # DeviceClass name for RDMA NICs
+  deviceClassName: dranet-rdma
+  # ResourceClaimTemplate name
+  resourceClaimTemplateName: rdma-nic-template
+  # Number of RDMA NICs to request per pod
+  nicCount: 8
+
 # --------------------------------------------------------------------
 # Below this point it is gonna be constant for all the tests
 ncclEnvVars:
   NCCL_NET_GDR_LEVEL: SYS        # Needed for MPI Job.
-  NCCL_IB_DISABLE: "0"             # Force NCCL to use Infiniband.
+  NCCL_IB_DISABLE: "0"           # Force NCCL to use Infiniband.
   # NCCL_DEBUG: INFO             # Valid values: VERSION, WARN, INFO, TRACE
   # NCCL_DEBUG_SUBSYS: INIT,NET
   # DEBUG: true                  # Enable script in verbose mode.