[dcgm][dcgm-exporter] add liveness and readiness probes

This commit adds liveness and readiness probes to
the dcgm and dcgm-exporter operands. Adding probes
to the DCGM pods ensure that these pods aren't marked
as "Ready" until the DCGM is actually ready to serve
traffic. The DCGM-Exporter probes have been taken
from the default probes configured in the helm chart
of the NVIDIA/dcgm-exporter project.

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>

Author: tariq1890

assets/state-dcgm-exporter/0800_daemonset.yaml

@@ -52,6 +52,17 @@ spec:
         ports:
         - name: "metrics"
           containerPort: 9400
+        livenessProbe:
+          httpGet:
+            port: 9400
+            path: /health
+          initialDelaySeconds: 45
+          periodSeconds: 5
+        readinessProbe:
+          httpGet:
+            port: 9400
+            path: /health
+          initialDelaySeconds: 45
         volumeMounts:
         - name: "pod-gpu-resources"
           readOnly: true

assets/state-dcgm/0400_dcgm.yml

@@ -43,6 +43,14 @@ spec:
         ports:
         - name: "dcgm"
           containerPort: 5555
+        livenessProbe:
+          tcpSocket:
+            port: 5555
+          initialDelaySeconds: 15
+        readinessProbe:
+          tcpSocket:
+            port: 5555
+          initialDelaySeconds: 15
       volumes:
         - name: run-nvidia
           hostPath: