feat: Allow device reboot/reset/reprovisioning through annotation

Author: swagner-de

api/core/v1alpha1/groupversion_info.go

@@ -63,6 +63,23 @@ const L2VNILabel = "networking.metal.ironcore.dev/evi-name"
 // the name of the VRF they belong to.
 const VRFLabel = "networking.metal.ironcore.dev/vrf-name"
 
+// DeviceMaintenanceAnnotation is an annotation that can be applied to Device objects
+// to trigger certain disruptive operations, such as reboots or firmware upgrades.
+const DeviceMaintenanceAnnotation = "networking.metal.ironcore.dev/maintenance"
+
+// Device maintenance actions that can be requested via the DeviceMaintenanceAnnotation.
+const (
+	// DeviceMaintenanceReboot requests a device reboot.
+	DeviceMaintenanceReboot = "reboot"
+	// DeviceMaintenanceFactoryReset requests a factory reset of the device. A factory reset
+	// will erase all configuration and return the device to its original state.
+	DeviceMaintenanceFactoryReset = "factory-reset"
+	// DeviceMaintenanceReprovision requests reprovisioning of the device, without completely resetting it.
+	DeviceMaintenanceReprovision = "reprovision"
+	// DeviceMaintenanceResetPhaseToProvisioning requests resetting the device's maintenance phase to "provisioning" without rebooting or preparing the device.
+	DeviceMaintenanceResetPhaseToProvisioning = "reset-phase-to-provisioning"
+)
+
 // Condition types that are used across different objects.
 const (
 	// ReadyCondition is the top-level status condition that reports if an object is ready.

config/default/kustomization.yaml

@@ -24,7 +24,7 @@ resources:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 - ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
+- ../prometheus
 # [METRICS] Expose the controller manager metrics service.
 - metrics_service.yaml
 # [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy.

internal/controller/core/device_controller.go

@@ -86,14 +86,33 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c
 		return ctrl.Result{}, err
 	}
 
+	prov, ok := r.Provider().(provider.DeviceProvider)
+	if !ok {
+		err := errors.New("provider does not implement DeviceProvider interface")
+		log.Error(err, "failed to reconcile resource")
+		return ctrl.Result{}, err
+	}
+	conn, err := deviceutil.GetDeviceConnection(ctx, r, obj)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to obtain device connection: %w", err)
+	}
+
 	orig := obj.DeepCopy()
+
 	if conditions.InitializeConditions(obj, v1alpha1.ReadyCondition) {
 		log.Info("Initializing status conditions")
 		return ctrl.Result{}, r.Status().Update(ctx, obj)
 	}
 
-	// Always attempt to update the status after reconciliation
+	// Always attempt to update the metadata/status after reconciliation
 	defer func() {
+		if !equality.Semantic.DeepEqual(orig.ObjectMeta, obj.ObjectMeta) {
+			// pass obj.DeepCopy() to avoid Patch() modifying obj and interfering with status update below
+			if err := r.Patch(ctx, obj.DeepCopy(), client.MergeFrom(orig)); err != nil {
+				log.Error(err, "Failed to update resource metadata")
+				reterr = kerrors.NewAggregate([]error{reterr, err})
+			}
+		}
 		if !equality.Semantic.DeepEqual(orig.Status, obj.Status) {
 			if err := r.Status().Patch(ctx, obj, client.MergeFrom(orig)); err != nil {
 				log.Error(err, "Failed to update status")
@@ -130,6 +149,9 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c
 		return ctrl.Result{}, nil
 
 	case v1alpha1.DevicePhaseProvisioning:
+		annotations := obj.GetAnnotations()
+		delete(annotations, v1alpha1.DeviceMaintenanceAnnotation)
+		obj.SetAnnotations(annotations)
 		activeProv := obj.GetActiveProvisioning()
 		if activeProv == nil {
 			log.Info("Device has not made a provisioning request yet")
@@ -157,10 +179,6 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c
 		}
 		log.Info("Device provisioning completed, running post provisioning checks")
 		prov, _ := r.Provider().(provider.ProvisioningProvider)
-		conn, err := deviceutil.GetDeviceConnection(ctx, r, obj)
-		if err != nil {
-			return ctrl.Result{}, fmt.Errorf("failed to obtain device connection: %w", err)
-		}
 		if ok := prov.VerifyProvisioned(ctx, conn, obj); !ok {
 			return ctrl.Result{RequeueAfter: r.RequeueInterval}, nil
 		}
@@ -169,7 +187,7 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c
 		obj.Status.Phase = v1alpha1.DevicePhaseRunning
 
 	case v1alpha1.DevicePhaseRunning:
-		if err := r.reconcile(ctx, obj); err != nil {
+		if err := r.reconcile(ctx, obj, prov, conn); err != nil {
 			log.Error(err, "Failed to reconcile resource")
 			return ctrl.Result{}, err
 		}
@@ -187,6 +205,10 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c
 		obj.Status.Phase = v1alpha1.DevicePhaseRunning
 	}
 
+	if err := r.reconcileMaintenance(ctx, obj, prov, conn); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	return ctrl.Result{}, nil
 }
 
@@ -231,73 +253,67 @@ func (r *DeviceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func (r *DeviceReconciler) reconcile(ctx context.Context, device *v1alpha1.Device) (reterr error) {
-	if prov, ok := r.Provider().(provider.DeviceProvider); ok {
-		conn, err := deviceutil.GetDeviceConnection(ctx, r, device)
-		if err != nil {
-			return err
+func (r *DeviceReconciler) reconcile(ctx context.Context, device *v1alpha1.Device, prov provider.DeviceProvider, conn *deviceutil.Connection) (reterr error) {
+	if err := prov.Connect(ctx, conn); err != nil {
+		conditions.Set(device, metav1.Condition{
+			Type:    v1alpha1.ReadyCondition,
+			Status:  metav1.ConditionFalse,
+			Reason:  v1alpha1.UnreachableReason,
+			Message: fmt.Sprintf("Failed to connect to provider: %v", err),
+		})
+		return fmt.Errorf("failed to connect to provider: %w", err)
+	}
+	defer func() {
+		if err := prov.Disconnect(ctx, conn); err != nil {
+			reterr = kerrors.NewAggregate([]error{reterr, err})
 		}
+	}()
 
-		if err := prov.Connect(ctx, conn); err != nil {
-			conditions.Set(device, metav1.Condition{
-				Type:    v1alpha1.ReadyCondition,
-				Status:  metav1.ConditionFalse,
-				Reason:  v1alpha1.UnreachableReason,
-				Message: fmt.Sprintf("Failed to connect to provider: %v", err),
-			})
-			return fmt.Errorf("failed to connect to provider: %w", err)
-		}
-		defer func() {
-			if err := prov.Disconnect(ctx, conn); err != nil {
-				reterr = kerrors.NewAggregate([]error{reterr, err})
-			}
-		}()
+	ports, err := prov.ListPorts(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list device ports: %w", err)
+	}
 
-		ports, err := prov.ListPorts(ctx)
-		if err != nil {
-			return fmt.Errorf("failed to list device ports: %w", err)
-		}
+	interfaces := new(v1alpha1.InterfaceList)
+	if err := r.List(ctx, interfaces, client.InNamespace(device.Namespace), client.MatchingLabels{v1alpha1.DeviceLabel: device.Name}); err != nil {
+		return fmt.Errorf("failed to list interface resources for device: %w", err)
+	}
 
-		interfaces := new(v1alpha1.InterfaceList)
-		if err := r.List(ctx, interfaces, client.InNamespace(device.Namespace), client.MatchingLabels{v1alpha1.DeviceLabel: device.Name}); err != nil {
-			return fmt.Errorf("failed to list interface resources for device: %w", err)
-		}
+	m := make(map[string]string) // ID => Resource Name
+	for _, intf := range interfaces.Items {
+		m[intf.Spec.Name] = intf.Name
+	}
 
-		m := make(map[string]string) // ID => Resource Name
-		for _, intf := range interfaces.Items {
-			m[intf.Spec.Name] = intf.Name
+	device.Status.Ports = make([]v1alpha1.DevicePort, len(ports))
+	n := int32(0)
+	for i, p := range ports {
+		var ref *v1alpha1.LocalObjectReference
+		if name, ok := m[p.ID]; ok {
+			ref = &v1alpha1.LocalObjectReference{Name: name}
+			n++
 		}
-
-		device.Status.Ports = make([]v1alpha1.DevicePort, len(ports))
-		n := int32(0)
-		for i, p := range ports {
-			var ref *v1alpha1.LocalObjectReference
-			if name, ok := m[p.ID]; ok {
-				ref = &v1alpha1.LocalObjectReference{Name: name}
-				n++
-			}
-			device.Status.Ports[i] = v1alpha1.DevicePort{
-				Name:                p.ID,
-				Type:                p.Type,
-				SupportedSpeedsGbps: p.SupportedSpeedsGbps,
-				Transceiver:         p.Transceiver,
-				InterfaceRef:        ref,
-			}
-			slices.Sort(device.Status.Ports[i].SupportedSpeedsGbps)
+		device.Status.Ports[i] = v1alpha1.DevicePort{
+			Name:                p.ID,
+			Type:                p.Type,
+			SupportedSpeedsGbps: p.SupportedSpeedsGbps,
+			Transceiver:         p.Transceiver,
+			InterfaceRef:        ref,
 		}
+		slices.Sort(device.Status.Ports[i].SupportedSpeedsGbps)
+	}
 
-		device.Status.PostSummary = PortSummary(device.Status.Ports)
-
-		info, err := prov.GetDeviceInfo(ctx)
-		if err != nil {
-			return fmt.Errorf("failed to get device details: %w", err)
-		}
+	device.Status.PostSummary = PortSummary(device.Status.Ports)
 
-		device.Status.Manufacturer = info.Manufacturer
-		device.Status.Model = info.Model
-		device.Status.SerialNumber = info.SerialNumber
-		device.Status.FirmwareVersion = info.FirmwareVersion
+	info, err := prov.GetDeviceInfo(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get device details: %w", err)
 	}
+
+	device.Status.Manufacturer = info.Manufacturer
+	device.Status.Model = info.Model
+	device.Status.SerialNumber = info.SerialNumber
+	device.Status.FirmwareVersion = info.FirmwareVersion
+
 	conditions.Set(device, metav1.Condition{
 		Type:    v1alpha1.ReadyCondition,
 		Status:  metav1.ConditionTrue,
@@ -308,6 +324,40 @@ func (r *DeviceReconciler) reconcile(ctx context.Context, device *v1alpha1.Devic
 	return nil
 }
 
+func (r *DeviceReconciler) reconcileMaintenance(ctx context.Context, obj *v1alpha1.Device, prov provider.DeviceProvider, conn *deviceutil.Connection) error {
+	action, ok := obj.Annotations[v1alpha1.DeviceMaintenanceAnnotation]
+	if !ok {
+		return nil
+	}
+	delete(obj.Annotations, v1alpha1.DeviceMaintenanceAnnotation)
+	switch action {
+	case v1alpha1.DeviceMaintenanceReboot:
+		r.Recorder.Event(obj, "Normal", "RebootRequested", "Device reboot has been requested")
+		if err := prov.Reboot(ctx, conn); err != nil {
+			return fmt.Errorf("failed to reboot device: %w", err)
+		}
+
+	case v1alpha1.DeviceMaintenanceFactoryReset:
+		r.Recorder.Event(obj, "Normal", "FactoryResetRequested", "Device factory reset has been requested")
+		if err := prov.FactoryReset(ctx, conn); err != nil {
+			return fmt.Errorf("failed to reset device to factory defaults: %w", err)
+		}
+
+	case v1alpha1.DeviceMaintenanceReprovision:
+		r.Recorder.Event(obj, "Normal", "ReprovisioningRequested", "Device reprovisioning has been requested. Preparing the device.")
+		if err := prov.Reprovision(ctx, conn); err != nil {
+			return fmt.Errorf("failed to reset device to factory defaults: %w", err)
+		}
+		obj.Status.Phase = v1alpha1.DevicePhasePending
+	case v1alpha1.DeviceMaintenanceResetPhaseToProvisioning:
+		r.Recorder.Event(obj, "Normal", "ResetPhaseToProvisioningRequested", "Device phase reset to Pending has been requested.")
+		obj.Status.Phase = v1alpha1.DevicePhasePending
+	default:
+		return fmt.Errorf("unknown device action: %s", action)
+	}
+	return nil
+}
+
 // secretToDevices is a [handler.MapFunc] to be used to enqueue requests for reconciliation
 // for a Device to update when one of its referenced Secrets gets updated.
 func (r *DeviceReconciler) secretToDevices(ctx context.Context, obj client.Object) []ctrl.Request {

internal/controller/core/device_controller_test.go

@@ -203,9 +203,8 @@ var _ = Describe("Device Controller", func() {
 				resource.Status.Phase = v1alpha1.DevicePhaseProvisioned
 				resource.Status.Provisioning = []v1alpha1.ProvisioningInfo{
 					{
-						Token:      "test-token",
-						StartTime:  metav1.NewTime(time.Now().Add(-3 * time.Minute)),
-						RebootTime: metav1.NewTime(time.Now().Add(-30 * time.Second)),
+						Token:     "test-token",
+						StartTime: metav1.NewTime(time.Now().Add(-3 * time.Minute)),
 					},
 				}
 				g.Expect(k8sClient.Status().Patch(ctx, resource, client.MergeFrom(device))).To(Succeed())
@@ -230,5 +229,68 @@ var _ = Describe("Device Controller", func() {
 				g.Expect(resource.Status.Conditions[0].Reason).To(Equal(v1alpha1.ReadyReason))
 			}).Should(Succeed())
 		})
+
+		It("Should transition from Active to Provisioning once the reset-phase-to-provisioning annotation is set", func() {
+			By("Creating a Device")
+			device := &v1alpha1.Device{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      key.Name,
+					Namespace: key.Namespace,
+				},
+				Spec: v1alpha1.DeviceSpec{
+					Endpoint: v1alpha1.Endpoint{
+						Address: "192.168.10.5:9339",
+						SecretRef: &v1alpha1.SecretReference{
+							Name: name,
+						},
+					},
+					Provisioning: &v1alpha1.Provisioning{
+						BootScript: v1alpha1.TemplateSource{
+							Inline: ptr.To("boot nxos.bin"),
+						},
+						Image: v1alpha1.Image{
+							URL:          "https://best-vendor-images.to/windows98",
+							Checksum:     "d41d8cd98f00b204e9800998ecf8427e",
+							ChecksumType: v1alpha1.ChecksumTypeMD5,
+						},
+					},
+				},
+			}
+			Expect(k8sClient.Create(ctx, device)).To(Succeed())
+
+			By("Setting the device to Running phase")
+			orig := device.DeepCopy()
+			device.Status.Phase = v1alpha1.DevicePhaseRunning
+			Expect(k8sClient.Status().Patch(ctx, device, client.MergeFrom(orig))).To(Succeed())
+
+			By("Verifying the device transitions to Running phase")
+			Eventually(func(g Gomega) {
+				resource := &v1alpha1.Device{}
+				g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
+				g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseRunning))
+				g.Expect(resource.Status.Conditions).To(HaveLen(1))
+				g.Expect(resource.Status.Conditions[0].Type).To(Equal(v1alpha1.ReadyCondition))
+			}).Should(Succeed())
+
+			By("Adding the reset-phase-to-provisioning annotation to the device")
+			Eventually(func(g Gomega) {
+				resource := &v1alpha1.Device{}
+				g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
+				patch := resource.DeepCopy()
+				annotations := make(map[string]string)
+				annotations[v1alpha1.DeviceMaintenanceAnnotation] = v1alpha1.DeviceMaintenanceResetPhaseToProvisioning
+				patch.SetAnnotations(annotations)
+				g.Expect(k8sClient.Patch(ctx, patch, client.MergeFrom(resource))).To(Succeed())
+			}).Should(Succeed())
+
+			By("Verifying the device transitions to Provisioning phase and the annotation is removed")
+			Eventually(func(g Gomega) {
+				resource := &v1alpha1.Device{}
+				g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
+				g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseProvisioning))
+				_, exists := resource.Annotations[v1alpha1.DeviceMaintenanceAnnotation]
+				g.Expect(exists).To(BeFalse(), "Maintenance annotation should be removed after processing")
+			}).WithTimeout(time.Second * 10).Should(Succeed())
+		})
 	})
 })

internal/controller/core/suite_test.go

@@ -466,6 +466,18 @@ func (p *Provider) VerifyProvisioned(context.Context, *deviceutil.Connection, *v
 	return true
 }
 
+func (p *Provider) Reboot(ctx context.Context, conn *deviceutil.Connection) error {
+	return nil
+}
+
+func (p *Provider) FactoryReset(ctx context.Context, conn *deviceutil.Connection) error {
+	return nil
+}
+
+func (p *Provider) Reprovision(ctx context.Context, conn *deviceutil.Connection) (reterr error) {
+	return nil
+}
+
 func (p *Provider) EnsureInterface(ctx context.Context, req *provider.EnsureInterfaceRequest) error {
 	p.Lock()
 	defer p.Unlock()