Update fork to patch v1.120.1 (#153)

* [release-v1.120] Fix credentials rotation without workers rollout for in-place update (gardener#12249)

* Use correct lastInitationTime during credentials rotation without workers rollout

* Use correct function in e2e test

This was wrongly replaced in gardener#11990

* Add e2e tests for in-place and without-workers-rollout combination

* Reconcile shoot when rotation status is WaitingForWorkersRollout and manual in-place pending workers are updated

---------

Co-authored-by: Shafeeque E S <[email protected]>

* Release v1.120.1

---------

Co-authored-by: Gardener Prow Robot <[email protected]>
Co-authored-by: Shafeeque E S <[email protected]>
Co-authored-by: gardener-robot-ci-2 <[email protected]>

Author: 4 people

VERSION

@@ -1 +1 @@
-v1.120.0
+v1.120.1

docs/deployment/getting_started_locally.md

@@ -266,6 +266,8 @@ cat <<EOF | sudo tee -a /etc/hosts
 172.18.255.1 api.e2e-rot-noroll.local.internal.local.gardener.cloud
 172.18.255.1 api.e2e-rot-ip.local.external.local.gardener.cloud
 172.18.255.1 api.e2e-rot-ip.local.internal.local.gardener.cloud
+172.18.255.1 api.e2e-rot-nr-ip.local.external.local.gardener.cloud
+172.18.255.1 api.e2e-rot-nr-ip.local.internal.local.gardener.cloud
 172.18.255.1 api.e2e-default.local.external.local.gardener.cloud
 172.18.255.1 api.e2e-default.local.internal.local.gardener.cloud
 172.18.255.1 api.e2e-default-wl.local.external.local.gardener.cloud
@@ -296,6 +298,7 @@ cat <<EOF | sudo tee -a /etc/hosts
 172.18.255.1 gu-local--e2e-rotate-wl.ingress.local.seed.local.gardener.cloud
 172.18.255.1 gu-local--e2e-rot-noroll.ingress.local.seed.local.gardener.cloud
 172.18.255.1 gu-local--e2e-rot-ip.ingress.local.seed.local.gardener.cloud
+172.18.255.1 gu-local--e2e-rot-nr-ip.ingress.local.seed.local.gardener.cloud
 # End of Gardener local setup section
 EOF
 ```

hack/test-e2e-local.sh

@@ -94,6 +94,7 @@ case $TYPE in
       e2e-rotate-wl.local
       e2e-rot-noroll.local
       e2e-rot-ip.local
+      e2e-rot-nr-ip.local
       e2e-default.local
       e2e-default-wl.local
       e2e-default-ip.local

pkg/component/extensions/operatingsystemconfig/operatingsystemconfig.go

@@ -731,12 +731,13 @@ func (o *operatingSystemConfig) newDeployer(version int, osc *extensionsv1alpha1
 
 	var caRotationLastInitiationTime, serviceAccountKeyRotationLastInitiationTime *metav1.Time
 
-	if o.values.CredentialsRotationStatus != nil {
-		if o.values.CredentialsRotationStatus.CertificateAuthorities != nil {
-			caRotationLastInitiationTime = o.values.CredentialsRotationStatus.CertificateAuthorities.LastInitiationTime
+	if credentialsRotation := o.values.CredentialsRotationStatus; credentialsRotation != nil {
+		if caRotation := credentialsRotation.CertificateAuthorities; caRotation != nil {
+			caRotationLastInitiationTime = v1beta1helper.LastInitiationTimeForWorkerPool(worker.Name, caRotation.PendingWorkersRollouts, caRotation.LastInitiationTime)
 		}
-		if o.values.CredentialsRotationStatus.ServiceAccountKey != nil {
-			serviceAccountKeyRotationLastInitiationTime = o.values.CredentialsRotationStatus.ServiceAccountKey.LastInitiationTime
+
+		if serviceAccountKeyRotation := credentialsRotation.ServiceAccountKey; serviceAccountKeyRotation != nil {
+			serviceAccountKeyRotationLastInitiationTime = v1beta1helper.LastInitiationTimeForWorkerPool(worker.Name, serviceAccountKeyRotation.PendingWorkersRollouts, serviceAccountKeyRotation.LastInitiationTime)
 		}
 	}
 

pkg/component/extensions/operatingsystemconfig/operatingsystemconfig_test.go

@@ -60,6 +60,9 @@ var _ = Describe("OperatingSystemConfig", func() {
 
 			worker1Name = "worker1"
 			worker2Name = "worker2"
+
+			inPlaceWorkerName1 = worker1Name + "-in-place"
+			inPlaceWorkerName2 = worker2Name + "-in-place"
 		)
 
 		var (
@@ -143,6 +146,8 @@ var _ = Describe("OperatingSystemConfig", func() {
 
 			empty    *extensionsv1alpha1.OperatingSystemConfig
 			expected []*extensionsv1alpha1.OperatingSystemConfig
+
+			globalLastInitiationTime = &metav1.Time{Time: time.Date(2020, 12, 2, 10, 0, 0, 0, time.UTC)}
 		)
 
 		computeExpectedOperatingSystemConfigs := func(sshAccessEnabled bool, workers []gardencorev1beta1.Worker, inPlaceUpdate bool) []*extensionsv1alpha1.OperatingSystemConfig {
@@ -315,15 +320,25 @@ var _ = Describe("OperatingSystemConfig", func() {
 					oscOriginal.Spec.Units = originalUnits
 					oscOriginal.Spec.Files = originalFiles
 
+					caRotationLastInitiationTime := globalLastInitiationTime
+					if worker.Name == inPlaceWorkerName1 {
+						caRotationLastInitiationTime = &metav1.Time{Time: time.Date(2020, 12, 2, 1, 0, 0, 0, time.UTC)}
+					}
+
+					serviceAccountKeyRotationLastInitiationTime := globalLastInitiationTime
+					if worker.Name == inPlaceWorkerName2 {
+						serviceAccountKeyRotationLastInitiationTime = &metav1.Time{Time: time.Date(2020, 12, 2, 2, 0, 0, 0, time.UTC)}
+					}
+
 					oscOriginal.Spec.InPlaceUpdates = &extensionsv1alpha1.InPlaceUpdates{
 						OperatingSystemVersion: *worker.Machine.Image.Version,
 						KubeletVersion:         k8sVersion.String(),
 						CredentialsRotation: &extensionsv1alpha1.CredentialsRotation{
 							CertificateAuthorities: &extensionsv1alpha1.CARotation{
-								LastInitiationTime: &metav1.Time{Time: time.Date(2020, 12, 2, 10, 0, 0, 0, time.UTC)},
+								LastInitiationTime: caRotationLastInitiationTime,
 							},
 							ServiceAccountKey: &extensionsv1alpha1.ServiceAccountKeyRotation{
-								LastInitiationTime: &metav1.Time{Time: time.Date(2020, 12, 2, 10, 0, 0, 0, time.UTC)},
+								LastInitiationTime: serviceAccountKeyRotationLastInitiationTime,
 							},
 						},
 					}
@@ -388,7 +403,7 @@ var _ = Describe("OperatingSystemConfig", func() {
 			}
 			inPlaceUpdateWorkers = []gardencorev1beta1.Worker{
 				{
-					Name: worker1Name + "-in-place",
+					Name: inPlaceWorkerName1,
 					Machine: gardencorev1beta1.Machine{
 						Architecture: ptr.To(v1beta1constants.ArchitectureAMD64),
 						Image: &gardencorev1beta1.ShootMachineImage{
@@ -410,7 +425,7 @@ var _ = Describe("OperatingSystemConfig", func() {
 					UpdateStrategy: ptr.To(gardencorev1beta1.AutoInPlaceUpdate),
 				},
 				{
-					Name: worker2Name + "-in-place",
+					Name: inPlaceWorkerName2,
 					Machine: gardencorev1beta1.Machine{
 						Architecture: ptr.To(v1beta1constants.ArchitectureAMD64),
 						Image: &gardencorev1beta1.ShootMachineImage{
@@ -728,10 +743,22 @@ var _ = Describe("OperatingSystemConfig", func() {
 						},
 						CredentialsRotationStatus: &gardencorev1beta1.ShootCredentialsRotation{
 							CertificateAuthorities: &gardencorev1beta1.CARotation{
-								LastInitiationTime: &metav1.Time{Time: time.Date(2020, 12, 2, 10, 0, 0, 0, time.UTC)},
+								LastInitiationTime: globalLastInitiationTime,
+								PendingWorkersRollouts: []gardencorev1beta1.PendingWorkersRollout{
+									{
+										Name:               inPlaceWorkerName1,
+										LastInitiationTime: &metav1.Time{Time: time.Date(2020, 12, 2, 1, 0, 0, 0, time.UTC)},
+									},
+								},
 							},
 							ServiceAccountKey: &gardencorev1beta1.ServiceAccountKeyRotation{
-								LastInitiationTime: &metav1.Time{Time: time.Date(2020, 12, 2, 10, 0, 0, 0, time.UTC)},
+								LastInitiationTime: globalLastInitiationTime,
+								PendingWorkersRollouts: []gardencorev1beta1.PendingWorkersRollout{
+									{
+										Name:               inPlaceWorkerName2,
+										LastInitiationTime: &metav1.Time{Time: time.Date(2020, 12, 2, 2, 0, 0, 0, time.UTC)},
+									},
+								},
 							},
 						},
 					}

pkg/gardenlet/controller/shoot/status/reconciler.go

@@ -150,19 +150,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
 		}
 	}
 
-	log.Info("Updating Shoot status with manual in-place pending workers", "shoot", shoot.Name, "manualInPlacePendingWorkers", sets.List(manualInPlacePendingWorkers))
+	log.Info("Updating Shoot status with manual in-place pending workers", "manualInPlacePendingWorkers", sets.List(manualInPlacePendingWorkers))
 	if err := r.GardenClient.Status().Patch(ctx, shoot, patch); err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to patch Shoot status: %w", err)
 	}
 
 	// The credentials rotation phases are not set to Prepared in the Shoot reconciliation flow if there are manual in-place pending workers. So we need to trigger a reconciliation after they are all updated.
 	if noManualInPlacePendingWorkers {
-		shootNeedsReconcile = (v1beta1helper.GetShootCARotationPhase(shoot.Status.Credentials) == gardencorev1beta1.RotationPreparing || v1beta1helper.GetShootServiceAccountKeyRotationPhase(shoot.Status.Credentials) == gardencorev1beta1.RotationPreparing)
+		shootNeedsReconcile = needsReconcile(shoot)
 	}
 
 	if shootNeedsReconcile || (noInPlacePendingWorkers && kubernetesutils.HasMetaDataAnnotation(shoot, v1beta1constants.GardenerOperation, v1beta1constants.ShootOperationForceInPlaceUpdate)) {
 		patch := client.MergeFromWithOptions(shoot.DeepCopy(), client.MergeFromWithOptimisticLock{})
 		if shootNeedsReconcile {
+			log.Info("Triggering a Shoot reconciliation after credential rotations for manual in-place workers are prepared")
 			metav1.SetMetaDataAnnotation(&shoot.ObjectMeta, v1beta1constants.GardenerOperation, v1beta1constants.GardenerOperationReconcile)
 		} else {
 			delete(shoot.Annotations, v1beta1constants.GardenerOperation)
@@ -175,3 +176,19 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
 
 	return reconcile.Result{}, nil
 }
+
+func needsReconcile(shoot *gardencorev1beta1.Shoot) bool {
+	caRotationPhase := v1beta1helper.GetShootCARotationPhase(shoot.Status.Credentials)
+	serviceAccountKeyRotationPhase := v1beta1helper.GetShootServiceAccountKeyRotationPhase(shoot.Status.Credentials)
+
+	if caRotationPhase == gardencorev1beta1.RotationPreparing || serviceAccountKeyRotationPhase == gardencorev1beta1.RotationPreparing {
+		return true
+	}
+
+	// If there are no pending workers rollouts for either CA or ServiceAccountKey rotation, then we need to reconcile the Shoot.
+	if caRotationPhase == gardencorev1beta1.RotationWaitingForWorkersRollout && len(shoot.Status.Credentials.Rotation.CertificateAuthorities.PendingWorkersRollouts) == 0 {
+		return true
+	}
+
+	return serviceAccountKeyRotationPhase == gardencorev1beta1.RotationWaitingForWorkersRollout && len(shoot.Status.Credentials.Rotation.ServiceAccountKey.PendingWorkersRollouts) == 0
+}

test/e2e/gardener/shoot/create_rotate_delete.go

@@ -75,6 +75,11 @@ func testCredentialRotation(s *ShootContext, shootVerifiers, utilsverifiers rota
 		}
 
 		ItShouldWaitForShootToBeReconciledAndHealthy(s)
+
+		if inPlaceUpdate {
+			inplace.ItShouldVerifyInPlaceUpdateCompletion(s.GardenClient, s.Shoot)
+		}
+
 		shootVerifiers.AfterPrepared(context.TODO())
 		for _, k := range utilsverifiers {
 			It(fmt.Sprintf("Verify after prepared for %T", k), func(ctx SpecContext) {
@@ -122,15 +127,15 @@ func testCredentialRotationComplete(s *ShootContext, shootVerifiers, utilsverifi
 	}
 }
 
-func testCredentialRotationWithoutWorkersRollout(s *ShootContext, shootVerifiers rotationutils.Verifiers, utilsverifiers rotationutils.Verifiers) {
+func testCredentialRotationWithoutWorkersRollout(s *ShootContext, shootVerifiers rotationutils.Verifiers, utilsverifiers rotationutils.Verifiers, inPlaceUpdate bool) {
 	shootVerifiers.Before(context.TODO())
 	for _, k := range utilsverifiers {
 		It(fmt.Sprintf("Verify before for %T", k), func(ctx SpecContext) {
 			k.Before(ctx)
 		}, SpecTimeout(5*time.Minute))
 	}
 
-	nodesOfInPlaceWorkersBeforeTest := inplace.ItShouldFindNodesOfInPlaceWorkers(s)
+	machinePodNamesBeforeTest := ItShouldFindAllMachinePodsBefore(s)
 
 	ItShouldAnnotateShoot(s, map[string]string{
 		v1beta1constants.GardenerOperation: v1beta1constants.OperationRotateCredentialsStartWithoutWorkersRollout,
@@ -154,8 +159,7 @@ func testCredentialRotationWithoutWorkersRollout(s *ShootContext, shootVerifiers
 		}).Should(Succeed())
 	}, SpecTimeout(time.Minute))
 
-	nodesOfInPlaceWorkersAfterTest := inplace.ItShouldFindNodesOfInPlaceWorkers(s)
-	Expect(nodesOfInPlaceWorkersBeforeTest.UnsortedList()).To(ConsistOf(nodesOfInPlaceWorkersAfterTest.UnsortedList()))
+	ItShouldCompareMachinePodNamesAfter(s, machinePodNamesBeforeTest)
 
 	It("Ensure all worker pools are marked as 'pending for roll out'", func() {
 		for _, worker := range s.Shoot.Spec.Provider.Workers {
@@ -207,8 +211,17 @@ func testCredentialRotationWithoutWorkersRollout(s *ShootContext, shootVerifiers
 		})).Should(Succeed())
 	}, SpecTimeout(time.Minute))
 
+	// In case of rotation without workers rollout, the in-place update status in only populated when the rollout for that worker pool is triggered
+	if inPlaceUpdate {
+		inplace.ItShouldVerifyInPlaceUpdateStart(s.GardenClient, s.Shoot, true, true)
+	}
+
 	ItShouldWaitForShootToBeReconciledAndHealthy(s)
 
+	if inPlaceUpdate {
+		inplace.ItShouldVerifyInPlaceUpdateCompletion(s.GardenClient, s.Shoot)
+	}
+
 	It("Credential rotation in status prepared", func() {
 		Expect(s.Shoot.Status.Credentials.Rotation.CertificateAuthorities.Phase).To(Equal(gardencorev1beta1.RotationPrepared))
 		Expect(s.Shoot.Status.Credentials.Rotation.ServiceAccountKey.Phase).To(Equal(gardencorev1beta1.RotationPrepared))
@@ -323,7 +336,7 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() {
 				// test rotation for every rotation type
 				testCredentialRotation(s, shootVerifiers, utilsVerifiers, v1beta1constants.OperationRotateCredentialsStart, v1beta1constants.OperationRotateCredentialsComplete, inPlaceUpdate)
 			} else {
-				testCredentialRotationWithoutWorkersRollout(s, shootVerifiers, utilsVerifiers)
+				testCredentialRotationWithoutWorkersRollout(s, shootVerifiers, utilsVerifiers, inPlaceUpdate)
 			}
 
 			if !v1beta1helper.IsWorkerless(s.Shoot) {
@@ -334,7 +347,6 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() {
 				if inPlaceUpdate {
 					nodesOfInPlaceWorkersAfterTest := inplace.ItShouldFindNodesOfInPlaceWorkers(s)
 					Expect(nodesOfInPlaceWorkersBeforeTest.UnsortedList()).To(ConsistOf(nodesOfInPlaceWorkersAfterTest.UnsortedList()))
-					inplace.ItShouldVerifyInPlaceUpdateCompletion(s.GardenClient, s.Shoot)
 				}
 			}
 
@@ -391,6 +403,38 @@ var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() {
 				test(s, true, false)
 			})
 
+			Context("without workers rollout, in-place update strategy", Label("without-workers-rollout", "in-place"), Ordered, func() {
+				var s *ShootContext
+				BeforeTestSetup(func() {
+					shoot := DefaultShoot("e2e-rot-nr-ip")
+
+					worker1 := DefaultWorker("auto", ptr.To(gardencorev1beta1.AutoInPlaceUpdate))
+					worker1.Minimum = 2
+					worker1.Maximum = 2
+					worker1.MaxUnavailable = ptr.To(intstr.FromInt(1))
+					worker1.MaxSurge = ptr.To(intstr.FromInt(0))
+
+					worker2 := DefaultWorker("manual", ptr.To(gardencorev1beta1.ManualInPlaceUpdate))
+
+					// Add a third worker pool when worker rollout should not be performed such that we can make proper
+					// assertions of the shoot status
+					worker3 := DefaultWorker("rolling", nil)
+
+					shoot.Spec.Provider.Workers = []gardencorev1beta1.Worker{worker1, worker2, worker3}
+
+					s = NewTestContext().ForShoot(shoot)
+				})
+
+				// Skip the test for IPv6 single-stack Shoot as it is extremely flaky (success rate < 30%).
+				//
+				// TODO(shafeeqes, acumino, ary1992): Debug and fix the flaky test for ipv6.
+				if os.Getenv("IPFAMILY") == "ipv6" {
+					s.Log.Info("Skip the flaky credentials rotation with in-place update strategy e2e test for ipv6")
+					return
+				}
+
+				test(s, true, true)
+			})
 		})
 
 		Context("Workerless Shoot", Label("workerless"), Ordered, func() {

test/e2e/gardener/shoot/shoot.go

@@ -13,6 +13,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -261,3 +262,44 @@ func ItShouldAnnotateShoot(s *ShootContext, annotations map[string]string) {
 		}).Should(Succeed())
 	}, SpecTimeout(time.Minute))
 }
+
+// ItShouldFindAllMachinePodsBefore finds all machine pods before running the required tests and returns their names.
+func ItShouldFindAllMachinePodsBefore(s *ShootContext) sets.Set[string] {
+	GinkgoHelper()
+
+	machinePodNamesBeforeTest := sets.New[string]()
+
+	It("Find all machine pods to ensure later that they weren't rolled out", func(ctx SpecContext) {
+		beforeStartMachinePodList := &corev1.PodList{}
+		Eventually(ctx, s.SeedKomega.List(beforeStartMachinePodList, client.InNamespace(s.Shoot.Status.TechnicalID), client.MatchingLabels{
+			"app":              "machine",
+			"machine-provider": "local",
+		})).Should(Succeed())
+
+		for _, item := range beforeStartMachinePodList.Items {
+			machinePodNamesBeforeTest.Insert(item.Name)
+		}
+	}, SpecTimeout(time.Minute))
+
+	return machinePodNamesBeforeTest
+}
+
+// ItShouldCompareMachinePodNamesAfter compares the machine pod names before and after running the required tests.
+func ItShouldCompareMachinePodNamesAfter(s *ShootContext, machinePodNamesBeforeTest sets.Set[string]) {
+	GinkgoHelper()
+
+	It("Compare machine pod names", func(ctx SpecContext) {
+		machinePodListAfterTest := &corev1.PodList{}
+		Eventually(ctx, s.SeedKomega.List(machinePodListAfterTest, client.InNamespace(s.Shoot.Status.TechnicalID), client.MatchingLabels{
+			"app":              "machine",
+			"machine-provider": "local",
+		})).Should(Succeed())
+
+		machinePodNamesAfterTest := sets.New[string]()
+		for _, item := range machinePodListAfterTest.Items {
+			machinePodNamesAfterTest.Insert(item.Name)
+		}
+
+		Expect(machinePodNamesBeforeTest.UnsortedList()).To(ConsistOf(machinePodNamesAfterTest.UnsortedList()))
+	}, SpecTimeout(time.Minute))
+}