Skip to content

Commit 384f05c

Browse files
gardener-ci-robotvpnachevialidzhikov
authored and
ske-prow[bot]
committed
[release-v1.130] Deploy earlier the ClusterRoleBindings granting access for Admin/ViewerKubeconfig credentials (gardener#13298)
* Deploy earlier the ClusterRoleBindings granting access for Admin/ViewerKubeconfig credentials * Set the keep-object annotation to let the old MR stop tracking the resources * Migrate CRBs on gardenlet migration * Address review feedback Co-authored-by: Ismail Alidzhikov <[email protected]> --------- Co-authored-by: vpnachev <[email protected]> Co-authored-by: Ismail Alidzhikov <[email protected]> (cherry picked from commit 3a947d2)
1 parent 85b64a3 commit 384f05c

File tree

7 files changed

+342
-175
lines changed

7 files changed

+342
-175
lines changed

‎charts/gardener/gardenlet/templates/clusterrole-gardenlet.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,12 @@ rules:
305305
- watch
306306
- patch
307307
- update
308+
- apiGroups:
309+
- resources.gardener.cloud
310+
resources:
311+
- managedresources/status # TODO(vpnachev): Remove patch managedresources/status permissions after v1.133.0 is released.
312+
verbs:
313+
- patch
308314
- apiGroups:
309315
- networking.k8s.io
310316
resources:

‎charts/gardener/gardenlet/test/test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,12 @@ func getGardenletClusterRole(labels map[string]string) *rbacv1.ClusterRole {
295295
Resources: []string{"managedresources"},
296296
Verbs: []string{"create", "delete", "deletecollection", "get", "list", "watch", "patch", "update"},
297297
},
298+
{
299+
// TODO(vpnachev): Drop patch managedresources/status permissions after v1.133.0 is released.
300+
APIGroups: []string{"resources.gardener.cloud"},
301+
Resources: []string{"managedresources/status"},
302+
Verbs: []string{"patch"},
303+
},
298304
{
299305
APIGroups: []string{"networking.k8s.io"},
300306
Resources: []string{"networkpolicies"},

‎cmd/gardenlet/app/migration.go‎

Lines changed: 143 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,152 @@ package app
66

77
import (
88
"context"
9+
"fmt"
10+
"slices"
911

1012
"github.com/go-logr/logr"
13+
corev1 "k8s.io/api/core/v1"
14+
apierrors "k8s.io/apimachinery/pkg/api/errors"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
17+
v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
18+
resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
19+
"github.com/gardener/gardener/pkg/client/kubernetes"
20+
gardeneraccess "github.com/gardener/gardener/pkg/component/gardener/access"
21+
shootsystem "github.com/gardener/gardener/pkg/component/shoot/system"
22+
"github.com/gardener/gardener/pkg/utils/flow"
23+
"github.com/gardener/gardener/pkg/utils/managedresources"
1124
)
1225

13-
func (g *garden) runMigrations(_ context.Context, _ logr.Logger) error {
26+
func (g *garden) runMigrations(ctx context.Context, log logr.Logger) error {
27+
log.Info("Migrating ClusterRoleBindings for shoot/adminkubeconfig and shoot/viewerkubeconfig")
28+
if err := migrateAdminViewerKubeconfigClusterRoleBindings(ctx, log, g.mgr.GetClient()); err != nil {
29+
return fmt.Errorf("failed migrating ClusterRoleBindings for shoot/adminkubeconfig and shoot/viewerkubeconfig: %w", err)
30+
}
31+
1432
return nil
1533
}
34+
35+
// migrateAdminViewerKubeconfigClusterRoleBindings moves the ClusterRoleBindings granting access to the
36+
// shoot/adminkubeconfig and shoot/viewerkubeconfig subresources from the shoot-core-system managed resource to the
37+
// shoot-core-gardeneraccess managed resource.
38+
// TODO(vpnachev): Remove this after v1.133.0 has been released.
39+
func migrateAdminViewerKubeconfigClusterRoleBindings(ctx context.Context, log logr.Logger, seedClient client.Client) error {
40+
namespaceList := &corev1.NamespaceList{}
41+
if err := seedClient.List(ctx, namespaceList, client.MatchingLabels{v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot}); err != nil {
42+
return fmt.Errorf("failed listing namespaces: %w", err)
43+
}
44+
45+
var (
46+
tasks []flow.TaskFn
47+
crbs = []string{v1beta1constants.ShootProjectAdminsGroupName, v1beta1constants.ShootProjectViewersGroupName, v1beta1constants.ShootSystemAdminsGroupName, v1beta1constants.ShootSystemViewersGroupName}
48+
)
49+
50+
for _, namespace := range namespaceList.Items {
51+
if namespace.DeletionTimestamp != nil || namespace.Status.Phase == corev1.NamespaceTerminating {
52+
continue
53+
}
54+
55+
tasks = append(tasks, func(ctx context.Context) error {
56+
var (
57+
shootSystemKey = client.ObjectKey{Namespace: namespace.Name, Name: shootsystem.ManagedResourceName}
58+
shootSystemManagedResource = &resourcesv1alpha1.ManagedResource{}
59+
60+
gardenerAccessKey = client.ObjectKey{Namespace: namespace.Name, Name: gardeneraccess.ManagedResourceName}
61+
gardenerAccessManagedResource = &resourcesv1alpha1.ManagedResource{}
62+
)
63+
64+
// Get the shoot-core-system managed resource and check if it is already migrated.
65+
if err := seedClient.Get(ctx, shootSystemKey, shootSystemManagedResource); err != nil {
66+
if apierrors.IsNotFound(err) {
67+
log.Info("Managed resource not found, skipping migration", "managedResource", shootSystemKey)
68+
return nil
69+
}
70+
return fmt.Errorf("failed to get ManagedResource %q: %w", shootSystemKey, err)
71+
}
72+
73+
if shootSystemManagedResource.DeletionTimestamp != nil {
74+
log.Info("Managed resource is in deletion, skipping migration", "managedResource", shootSystemKey)
75+
return nil
76+
}
77+
78+
shootSystemObjects, err := managedresources.GetObjects(ctx, seedClient, shootSystemManagedResource.Namespace, shootSystemManagedResource.Name)
79+
if err != nil {
80+
if apierrors.IsNotFound(err) {
81+
log.Info("Managed resource secret not found, skipping migration", "managedResource", shootSystemKey)
82+
return nil
83+
}
84+
return fmt.Errorf("failed to get objects for ManagedResource %q: %w", shootSystemKey, err)
85+
}
86+
87+
oldShootSystemObjectsCount := len(shootSystemObjects)
88+
shootSystemObjects = slices.DeleteFunc(shootSystemObjects, func(obj client.Object) bool {
89+
return slices.Contains(crbs, obj.GetName())
90+
})
91+
92+
if oldShootSystemObjectsCount == len(shootSystemObjects) {
93+
log.Info("ClusterRoleBindings for shoot/adminkubeconfig and shoot/viewerkubeconfig have already been migrated, skipping migration", "managedResource", shootSystemKey)
94+
return nil
95+
}
96+
97+
// Move the ClusterRoleBindings to the shoot-core-gardeneraccess managed resource firstly.
98+
if err := seedClient.Get(ctx, gardenerAccessKey, gardenerAccessManagedResource); err != nil {
99+
if apierrors.IsNotFound(err) {
100+
log.Info("Managed resource not found, skipping migration", "managedResource", gardenerAccessKey)
101+
return nil
102+
}
103+
return fmt.Errorf("failed to get ManagedResource %q: %w", gardenerAccessKey, err)
104+
}
105+
106+
if gardenerAccessManagedResource.DeletionTimestamp != nil {
107+
log.Info("Managed resource is in deletion, skipping migration", "managedResource", gardenerAccessKey)
108+
return nil
109+
}
110+
111+
gardenerAccessObjects, err := managedresources.GetObjects(ctx, seedClient, gardenerAccessManagedResource.Namespace, gardenerAccessManagedResource.Name)
112+
if err != nil {
113+
if apierrors.IsNotFound(err) {
114+
log.Info("Managed resource secret not found, skipping migration", "managedResource", gardenerAccessKey)
115+
return nil
116+
}
117+
return fmt.Errorf("failed to get objects for ManagedResource %q: %w", gardenerAccessKey, err)
118+
}
119+
120+
gardenerAccessObjects = append(gardenerAccessObjects, gardeneraccess.ShootAccessClusterRoleBindings()...)
121+
gardenerAccessRegistry := managedresources.NewRegistry(kubernetes.ShootScheme, kubernetes.ShootCodec, kubernetes.ShootSerializer)
122+
gardenerAccessResources, err := gardenerAccessRegistry.AddAllAndSerialize(gardenerAccessObjects...)
123+
if err != nil {
124+
return fmt.Errorf("failed serializing objects for ManagedResource %q: %w", gardenerAccessKey, err)
125+
}
126+
127+
if err := managedresources.CreateForShoot(ctx, seedClient, gardenerAccessManagedResource.Namespace, gardenerAccessManagedResource.Name, managedresources.LabelValueGardener, false, gardenerAccessResources); err != nil {
128+
return fmt.Errorf("failed updating ManagedResource %q: %w", gardenerAccessKey, err)
129+
}
130+
131+
// Remove the ClusterRoleBindings from the shoot-core-system managed resource.
132+
log.Info("Updating ManagedResource status to remove migrated ClusterRoleBindings", "managedResource", shootSystemKey)
133+
patch := client.MergeFrom(shootSystemManagedResource.DeepCopy())
134+
shootSystemManagedResource.Status.Resources = slices.DeleteFunc(shootSystemManagedResource.Status.Resources, func(objRef resourcesv1alpha1.ObjectReference) bool {
135+
return objRef.APIVersion == "rbac.authorization.k8s.io/v1" && objRef.Kind == "ClusterRoleBinding" && slices.Contains(crbs, objRef.Name)
136+
})
137+
138+
if err := seedClient.Status().Patch(ctx, shootSystemManagedResource, patch); err != nil {
139+
return fmt.Errorf("failed updating status of ManagedResource %q: %w", shootSystemKey, err)
140+
}
141+
142+
log.Info("Cleaning ClusterRoleBindings for shoot/adminkubeconfig and shoot/viewerkubeconfig access in managed resource", "managedResource", shootSystemKey)
143+
shootSystemRegistry := managedresources.NewRegistry(kubernetes.ShootScheme, kubernetes.ShootCodec, kubernetes.ShootSerializer)
144+
shootSystemResources, err := shootSystemRegistry.AddAllAndSerialize(shootSystemObjects...)
145+
if err != nil {
146+
return fmt.Errorf("failed serializing objects for ManagedResource %q: %w", shootSystemKey, err)
147+
}
148+
if err := managedresources.CreateForShoot(ctx, seedClient, shootSystemManagedResource.Namespace, shootSystemManagedResource.Name, managedresources.LabelValueGardener, false, shootSystemResources); err != nil {
149+
return fmt.Errorf("failed updating ManagedResource %q: %w", shootSystemKey, err)
150+
}
151+
152+
return nil
153+
})
154+
}
155+
156+
return flow.Parallel(tasks...)(ctx)
157+
}

‎pkg/component/gardener/access/access.go‎

Lines changed: 99 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"sigs.k8s.io/controller-runtime/pkg/client"
1717

1818
v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
19+
resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
1920
"github.com/gardener/gardener/pkg/client/kubernetes"
2021
"github.com/gardener/gardener/pkg/component"
2122
gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
@@ -136,7 +137,7 @@ func (g *gardener) computeResourcesData(serviceAccountNames ...string) (map[stri
136137
var (
137138
registry = managedresources.NewRegistry(kubernetes.ShootScheme, kubernetes.ShootCodec, kubernetes.ShootSerializer)
138139

139-
clusterRoleBinding = &rbacv1.ClusterRoleBinding{
140+
gardenerSystemClusterRoleBinding = &rbacv1.ClusterRoleBinding{
140141
ObjectMeta: metav1.ObjectMeta{
141142
Name: "gardener.cloud:system:gardener",
142143
},
@@ -149,12 +150,107 @@ func (g *gardener) computeResourcesData(serviceAccountNames ...string) (map[stri
149150
)
150151

151152
for _, name := range serviceAccountNames {
152-
clusterRoleBinding.Subjects = append(clusterRoleBinding.Subjects, rbacv1.Subject{
153+
gardenerSystemClusterRoleBinding.Subjects = append(gardenerSystemClusterRoleBinding.Subjects, rbacv1.Subject{
153154
Kind: rbacv1.ServiceAccountKind,
154155
Name: name,
155156
Namespace: metav1.NamespaceSystem,
156157
})
157158
}
158159

159-
return registry.AddAllAndSerialize(clusterRoleBinding)
160+
resources := append(adminClusterRoleBindings(), viewerClusterRoleBindings()...)
161+
resources = append(resources, gardenerSystemClusterRoleBinding)
162+
163+
return registry.AddAllAndSerialize(resources...)
164+
}
165+
166+
// adminClusterRoleBindings returns the ClusterRoleBindings granting access to credentials obtained via the shoot/adminkubeconfig subresource.
167+
func adminClusterRoleBindings() []client.Object {
168+
return []client.Object{
169+
&rbacv1.ClusterRoleBinding{
170+
ObjectMeta: metav1.ObjectMeta{
171+
Name: v1beta1constants.ShootSystemAdminsGroupName,
172+
Annotations: map[string]string{
173+
resourcesv1alpha1.DeleteOnInvalidUpdate: "true",
174+
},
175+
},
176+
RoleRef: rbacv1.RoleRef{
177+
APIGroup: rbacv1.GroupName,
178+
Kind: "ClusterRole",
179+
Name: "cluster-admin",
180+
},
181+
Subjects: []rbacv1.Subject{{
182+
APIGroup: rbacv1.GroupName,
183+
Kind: rbacv1.GroupKind,
184+
Name: v1beta1constants.ShootSystemAdminsGroupName,
185+
}},
186+
},
187+
&rbacv1.ClusterRoleBinding{
188+
ObjectMeta: metav1.ObjectMeta{
189+
Name: v1beta1constants.ShootProjectAdminsGroupName,
190+
Annotations: map[string]string{
191+
resourcesv1alpha1.DeleteOnInvalidUpdate: "true",
192+
},
193+
},
194+
RoleRef: rbacv1.RoleRef{
195+
APIGroup: rbacv1.GroupName,
196+
Kind: "ClusterRole",
197+
Name: "cluster-admin",
198+
},
199+
Subjects: []rbacv1.Subject{{
200+
APIGroup: rbacv1.GroupName,
201+
Kind: rbacv1.GroupKind,
202+
Name: v1beta1constants.ShootProjectAdminsGroupName,
203+
}},
204+
},
205+
}
206+
}
207+
208+
// viewerClusterRoleBindings returns the ClusterRoleBindings granting access to credentials obtained via the shoot/viewerkubeconfig subresource.
209+
func viewerClusterRoleBindings() []client.Object {
210+
return []client.Object{
211+
&rbacv1.ClusterRoleBinding{
212+
ObjectMeta: metav1.ObjectMeta{
213+
Name: v1beta1constants.ShootSystemViewersGroupName,
214+
Annotations: map[string]string{
215+
resourcesv1alpha1.DeleteOnInvalidUpdate: "true",
216+
},
217+
},
218+
RoleRef: rbacv1.RoleRef{
219+
APIGroup: rbacv1.GroupName,
220+
Kind: "ClusterRole",
221+
Name: v1beta1constants.ShootReadOnlyClusterRoleName,
222+
},
223+
Subjects: []rbacv1.Subject{{
224+
APIGroup: rbacv1.GroupName,
225+
Kind: rbacv1.GroupKind,
226+
Name: v1beta1constants.ShootSystemViewersGroupName,
227+
}},
228+
},
229+
&rbacv1.ClusterRoleBinding{
230+
ObjectMeta: metav1.ObjectMeta{
231+
Name: v1beta1constants.ShootProjectViewersGroupName,
232+
Annotations: map[string]string{
233+
resourcesv1alpha1.DeleteOnInvalidUpdate: "true",
234+
},
235+
},
236+
RoleRef: rbacv1.RoleRef{
237+
APIGroup: rbacv1.GroupName,
238+
Kind: "ClusterRole",
239+
Name: v1beta1constants.ShootReadOnlyClusterRoleName,
240+
},
241+
Subjects: []rbacv1.Subject{{
242+
APIGroup: rbacv1.GroupName,
243+
Kind: rbacv1.GroupKind,
244+
Name: v1beta1constants.ShootProjectViewersGroupName,
245+
}},
246+
},
247+
}
248+
}
249+
250+
// ShootAccessClusterRoleBindings returns all ClusterRoleBindings granting access to credentials obtained via the shoot/adminkubeconfig and shoot/viewerkubeconfig subresources.
251+
//
252+
// Deprecated: The function is just temporary exported for migration purposes. It will be removed after v1.133.0 is released.
253+
func ShootAccessClusterRoleBindings() []client.Object {
254+
// TODO(vpnachev): Remove this function after v1.133.0 has been released.
255+
return append(adminClusterRoleBindings(), viewerClusterRoleBindings()...)
160256
}

0 commit comments

Comments
 (0)