Skip to content

Commit 946a319

Browse files
michaelhtmmichaelhtm
authored
Merge pull request #117 from awslabs/recovery
fix: update handlers when informer already exists and add recovery tests
2 parents fd38d89 + 86ddbc0 commit 946a319

File tree

2 files changed

+260
-0
lines changed

2 files changed

+260
-0
lines changed

‎internal/dynamiccontroller/dynamic_controller.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,9 @@ func (dc *DynamicController) StartServingGVK(ctx context.Context, gvr schema.Gro
427427

428428
_, exists := dc.informers.Load(gvr)
429429
if exists {
430+
// Even thought the informer is already registered, we should still
431+
// still update the handler, as it might have changed.
432+
dc.handlers.Store(gvr, handler)
430433
return nil
431434
}
432435

‎test/integration/suites/core/recover_test.go‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
// not use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
// express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
package core_test
15+
16+
import (
17+
"context"
18+
"fmt"
19+
"time"
20+
21+
. "github.com/onsi/ginkgo/v2"
22+
. "github.com/onsi/gomega"
23+
appsv1 "k8s.io/api/apps/v1"
24+
corev1 "k8s.io/api/core/v1"
25+
"k8s.io/apimachinery/pkg/api/errors"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
28+
"k8s.io/apimachinery/pkg/types"
29+
"k8s.io/apimachinery/pkg/util/rand"
30+
31+
krov1alpha1 "github.com/awslabs/kro/api/v1alpha1"
32+
"github.com/awslabs/kro/internal/testutil/generator"
33+
)
34+
35+
var _ = Describe("Recovery", func() {
36+
var (
37+
ctx context.Context
38+
namespace string
39+
)
40+
41+
BeforeEach(func() {
42+
ctx = context.Background()
43+
namespace = fmt.Sprintf("test-%s", rand.String(5))
44+
// Create namespace
45+
ns := &corev1.Namespace{
46+
ObjectMeta: metav1.ObjectMeta{
47+
Name: namespace,
48+
},
49+
}
50+
Expect(env.Client.Create(ctx, ns)).To(Succeed())
51+
})
52+
53+
It("should recover from invalid state and use latest valid configuration", func() {
54+
// Create initial valid ResourceGroup
55+
rg := generator.NewResourceGroup("test-recovery",
56+
generator.WithNamespace(namespace),
57+
generator.WithSchema(
58+
"TestRecovery", "v1alpha1",
59+
map[string]interface{}{
60+
"name": "string",
61+
"configKey": "string",
62+
},
63+
nil,
64+
),
65+
generator.WithResource("initialConfig", map[string]interface{}{
66+
"apiVersion": "v1",
67+
"kind": "ConfigMap",
68+
"metadata": map[string]interface{}{
69+
"name": "${schema.spec.name}",
70+
},
71+
"data": map[string]interface{}{
72+
"key": "${schema.spec.configKey}",
73+
"version": "initial",
74+
},
75+
}, nil, nil),
76+
)
77+
78+
// Create ResourceGroup
79+
Expect(env.Client.Create(ctx, rg)).To(Succeed())
80+
81+
// Verify initial ResourceGroup becomes active
82+
Eventually(func(g Gomega) {
83+
err := env.Client.Get(ctx, types.NamespacedName{
84+
Name: rg.Name,
85+
Namespace: namespace,
86+
}, rg)
87+
g.Expect(err).ToNot(HaveOccurred())
88+
g.Expect(rg.Status.State).To(Equal(krov1alpha1.ResourceGroupStateActive))
89+
}, 10*time.Second, time.Second).Should(Succeed())
90+
91+
// Update to invalid state with a cyclic dependency
92+
Eventually(func(g Gomega) {
93+
err := env.Client.Get(ctx, types.NamespacedName{
94+
Name: rg.Name,
95+
Namespace: namespace,
96+
}, rg)
97+
g.Expect(err).ToNot(HaveOccurred())
98+
99+
// Add resources with circular dependency
100+
rg.Spec.Resources = append(rg.Spec.Resources,
101+
&krov1alpha1.Resource{
102+
Name: "serviceA",
103+
Template: toRawExtension(map[string]interface{}{
104+
"apiVersion": "v1",
105+
"kind": "Service",
106+
"metadata": map[string]interface{}{
107+
"name": "${serviceB.metadata.name}",
108+
},
109+
}),
110+
},
111+
&krov1alpha1.Resource{
112+
Name: "serviceB",
113+
Template: toRawExtension(map[string]interface{}{
114+
"apiVersion": "v1",
115+
"kind": "Service",
116+
"metadata": map[string]interface{}{
117+
"name": "${serviceA.metadata.name}",
118+
},
119+
}),
120+
},
121+
)
122+
123+
err = env.Client.Update(ctx, rg)
124+
g.Expect(err).ToNot(HaveOccurred())
125+
}, 10*time.Second, time.Second).Should(Succeed())
126+
127+
// Verify ResourceGroup becomes inactive
128+
Eventually(func(g Gomega) {
129+
err := env.Client.Get(ctx, types.NamespacedName{
130+
Name: rg.Name,
131+
Namespace: namespace,
132+
}, rg)
133+
g.Expect(err).ToNot(HaveOccurred())
134+
g.Expect(rg.Status.State).To(Equal(krov1alpha1.ResourceGroupStateInactive))
135+
}, 10*time.Second, time.Second).Should(Succeed())
136+
137+
// Update to new valid state with different configuration
138+
Eventually(func(g Gomega) {
139+
err := env.Client.Get(ctx, types.NamespacedName{
140+
Name: rg.Name,
141+
Namespace: namespace,
142+
}, rg)
143+
g.Expect(err).ToNot(HaveOccurred())
144+
145+
// Replace with new valid resource
146+
rg.Spec.Resources = []*krov1alpha1.Resource{
147+
{
148+
Name: "itsapodnow",
149+
Template: toRawExtension(map[string]interface{}{
150+
"apiVersion": "apps/v1",
151+
"kind": "Deployment",
152+
"metadata": map[string]interface{}{
153+
"name": "${schema.spec.name}",
154+
},
155+
"spec": map[string]interface{}{
156+
"replicas": 1,
157+
"selector": map[string]interface{}{
158+
"matchLabels": map[string]interface{}{
159+
"app": "deployment",
160+
},
161+
},
162+
"template": map[string]interface{}{
163+
"metadata": map[string]interface{}{
164+
"labels": map[string]interface{}{
165+
"app": "deployment",
166+
},
167+
},
168+
"spec": map[string]interface{}{
169+
"containers": []interface{}{
170+
map[string]interface{}{
171+
"name": "${schema.spec.name}-deployment",
172+
"image": "nginx",
173+
"ports": []interface{}{
174+
map[string]interface{}{
175+
"containerPort": 777,
176+
},
177+
},
178+
},
179+
},
180+
},
181+
},
182+
},
183+
}),
184+
},
185+
}
186+
187+
err = env.Client.Update(ctx, rg)
188+
g.Expect(err).ToNot(HaveOccurred())
189+
}, 10*time.Second, time.Second).Should(Succeed())
190+
191+
// Verify ResourceGroup becomes active again
192+
Eventually(func(g Gomega) {
193+
err := env.Client.Get(ctx, types.NamespacedName{
194+
Name: rg.Name,
195+
Namespace: namespace,
196+
}, rg)
197+
g.Expect(err).ToNot(HaveOccurred())
198+
g.Expect(rg.Status.State).To(Equal(krov1alpha1.ResourceGroupStateActive))
199+
}, 10*time.Second, time.Second).Should(Succeed())
200+
201+
// Create instance
202+
name := "test-recovery"
203+
instance := &unstructured.Unstructured{
204+
Object: map[string]interface{}{
205+
"apiVersion": fmt.Sprintf("%s/%s", krov1alpha1.KroDomainName, "v1alpha1"),
206+
"kind": "TestRecovery",
207+
"metadata": map[string]interface{}{
208+
"name": name,
209+
"namespace": namespace,
210+
},
211+
"spec": map[string]interface{}{
212+
"name": name,
213+
"configKey": "testKey",
214+
},
215+
},
216+
}
217+
Expect(env.Client.Create(ctx, instance)).To(Succeed())
218+
219+
// Verify instance created Deployment with updated configuration
220+
Eventually(func(g Gomega) {
221+
deploy := &appsv1.Deployment{}
222+
err := env.Client.Get(ctx, types.NamespacedName{
223+
Name: name,
224+
Namespace: namespace,
225+
}, deploy)
226+
g.Expect(err).ToNot(HaveOccurred())
227+
g.Expect(deploy.Spec.Template.Spec.Containers[0].Image).To(Equal("nginx"))
228+
g.Expect(deploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort).To(Equal(int32(777)))
229+
230+
}, 20*time.Second, time.Second).Should(Succeed())
231+
232+
// Cleanup
233+
// Delete instance
234+
Expect(env.Client.Delete(ctx, instance)).To(Succeed())
235+
236+
// Verify instance is deleted
237+
Eventually(func() bool {
238+
err := env.Client.Get(ctx, types.NamespacedName{
239+
Name: name,
240+
Namespace: namespace,
241+
}, instance)
242+
return errors.IsNotFound(err)
243+
}, 20*time.Second, time.Second).Should(BeTrue())
244+
245+
// Delete ResourceGroup
246+
Expect(env.Client.Delete(ctx, rg)).To(Succeed())
247+
248+
// Verify ResourceGroup is deleted
249+
Eventually(func() bool {
250+
err := env.Client.Get(ctx, types.NamespacedName{
251+
Name: rg.Name,
252+
Namespace: namespace,
253+
}, &krov1alpha1.ResourceGroup{})
254+
return errors.IsNotFound(err)
255+
}, 20*time.Second, time.Second).Should(BeTrue())
256+
})
257+
})

0 commit comments

Comments
 (0)