fix: ensure conversion webhook CA bundles are injected during upgrades (#7772)

* fix: ensure conversion webhook CA bundles are injected during upgrades

During upgrades with pre-existing LQ/CQ, the API server fails conversion
with 'x509: certificate signed by unknown authority' because cert-controller
doesn't inject CA bundles into CRD conversion webhook configurations.

This fix manually injects CA bundles into kueue.x-k8s.io CRDs, registers
webhooks before manager start so conversion endpoints are ready when caches
sync.

Fixes #7344

Signed-off-by: Sohan Kunkerkar <[email protected]>

* test: add e2e coverage for upgrade with webhook conversion

Signed-off-by: Sohan Kunkerkar <[email protected]>

---------

Signed-off-by: Sohan Kunkerkar <[email protected]>

Author: sohankunkerkar

Makefile-test.mk

@@ -46,6 +46,7 @@ E2E_K8S_FULL_VERSION := $(or $(E2E_K8S_FULL_VERSION),$(E2E_K8S_VERSION).0)
 E2E_KIND_VERSION ?= kindest/node:v$(E2E_K8S_FULL_VERSION)
 E2E_RUN_ONLY_ENV ?= false
 E2E_USE_HELM ?= false
+KUEUE_UPGRADE_FROM_VERSION ?= v0.14.4
 
 # For local testing, we should allow user to use different kind cluster name
 # Default will delete default kind cluster
@@ -129,6 +130,12 @@ test-e2e-customconfigs-helm: test-e2e-customconfigs
 .PHONY: test-e2e-certmanager
 test-e2e-certmanager: setup-e2e-env run-test-e2e-certmanager-$(E2E_KIND_VERSION:kindest/node:v%=%)
 
+.PHONY: test-e2e-upgrade
+test-e2e-upgrade: setup-e2e-env run-test-e2e-upgrade-$(E2E_KIND_VERSION:kindest/node:v%=%)
+
+.PHONY: test-e2e-certmanager-upgrade
+test-e2e-certmanager-upgrade: setup-e2e-env run-test-e2e-certmanager-upgrade-$(E2E_KIND_VERSION:kindest/node:v%=%)
+
 run-test-e2e-singlecluster-%: K8S_VERSION = $(@:run-test-e2e-singlecluster-%=%)
 run-test-e2e-singlecluster-%:
 	@echo Running e2e for k8s ${K8S_VERSION}
@@ -202,6 +209,29 @@ run-test-e2e-certmanager-%:
 		E2E_USE_HELM=$(E2E_USE_HELM) \
 		./hack/e2e-test.sh
 
+run-test-e2e-upgrade-%: K8S_VERSION = $(@:run-test-e2e-upgrade-%=%)
+run-test-e2e-upgrade-%:
+	@echo Running upgrade e2e for k8s ${K8S_VERSION}
+	E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) \
+		ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" \
+		KIND_CLUSTER_FILE="kind-cluster.yaml" E2E_TARGET_FOLDER="upgrade" \
+		KUEUE_UPGRADE_FROM_VERSION=$(KUEUE_UPGRADE_FROM_VERSION) \
+		TEST_LOG_LEVEL=$(TEST_LOG_LEVEL) \
+		E2E_RUN_ONLY_ENV=$(E2E_RUN_ONLY_ENV) \
+		./hack/e2e-test.sh
+
+run-test-e2e-certmanager-upgrade-%: K8S_VERSION = $(@:run-test-e2e-certmanager-upgrade-%=%)
+run-test-e2e-certmanager-upgrade-%:
+	@echo Running upgrade e2e for k8s ${K8S_VERSION}
+	E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) \
+		ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" \
+		KIND_CLUSTER_FILE="kind-cluster.yaml" E2E_TARGET_FOLDER="upgrade" \
+		KUEUE_UPGRADE_FROM_VERSION=$(KUEUE_UPGRADE_FROM_VERSION) \
+		CERTMANAGER_VERSION=$(CERTMANAGER_VERSION) \
+		TEST_LOG_LEVEL=$(TEST_LOG_LEVEL) \
+		E2E_RUN_ONLY_ENV=$(E2E_RUN_ONLY_ENV) \
+		./hack/e2e-test.sh
+
 SCALABILITY_RUNNER := $(BIN_DIR)/performance-scheduler-runner
 .PHONY: performance-scheduler-runner
 performance-scheduler-runner:

cmd/kueue/main.go

@@ -205,6 +205,16 @@ func main() {
 		kubeConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(*cfg.ClientConnection.QPS, int(*cfg.ClientConnection.Burst))
 	}
 	setupLog.V(2).Info("K8S Client", "qps", *cfg.ClientConnection.QPS, "burst", *cfg.ClientConnection.Burst)
+
+	// Bootstrap certificates before creating the main manager
+	// This ensures certs are ready and CA bundles are injected into conversion CRDs
+	if cfg.InternalCertManagement != nil && *cfg.InternalCertManagement.Enable {
+		if err := cert.BootstrapCerts(kubeConfig, cfg); err != nil {
+			setupLog.Error(err, "Unable to bootstrap certificates")
+			os.Exit(1)
+		}
+	}
+
 	mgr, err := ctrl.NewManager(kubeConfig, options)
 	if err != nil {
 		setupLog.Error(err, "Unable to start manager")
@@ -266,15 +276,15 @@ func main() {
 		os.Exit(1)
 	}
 
-	// Cert won't be ready until manager starts, so start a goroutine here which
-	// will block until the cert is ready before setting up the controllers.
-	// Controllers who register after manager starts will start directly.
-	go func() {
-		if err := setupControllers(ctx, mgr, cCache, queues, certsReady, &cfg, serverVersionFetcher); err != nil {
-			setupLog.Error(err, "Unable to setup controllers")
-			os.Exit(1)
-		}
-	}()
+	if err := setupControllers(ctx, mgr, cCache, queues, &cfg, serverVersionFetcher); err != nil {
+		setupLog.Error(err, "Unable to setup controllers")
+		os.Exit(1)
+	}
+
+	if failedWebhook, err := webhooks.Setup(mgr); err != nil {
+		setupLog.Error(err, "Unable to create webhook", "webhook", failedWebhook)
+		os.Exit(1)
+	}
 
 	go queues.CleanUpOnContext(ctx)
 	go cCache.CleanUpOnContext(ctx)
@@ -331,11 +341,7 @@ func setupIndexes(ctx context.Context, mgr ctrl.Manager, cfg *configapi.Configur
 	return jobframework.SetupIndexes(ctx, mgr.GetFieldIndexer(), opts...)
 }
 
-func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *schdcache.Cache, queues *qcache.Manager, certsReady chan struct{}, cfg *configapi.Configuration, serverVersionFetcher *kubeversion.ServerVersionFetcher) error {
-	// The controllers won't work until the webhooks are operating, and the webhook won't work until the
-	// certs are all in place.
-	cert.WaitForCertsReady(setupLog, certsReady)
-
+func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *schdcache.Cache, queues *qcache.Manager, cfg *configapi.Configuration, serverVersionFetcher *kubeversion.ServerVersionFetcher) error {
 	if failedCtrl, err := core.SetupControllers(mgr, queues, cCache, cfg); err != nil {
 		return fmt.Errorf("unable to create controller %s: %w", failedCtrl, err)
 	}
@@ -401,10 +407,6 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *schdcache.C
 		}
 	}
 
-	if failedWebhook, err := webhooks.Setup(mgr); err != nil {
-		return fmt.Errorf("unable to create webhook %s: %w", failedWebhook, err)
-	}
-
 	opts := []jobframework.Option{
 		jobframework.WithManageJobsWithoutQueueName(cfg.ManageJobsWithoutQueueName),
 		jobframework.WithWaitForPodsReady(cfg.WaitForPodsReady),

config/components/crd/patches/cainjection_in_localqueues.yaml

@@ -0,0 +1,7 @@
+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: localqueues.kueue.x-k8s.io

hack/e2e-common.sh

@@ -72,6 +72,10 @@ if [[ -n "${CERTMANAGER_VERSION:-}" ]]; then
     export CERTMANAGER_MANIFEST="https://github.com/cert-manager/cert-manager/releases/download/${CERTMANAGER_VERSION}/cert-manager.yaml"
 fi
 
+if [[ -n "${KUEUE_UPGRADE_FROM_VERSION:-}" ]]; then
+    export KUEUE_OLD_VERSION_MANIFEST="https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_UPGRADE_FROM_VERSION}/manifests.yaml"
+fi
+
 # agnhost image to use for testing.
 E2E_TEST_AGNHOST_IMAGE_OLD_WITH_SHA=registry.k8s.io/e2e-test-images/agnhost:2.52@sha256:b173c7d0ffe3d805d49f4dfe48375169b7b8d2e1feb81783efd61eb9d08042e6
 export E2E_TEST_AGNHOST_IMAGE_OLD=${E2E_TEST_AGNHOST_IMAGE_OLD_WITH_SHA%%@*}
@@ -141,13 +145,21 @@ function prepare_docker_images {
     if [[ -n ${LEADERWORKERSET_VERSION:-} ]]; then
         docker pull "${LEADERWORKERSET_IMAGE}"
     fi
+    if [[ -n ${KUEUE_UPGRADE_FROM_VERSION:-} ]]; then
+        local current_image="${IMAGE_TAG%:*}:${KUEUE_UPGRADE_FROM_VERSION}"
+        docker pull "${current_image}"
+    fi
 }
 
 # $1 cluster
 function cluster_kind_load {
     cluster_kind_load_image "$1" "${E2E_TEST_AGNHOST_IMAGE_OLD}"
     cluster_kind_load_image "$1" "${E2E_TEST_AGNHOST_IMAGE}"
     cluster_kind_load_image "$1" "$IMAGE_TAG"
+    if [[ -n ${KUEUE_UPGRADE_FROM_VERSION:-} ]]; then
+        local old_image="${IMAGE_TAG%:*}:${KUEUE_UPGRADE_FROM_VERSION}"
+        cluster_kind_load_image "$1" "${old_image}"
+    fi
 }
 
 # $1 cluster
@@ -218,6 +230,7 @@ function deploy_with_certmanager() {
         cd "${ROOT_DIR}/config/components/crd" || exit
         $KUSTOMIZE edit add patch --path "patches/cainjection_in_clusterqueues.yaml"
         $KUSTOMIZE edit add patch --path "patches/cainjection_in_cohorts.yaml"
+        $KUSTOMIZE edit add patch --path "patches/cainjection_in_localqueues.yaml"
         $KUSTOMIZE edit add patch --path "patches/cainjection_in_resourceflavors.yaml"
         $KUSTOMIZE edit add patch --path "patches/cainjection_in_workloads.yaml"
     )
@@ -240,6 +253,12 @@ function deploy_with_certmanager() {
 
 # $1 kubeconfig
 function cluster_kueue_deploy {
+    # Handle upgrade test mode
+    if [[ -n ${KUEUE_UPGRADE_FROM_VERSION:-} ]]; then
+        upgrade_test_flow "$1"
+        return
+    fi
+    # Normal deployment flows
     if [[ -n ${CERTMANAGER_VERSION:-} ]]; then
         kubectl -n cert-manager wait --for condition=ready pod \
             -l app.kubernetes.io/instance=cert-manager \
@@ -274,8 +293,7 @@ function helm_install {
 function build_and_apply_kueue_manifests {
     local build_output
     build_output=$($KUSTOMIZE build "$2")
-    # shellcheck disable=SC2001 # bash parameter substitution does not work on macOS
-    build_output=$(echo "$build_output" | sed "s/kueue-system/$KUEUE_NAMESPACE/g")
+    build_output=${build_output//kueue-system/$KUEUE_NAMESPACE}
     echo "$build_output" | kubectl apply --kubeconfig="$1" --server-side -f -
 }
 
@@ -405,3 +423,79 @@ EOF
         --user="$kind_name"
     fi
 }
+
+# Upgrade test flow: install old version, create resources, upgrade to current
+# $1 kubeconfig
+function upgrade_test_flow {
+    local old_version="${KUEUE_UPGRADE_FROM_VERSION}"
+    local old_image="${IMAGE_TAG%:*}:${old_version}"
+
+    echo "Upgrade Test: $old_version -> current"
+    echo "Old image: $old_image"
+    echo "New image: $IMAGE_TAG"
+    
+    # Step 1: Install old version
+    echo "Installing $old_version..."
+    echo "  Manifest URL: ${KUEUE_OLD_VERSION_MANIFEST}"
+    echo "  Downloading and modifying manifests..."
+    
+    # Download and modify manifests inline
+    curl -sL "${KUEUE_OLD_VERSION_MANIFEST}" | \
+      sed "s|registry.k8s.io/kueue/kueue:${old_version}|${old_image}|g" | \
+      sed 's|imagePullPolicy: Always|imagePullPolicy: IfNotPresent|g' | \
+      kubectl apply --server-side -f -
+
+    kubectl wait --for=condition=available --timeout=180s deployment/kueue-controller-manager -n kueue-system
+    echo "✓ $old_version ready"
+    
+    # Step 2: Create test resources
+    echo "Creating test resources..."
+    kubectl apply --kubeconfig="$1" -f - <<EOF
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ResourceFlavor
+metadata:
+  name: upgrade-test-flavor
+---
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ClusterQueue
+metadata:
+  name: upgrade-test-cq
+spec:
+  namespaceSelector: {}
+  resourceGroups:
+  - coveredResources: ["cpu", "memory"]
+    flavors:
+    - name: upgrade-test-flavor
+      resources:
+      - name: "cpu"
+        nominalQuota: 10
+      - name: "memory"
+        nominalQuota: 10Gi
+---
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: LocalQueue
+metadata:
+  name: upgrade-test-lq
+  namespace: default
+spec:
+  clusterQueue: upgrade-test-cq
+EOF
+    echo "✓ Resources created"
+    
+    # Step 3: Upgrade to current (rolling update)
+    echo "Upgrading to current..."
+    
+    # Apply upgrade - rolling update will replace pods
+    (
+        set_managers_image
+        trap restore_managers_image EXIT
+        
+        local build_output
+        build_output=$($KUSTOMIZE build "${ROOT_DIR}/test/e2e/config/default")
+        build_output=${build_output//kueue-system/$KUEUE_NAMESPACE}
+        echo "$build_output" | kubectl apply --kubeconfig="$1" --server-side --force-conflicts -f -
+    )
+    
+    echo "✓ Upgrade complete (rolling update in progress)"
+    echo "========================================="
+}