diff --git a/images/calico-go-build/Dockerfile b/images/calico-go-build/Dockerfile
index 4497cf18..b3b042f9 100644
--- a/images/calico-go-build/Dockerfile
+++ b/images/calico-go-build/Dockerfile
@@ -7,7 +7,7 @@ FROM registry.access.redhat.com/ubi8/ubi:latest AS ubi
 ARG TARGETARCH
 
 ARG CONTAINERREGISTRY_VERSION=v0.20.7
-ARG CONTROLLER_TOOLS_VERSION=v0.18.0
+ARG CONTROLLER_TOOLS_VERSION=v0.20.0
 ARG GO_LINT_VERSION=v2.6.2
 ARG MOCKERY_VERSION=3.5.5
 ARG PROTOC_VERSION=33.1
@@ -158,13 +158,8 @@ RUN set -eux; \
 # Install Go utilities
 
 # controller-gen is used for generating CRD files.
-COPY patches/controller-gen-Support-Calico-NumOrString-types.patch /tmp/controller-tools/calico.patch
-
 RUN set -eux; \
-    curl -sfL https://github.com/kubernetes-sigs/controller-tools/archive/refs/tags/${CONTROLLER_TOOLS_VERSION}.tar.gz | tar xz --strip-components 1 -C /tmp/controller-tools; \
-    cd /tmp/controller-tools && patch -p1 < calico.patch && CGO_ENABLED=0 go build -o /usr/local/bin/controller-gen -v -buildvcs=false \
-        -ldflags "-X sigs.k8s.io/controller-tools/pkg/version.version=${CONTROLLER_TOOLS_VERSION} -s -w" ./cmd/controller-gen; \
-    rm -fr /tmp/controller-tools
+    curl -sfL https://github.com/kubernetes-sigs/controller-tools/releases/download/${CONTROLLER_TOOLS_VERSION}/controller-gen-linux-${TARGETARCH} -o /usr/local/bin/controller-gen && chmod +x /usr/local/bin/controller-gen
 
 # crane is needed for our release targets to copy images from the dev registries to the release registries.
 RUN set -eux; \
diff --git a/images/calico-go-build/patches/controller-gen-Support-Calico-NumOrString-types.patch b/images/calico-go-build/patches/controller-gen-Support-Calico-NumOrString-types.patch
deleted file mode 100644
index f237c409..00000000
--- a/images/calico-go-build/patches/controller-gen-Support-Calico-NumOrString-types.patch
+++ /dev/null
@@ -1,65 +0,0 @@
-From d3db9cd382359bdb1afa94f40e29ad9bebdeaf61 Mon Sep 17 00:00:00 2001
-From: Jiawei Huang <jiawei@tigera.io>
-Date: Fri, 6 Dec 2024 15:35:46 -0800
-Subject: [PATCH] Support Calico NumOrString types
-
----
- pkg/crd/known_types.go | 35 +++++++++++++++++++++++++++++++++++
- 1 file changed, 35 insertions(+)
-
-diff --git a/pkg/crd/known_types.go b/pkg/crd/known_types.go
-index ab939328..8f9fa67d 100644
---- a/pkg/crd/known_types.go
-+++ b/pkg/crd/known_types.go
-@@ -22,6 +22,35 @@ import (
- 	"sigs.k8s.io/controller-tools/pkg/loader"
- )
- 
-+// Custom logic for numOrString types.
-+var numOrString = func(p *Parser, pkg *loader.Package) {
-+	p.Schemata[TypeIdent{Name: "NumOrString", Package: pkg}] = apiext.JSONSchemaProps{
-+		XIntOrString: true,
-+		AnyOf: []apiext.JSONSchemaProps{
-+			{Type: "integer"},
-+			{Type: "string"},
-+		},
-+		Pattern: "^.*",
-+	}
-+	p.Schemata[TypeIdent{Name: "Protocol", Package: pkg}] = apiext.JSONSchemaProps{
-+		XIntOrString: true,
-+		AnyOf: []apiext.JSONSchemaProps{
-+			{Type: "integer"},
-+			{Type: "string"},
-+		},
-+		Pattern: "^.*",
-+	}
-+	p.Schemata[TypeIdent{Name: "Port", Package: pkg}] = apiext.JSONSchemaProps{
-+		XIntOrString: true,
-+		AnyOf: []apiext.JSONSchemaProps{
-+			{Type: "integer"},
-+			{Type: "string"},
-+		},
-+		Pattern: "^.*",
-+	}
-+	p.AddPackage(pkg) // get the rest of the types
-+}
-+
- // KnownPackages overrides types in some comment packages that have custom validation
- // but don't have validation markers on them (since they're from core Kubernetes).
- var KnownPackages = map[string]PackageOverride{
-@@ -50,6 +79,12 @@ var KnownPackages = map[string]PackageOverride{
- 		p.AddPackage(pkg) // get the rest of the types
- 	},
- 
-+	// numorstring could come from different places. It was moved to the api repository
-+	// around the time of Calico v3.20.
-+	"github.com/projectcalico/libcalico-go/lib/numorstring": numOrString,
-+	"github.com/projectcalico/api/pkg/lib/numorstring":      numOrString,
-+	"github.com/tigera/api/pkg/lib/numorstring":             numOrString,
-+
- 	"k8s.io/apimachinery/pkg/api/resource": func(p *Parser, pkg *loader.Package) {
- 		p.Schemata[TypeIdent{Name: "Quantity", Package: pkg}] = apiext.JSONSchemaProps{
- 			// TODO(directxman12): regexp validation for this (or get kube to support it as a format value)
--- 
-2.47.1
-