From cf5bc34cf682b3025f68fd8dda02388ba2481a3c Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Fri, 20 Dec 2024 14:14:41 +0900 Subject: [PATCH 1/8] calculate cpu and mem changes from replica changes --- pkg/hpa/service.go | 27 ++++++++++++++-------- pkg/metrics/metrics.go | 50 ++++++++++++++++++++-------------------- pkg/tortoise/tortoise.go | 6 ----- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index 444f98da..068b09a4 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -446,16 +446,25 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet hpa.Spec.MinReplicas = &minToActuallyApply if tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff && recordMetrics { // We don't want to record applied* metric when UpdateMode is Off. - netChangeMaxReplicas := float64(hpa.Spec.MaxReplicas - recommendMax) - netChangeMinReplicas := float64(*hpa.Spec.MinReplicas) - float64(recommendMin) - if netChangeMaxReplicas > 0 || netChangeMinReplicas < 0 { - metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) - } - if netChangeMaxReplicas < 0 || netChangeMinReplicas > 0 { - metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) + netChangeMaxReplicas := float64(recommendMax - hpa.Spec.MaxReplicas) + netChangeMinReplicas := float64(recommendMin) - float64(*hpa.Spec.MinReplicas) + cpu := float64(0) + mem := float64(0) + if len(tortoise.Status.Conditions.ContainerResourceRequests) > 0 { + for _, requests := range tortoise.Status.Conditions.ContainerResourceRequests { + cpu += float64(requests.Resource.Cpu().Value()) + mem += float64(requests.Resource.Memory().Value()) + } } - metrics.NetHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Name).Set(netChangeMinReplicas) - metrics.NetHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Name).Set(netChangeMaxReplicas) + netChangeMaxReplicasCpu := netChangeMaxReplicas * cpu + netChangeMinReplicasCpu := netChangeMinReplicas * cpu + netChangeMinReplicasMem := netChangeMinReplicas * mem + netChangeMaxReplicasMem := netChangeMaxReplicas * mem + + metrics.NetHPAMinReplicasCPUCores.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMinReplicasCpu) + metrics.NetHPAMaxReplicasCPUCores.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMaxReplicasCpu) + metrics.NetHPAMinReplicasMemory.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMinReplicasMem) + metrics.NetHPAMaxReplicasMemory.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMaxReplicasMem) metrics.AppliedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas)) metrics.AppliedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas)) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index f29fa891..7a84e918 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -46,35 +46,35 @@ var ( Help: "memory request (byte) that tortoises actually applys", }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) - DecreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "decrease_apply_counter", - Help: "counter for number of resource decreases applied by tortoise", - }, []string{"tortoise_name", "namespace"}) - - IncreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "increase_apply_counter", - Help: "counter for number of resource increases applied by tortoise", - }, []string{"tortoise_name", "namespace"}) - - NetHPAMinReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "net_hpa_minreplicas", - Help: "net hpa minReplicas that tortoises actually applys to hpa", - }, []string{"tortoise_name", "namespace", "hpa_name", "kube_deployment"}) - - NetHPAMaxReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "net_hpa_maxreplicas", - Help: "net hpa maxReplicas that tortoises actually applys to hpa", - }, []string{"tortoise_name", "namespace", "hpa_name", "kube_deployment"}) + NetHPAMinReplicasCPUCores = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_hpa_minreplicas_cpu_cores", + Help: "net cpu cores changed by minReplicas that tortoises actually applys to hpa", + }, []string{"tortoise_name", "namespace", "hpa_name"}) + + NetHPAMinReplicasMemory = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_hpa_minreplicas_memory", + Help: "net memory changed by minReplicas that tortoises actually applys to hpa", + }, []string{"tortoise_name", "namespace", "hpa_name"}) + + NetHPAMaxReplicasCPUCores = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_hpa_maxreplicas_cpu_cores", + Help: "net cpu cores changed by maxReplicas that tortoises actually applys to hpa", + }, []string{"tortoise_name", "namespace", "hpa_name"}) + + NetHPAMaxReplicasMemory = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_hpa_maxreplicas_memory", + Help: "net memory changed by maxReplicas that tortoises actually applys to hpa", + }, []string{"tortoise_name", "namespace", "hpa_name"}) NetCPURequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "net_cpu_request", Help: "net cpu request (millicore) that tortoises actually applys", - }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) + }, []string{"tortoise_name", "namespace", "container_name", "kube_deployment", "controller_kind"}) NetMemoryRequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "net_memory_request", Help: "net memory request (byte) that tortoises actually applys", - }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) + }, []string{"tortoise_name", "namespace", "container_name", "kube_deployment", "controller_kind"}) ProposedHPATargetUtilization = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "proposed_hpa_utilization_target", @@ -117,10 +117,10 @@ func init() { AppliedHPAMinReplicas, AppliedCPURequest, AppliedMemoryRequest, - IncreaseApplyCounter, - DecreaseApplyCounter, - NetHPAMaxReplicas, - NetHPAMinReplicas, + NetHPAMinReplicasCPUCores, + NetHPAMinReplicasMemory, + NetHPAMaxReplicasCPUCores, + NetHPAMaxReplicasMemory, NetCPURequest, NetMemoryRequest, ProposedHPATargetUtilization, diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go index ef1e0490..c84fee46 100644 --- a/pkg/tortoise/tortoise.go +++ b/pkg/tortoise/tortoise.go @@ -778,12 +778,6 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T metrics.AppliedMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.Value())) metrics.NetMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(netChange)) } - if netChange > 0 { - metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) - } - if netChange < 0 { - metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) - } } } From e3547a69cfc75158e907d767dd8c2d225890a6a8 Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Fri, 20 Dec 2024 16:09:57 +0900 Subject: [PATCH 2/8] minor changes --- pkg/tortoise/tortoise.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go index c84fee46..c7b284f3 100644 --- a/pkg/tortoise/tortoise.go +++ b/pkg/tortoise/tortoise.go @@ -768,7 +768,7 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T // only record metrics once in every reconcile loop. for resourcename, value := range r.Resource { oldRequest := oldRequestMap[r.ContainerName][resourcename] - netChange := float64(oldRequest.MilliValue() - value.MilliValue()) + netChange := float64(oldRequest.MilliValue()-value.MilliValue()) * float64(replica) if resourcename == corev1.ResourceCPU { // We don't want to record applied* metric when UpdateMode is Off. metrics.AppliedCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.MilliValue())) @@ -776,7 +776,7 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T } if resourcename == corev1.ResourceMemory { metrics.AppliedMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.Value())) - metrics.NetMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(netChange)) + metrics.NetMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(netChange / float64(1000)) } } } From c076cc4df42e0601ccb7b8301387a0401497d53c Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Mon, 23 Dec 2024 18:26:24 +0900 Subject: [PATCH 3/8] minor changes --- pkg/hpa/service.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index 068b09a4..cc3e9f22 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -450,10 +450,14 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet netChangeMinReplicas := float64(recommendMin) - float64(*hpa.Spec.MinReplicas) cpu := float64(0) mem := float64(0) - if len(tortoise.Status.Conditions.ContainerResourceRequests) > 0 { - for _, requests := range tortoise.Status.Conditions.ContainerResourceRequests { - cpu += float64(requests.Resource.Cpu().Value()) - mem += float64(requests.Resource.Memory().Value()) + for _, r := range tortoise.Status.Conditions.ContainerResourceRequests { + for resourcename, value := range r.Resource { + if resourcename == corev1.ResourceCPU { + cpu += value.AsApproximateFloat64() + } + if resourcename == corev1.ResourceMemory { + cpu += value.AsApproximateFloat64() + } } } netChangeMaxReplicasCpu := netChangeMaxReplicas * cpu From 8802f3be145f4a643411b08c61fbacf14a193dcc Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Wed, 25 Dec 2024 12:47:54 +0900 Subject: [PATCH 4/8] fix --- .../after/tortoise.yaml | 10 +++++----- .../after/tortoise.yaml | 10 +++++----- .../after/tortoise.yaml | 10 +++++----- .../after/tortoise.yaml | 10 +++++----- .../after/tortoise.yaml | 10 +++++----- .../after/tortoise.yaml | 10 +++++----- .../after/tortoise.yaml | 10 +++++----- .../after/tortoise.yaml | 10 +++++----- internal/controller/tortoise_controller.go | 8 ++++---- pkg/hpa/service.go | 2 +- 10 files changed, 45 insertions(+), 45 deletions(-) diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml index 1b8df961..526ca739 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml @@ -70,15 +70,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/tortoise.yaml b/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/tortoise.yaml index 3be0827a..2e489027 100644 --- a/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/tortoise.yaml @@ -70,15 +70,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml index ae5d587b..7509fbeb 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml @@ -70,15 +70,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/tortoise.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/tortoise.yaml index d03f3ee5..7f3efce9 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/tortoise.yaml @@ -70,15 +70,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml index 3be0827a..2e489027 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml @@ -70,15 +70,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/tortoise.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/tortoise.yaml index d7d01941..7f6f5a8d 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/tortoise.yaml @@ -47,15 +47,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/tortoise.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/tortoise.yaml index a8e2611a..05f63755 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/tortoise.yaml @@ -53,15 +53,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/tortoise.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/tortoise.yaml index 9a5cd103..d87c9651 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/tortoise.yaml @@ -47,15 +47,15 @@ status: type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: HPA target utilization is updated - reason: HPATargetUtilizationUpdated + message: The recommendation is provided status: "True" - type: HPATargetUtilizationUpdated + type: VerticalRecommendationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated status: "True" - type: VerticalRecommendationUpdated + type: HPATargetUtilizationUpdated - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" status: "False" diff --git a/internal/controller/tortoise_controller.go b/internal/controller/tortoise_controller.go index f08fe9f6..b8bb8c96 100644 --- a/internal/controller/tortoise_controller.go +++ b/internal/controller/tortoise_controller.go @@ -258,15 +258,15 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ return ctrl.Result{RequeueAfter: r.Interval}, nil } - _, tortoise, err = r.HpaService.UpdateHPAFromTortoiseRecommendation(ctx, tortoise, now) + tortoise, err = r.TortoiseService.UpdateResourceRequest(ctx, tortoise, currentDesiredReplicaNum, now) if err != nil { - logger.Error(err, "update HPA based on the recommendation in tortoise", "tortoise", req.NamespacedName) + logger.Error(err, "update VPA based on the recommendation in tortoise", "tortoise", req.NamespacedName) return ctrl.Result{}, err } - tortoise, err = r.TortoiseService.UpdateResourceRequest(ctx, tortoise, currentDesiredReplicaNum, now) + _, tortoise, err = r.HpaService.UpdateHPAFromTortoiseRecommendation(ctx, tortoise, now) if err != nil { - logger.Error(err, "update VPA based on the recommendation in tortoise", "tortoise", req.NamespacedName) + logger.Error(err, "update HPA based on the recommendation in tortoise", "tortoise", req.NamespacedName) return ctrl.Result{}, err } diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index cc3e9f22..a53e2d42 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -456,7 +456,7 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet cpu += value.AsApproximateFloat64() } if resourcename == corev1.ResourceMemory { - cpu += value.AsApproximateFloat64() + mem += value.AsApproximateFloat64() } } } From d634c7918628cbb0c79229f673d5b7c69b037af8 Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Wed, 25 Dec 2024 16:25:01 +0900 Subject: [PATCH 5/8] fix --- pkg/hpa/service.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index a53e2d42..f08f0e01 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -405,6 +405,7 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet recommendMax = c.maximumMaxReplica } + oldMax := hpa.Spec.MaxReplicas hpa.Spec.MaxReplicas = recommendMax recommendMin, err := GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MinReplicas, now) @@ -443,11 +444,12 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet minToActuallyApply = recommendMin } + oldMin := *hpa.Spec.MinReplicas hpa.Spec.MinReplicas = &minToActuallyApply if tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff && recordMetrics { // We don't want to record applied* metric when UpdateMode is Off. - netChangeMaxReplicas := float64(recommendMax - hpa.Spec.MaxReplicas) - netChangeMinReplicas := float64(recommendMin) - float64(*hpa.Spec.MinReplicas) + netChangeMaxReplicas := float64(recommendMax - oldMax) + netChangeMinReplicas := float64(recommendMin - oldMin) cpu := float64(0) mem := float64(0) for _, r := range tortoise.Status.Conditions.ContainerResourceRequests { From 39821229454dddf0e3638404a7c6911cf2277ab3 Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Fri, 17 Jan 2025 10:32:15 +0900 Subject: [PATCH 6/8] add alpha description to new metrics --- pkg/metrics/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 7a84e918..6005b256 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -69,12 +69,12 @@ var ( NetCPURequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "net_cpu_request", Help: "net cpu request (millicore) that tortoises actually applys", - }, []string{"tortoise_name", "namespace", "container_name", "kube_deployment", "controller_kind"}) + }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) NetMemoryRequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "net_memory_request", Help: "net memory request (byte) that tortoises actually applys", - }, []string{"tortoise_name", "namespace", "container_name", "kube_deployment", "controller_kind"}) + }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) ProposedHPATargetUtilization = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "proposed_hpa_utilization_target", From ad261c76064a8909d61c55b7a4343a8921dbc2d7 Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Mon, 31 Mar 2025 16:42:53 +0900 Subject: [PATCH 7/8] remove net hpa metrics --- pkg/hpa/service.go | 25 ------------------------- pkg/metrics/metrics.go | 24 ------------------------ 2 files changed, 49 deletions(-) diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index f08f0e01..e0f9fd3d 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -405,7 +405,6 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet recommendMax = c.maximumMaxReplica } - oldMax := hpa.Spec.MaxReplicas hpa.Spec.MaxReplicas = recommendMax recommendMin, err := GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MinReplicas, now) @@ -444,33 +443,9 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet minToActuallyApply = recommendMin } - oldMin := *hpa.Spec.MinReplicas hpa.Spec.MinReplicas = &minToActuallyApply if tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff && recordMetrics { // We don't want to record applied* metric when UpdateMode is Off. - netChangeMaxReplicas := float64(recommendMax - oldMax) - netChangeMinReplicas := float64(recommendMin - oldMin) - cpu := float64(0) - mem := float64(0) - for _, r := range tortoise.Status.Conditions.ContainerResourceRequests { - for resourcename, value := range r.Resource { - if resourcename == corev1.ResourceCPU { - cpu += value.AsApproximateFloat64() - } - if resourcename == corev1.ResourceMemory { - mem += value.AsApproximateFloat64() - } - } - } - netChangeMaxReplicasCpu := netChangeMaxReplicas * cpu - netChangeMinReplicasCpu := netChangeMinReplicas * cpu - netChangeMinReplicasMem := netChangeMinReplicas * mem - netChangeMaxReplicasMem := netChangeMaxReplicas * mem - - metrics.NetHPAMinReplicasCPUCores.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMinReplicasCpu) - metrics.NetHPAMaxReplicasCPUCores.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMaxReplicasCpu) - metrics.NetHPAMinReplicasMemory.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMinReplicasMem) - metrics.NetHPAMaxReplicasMemory.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMaxReplicasMem) metrics.AppliedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas)) metrics.AppliedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas)) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 6005b256..24ce8578 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -46,26 +46,6 @@ var ( Help: "memory request (byte) that tortoises actually applys", }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) - NetHPAMinReplicasCPUCores = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "net_hpa_minreplicas_cpu_cores", - Help: "net cpu cores changed by minReplicas that tortoises actually applys to hpa", - }, []string{"tortoise_name", "namespace", "hpa_name"}) - - NetHPAMinReplicasMemory = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "net_hpa_minreplicas_memory", - Help: "net memory changed by minReplicas that tortoises actually applys to hpa", - }, []string{"tortoise_name", "namespace", "hpa_name"}) - - NetHPAMaxReplicasCPUCores = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "net_hpa_maxreplicas_cpu_cores", - Help: "net cpu cores changed by maxReplicas that tortoises actually applys to hpa", - }, []string{"tortoise_name", "namespace", "hpa_name"}) - - NetHPAMaxReplicasMemory = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "net_hpa_maxreplicas_memory", - Help: "net memory changed by maxReplicas that tortoises actually applys to hpa", - }, []string{"tortoise_name", "namespace", "hpa_name"}) - NetCPURequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "net_cpu_request", Help: "net cpu request (millicore) that tortoises actually applys", @@ -117,10 +97,6 @@ func init() { AppliedHPAMinReplicas, AppliedCPURequest, AppliedMemoryRequest, - NetHPAMinReplicasCPUCores, - NetHPAMinReplicasMemory, - NetHPAMaxReplicasCPUCores, - NetHPAMaxReplicasMemory, NetCPURequest, NetMemoryRequest, ProposedHPATargetUtilization, From 220d4348f2b978d2971e7694fe9011dcacbd6db7 Mon Sep 17 00:00:00 2001 From: randytqwjp Date: Tue, 1 Apr 2025 04:22:38 +0900 Subject: [PATCH 8/8] fix net change in request --- pkg/tortoise/tortoise.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go index c7b284f3..f54bd70d 100644 --- a/pkg/tortoise/tortoise.go +++ b/pkg/tortoise/tortoise.go @@ -768,7 +768,7 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T // only record metrics once in every reconcile loop. for resourcename, value := range r.Resource { oldRequest := oldRequestMap[r.ContainerName][resourcename] - netChange := float64(oldRequest.MilliValue()-value.MilliValue()) * float64(replica) + netChange := float64(value.MilliValue()-oldRequest.MilliValue()) * float64(replica) if resourcename == corev1.ResourceCPU { // We don't want to record applied* metric when UpdateMode is Off. metrics.AppliedCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.MilliValue()))