Merge pull request #1 from kabisa/kubernetes-alert-updates

obeleh · web-flow · commit 1dedbb9e0327 · 2021-04-07T11:21:18.000+02:00
updating alerts to v0.5 and replicaset unavailable fix for replicaset…
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+.terraform
diff --git a/daemonset-incomplete-variables.tf b/daemonset-incomplete-variables.tf
@@ -4,8 +4,8 @@ variable "daemonset_incomplete_enabled" {
 }
 
 variable "daemonset_incomplete_critical" {
-  type    = number
-  default = 0
+  type        = number
+  default     = 0
   description = "alert is raised when (desired - running) > daemonset_incomplete_critical"
 }
 
@@ -38,3 +38,28 @@ variable "daemonset_incomplete_alerting_enabled" {
   type    = bool
   default = true
 }
+
+variable "daemonset_incomplete_no_data_timeframe" {
+  type    = number
+  default = null
+}
+
+variable "daemonset_incomplete_notify_no_data" {
+  type    = bool
+  default = false
+}
+
+variable "daemonset_incomplete_ok_threshold" {
+  type    = number
+  default = null
+}
+
+variable "daemonset_incomplete_name_prefix" {
+  type    = string
+  default = ""
+}
+
+variable "daemonset_incomplete_name_suffix" {
+  type    = string
+  default = ""
+}
diff --git a/daemonset-incomplete.tf b/daemonset-incomplete.tf
@@ -6,7 +6,7 @@ locals {
 }
 
 module "daemonset_incomplete" {
-  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"
+  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"
 
   name  = "Daemonset Incomplete"
   query = "min(${var.daemonset_incomplete_evaluation_period}):max:kubernetes_state.daemonset.scheduled{${local.daemonset_incomplete_filter}} by {daemonset,cluster_name} - min:kubernetes_state.daemonset.ready{${local.daemonset_incomplete_filter}} by {daemonset,cluster_name} > 0"
@@ -29,4 +29,5 @@ module "daemonset_incomplete" {
   require_full_window = true
 
   critical_threshold = var.daemonset_incomplete_critical
+  locked             = var.locked
 }
diff --git a/hpa-status-variables.tf b/hpa-status-variables.tf
@@ -32,3 +32,28 @@ variable "hpa_status_alerting_enabled" {
   type    = bool
   default = true
 }
+
+variable "hpa_status_no_data_timeframe" {
+  type    = number
+  default = null
+}
+
+variable "hpa_status_notify_no_data" {
+  type    = bool
+  default = false
+}
+
+variable "hpa_status_ok_threshold" {
+  type    = number
+  default = null
+}
+
+variable "hpa_status_name_prefix" {
+  type    = string
+  default = ""
+}
+
+variable "hpa_status_name_suffix" {
+  type    = string
+  default = ""
+}
diff --git a/hpa-status.tf b/hpa-status.tf
@@ -1,13 +1,13 @@
 locals {
   hpa_status_filter = (
     var.hpa_status_filter_override != ""
-      ? var.hpa_status_filter_override
-      : var.filter_str
+    ? var.hpa_status_filter_override
+    : var.filter_str
   )
 }
 
 module "hpa_status" {
-  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"
+  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"
 
   name  = "HPA Status not OK"
   query = "avg(${var.hpa_status_evaluation_period}):avg:kubernetes_state.hpa.condition{${local.hpa_status_filter}} by {hpa,kube_namespace,status,condition} < 1"
@@ -30,4 +30,5 @@ module "hpa_status" {
   require_full_window = false
 
   critical_threshold = 1
+  locked             = var.locked
 }
diff --git a/node-memory-used-percent-variables.tf b/node-memory-used-percent-variables.tf
@@ -44,3 +44,28 @@ variable "node_memory_used_percent_alerting_enabled" {
   type    = bool
   default = true
 }
+
+variable "node_memory_used_percent_no_data_timeframe" {
+  type    = number
+  default = null
+}
+
+variable "node_memory_used_percent_notify_no_data" {
+  type    = bool
+  default = false
+}
+
+variable "node_memory_used_percent_ok_threshold" {
+  type    = number
+  default = null
+}
+
+variable "node_memory_used_percent_name_prefix" {
+  type    = string
+  default = ""
+}
+
+variable "node_memory_used_percent_name_suffix" {
+  type    = string
+  default = ""
+}
diff --git a/node-memory-used-percent.tf b/node-memory-used-percent.tf
@@ -6,7 +6,7 @@ locals {
 }
 
 module "node_memory_used_percent" {
-  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"
+  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"
 
   name  = "Memory Used Percent"
   query = "avg(${var.node_memory_used_percent_evaluation_period}):( 100 * max:kubernetes.memory.usage{${local.node_memory_used_percent_filter}} by {host,cluster_name} ) / max:kubernetes.memory.capacity{${local.node_memory_used_percent_filter}} by {host,cluster_name} > ${var.node_memory_used_percent_critical}"
@@ -30,4 +30,5 @@ module "node_memory_used_percent" {
 
   critical_threshold = var.node_memory_used_percent_critical
   warning_threshold  = var.node_memory_used_percent_warning
+  locked             = var.locked
 }
diff --git a/node-status-variables.tf b/node-status-variables.tf
@@ -32,3 +32,28 @@ variable "node_status_alerting_enabled" {
   type    = bool
   default = true
 }
+
+variable "node_status_no_data_timeframe" {
+  type    = number
+  default = null
+}
+
+variable "node_status_notify_no_data" {
+  type    = bool
+  default = false
+}
+
+variable "node_status_ok_threshold" {
+  type    = number
+  default = null
+}
+
+variable "node_status_name_prefix" {
+  type    = string
+  default = ""
+}
+
+variable "node_status_name_suffix" {
+  type    = string
+  default = ""
+}
diff --git a/node-status.tf b/node-status.tf
@@ -6,7 +6,7 @@ locals {
 }
 
 module "node_status" {
-  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"
+  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"
 
   name  = "Node Status not OK"
   query = "avg(${var.node_status_evaluation_period}):avg:kubernetes_state.node.status{${local.node_status_filter}} by {cluster_name,node} < 1"
@@ -29,4 +29,5 @@ module "node_status" {
   require_full_window = false
 
   critical_threshold = 1
+  locked             = var.locked
 }
diff --git a/pod-ready-variables.tf b/pod-ready-variables.tf
@@ -32,3 +32,28 @@ variable "pod_ready_alerting_enabled" {
   type    = bool
   default = true
 }
+
+variable "pod_ready_no_data_timeframe" {
+  type    = number
+  default = null
+}
+
+variable "pod_ready_notify_no_data" {
+  type    = bool
+  default = false
+}
+
+variable "pod_ready_ok_threshold" {
+  type    = number
+  default = null
+}
+
+variable "pod_ready_name_prefix" {
+  type    = string
+  default = ""
+}
+
+variable "pod_ready_name_suffix" {
+  type    = string
+  default = ""
+}
diff --git a/pod-ready.tf b/pod-ready.tf
@@ -6,7 +6,7 @@ locals {
 }
 
 module "pod_ready" {
-  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"
+  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"
 
   name  = "Pod status not ready"
   query = "min(${var.pod_ready_evaluation_period}):sum:kubernetes_state.pod.count{${local.pod_ready_filter}} by {cluster_name,namespace} - sum:kubernetes_state.pod.ready{${local.pod_ready_filter}} by {cluster_name,namespace} > 0"
@@ -29,4 +29,5 @@ module "pod_ready" {
   require_full_window = true
 
   critical_threshold = 0
+  locked             = var.locked
 }
diff --git a/provider.tf b/provider.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     datadog = {
-      source = "DataDog/datadog"
+      source  = "DataDog/datadog"
       version = ">= 2.21"
     }
   }
diff --git a/replicaset-incomplete-variables.tf b/replicaset-incomplete-variables.tf
@@ -4,8 +4,8 @@ variable "replicaset_incomplete_enabled" {
 }
 
 variable "replicaset_incomplete_critical" {
-  type    = number
-  default = 0
+  type        = number
+  default     = 0
   description = "alert is raised when (desired - running) > replicaset_incomplete_critical"
 }
 
@@ -38,3 +38,28 @@ variable "replicaset_incomplete_alerting_enabled" {
   type    = bool
   default = true
 }
+
+variable "replicaset_incomplete_no_data_timeframe" {
+  type    = number
+  default = null
+}
+
+variable "replicaset_incomplete_notify_no_data" {
+  type    = bool
+  default = false
+}
+
+variable "replicaset_incomplete_ok_threshold" {
+  type    = number
+  default = null
+}
+
+variable "replicaset_incomplete_name_prefix" {
+  type    = string
+  default = ""
+}
+
+variable "replicaset_incomplete_name_suffix" {
+  type    = string
+  default = ""
+}
diff --git a/replicaset-incomplete.tf b/replicaset-incomplete.tf
@@ -6,7 +6,7 @@ locals {
 }
 
 module "replicaset_incomplete" {
-  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"
+  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"
 
   name  = "Replicaset Incomplete"
   query = "min(${var.replicaset_incomplete_evaluation_period}):max:kubernetes_state.replicaset.replicas_desired{${local.replicaset_incomplete_filter}} by {kube_replica_set,cluster_name} - min:kubernetes_state.replicaset.replicas_ready{${local.replicaset_incomplete_filter}} by {kube_replica_set,cluster_name} > ${var.replicaset_incomplete_critical}"
@@ -29,4 +29,5 @@ module "replicaset_incomplete" {
   require_full_window = true
 
   critical_threshold = var.replicaset_incomplete_critical
+  locked             = var.locked
 }
diff --git a/replicaset-unavailable-variables.tf b/replicaset-unavailable-variables.tf
@@ -4,8 +4,8 @@ variable "replicaset_unavailable_enabled" {
 }
 
 variable "replicaset_unavailable_critical" {
-  type    = number
-  default = 0
+  type        = number
+  default     = 0
   description = "alert is raised when (desired - running) == 0"
 }
 
@@ -26,7 +26,7 @@ variable "replicaset_unavailable_note" {
 
 variable "replicaset_unavailable_docs" {
   type    = string
-  default = "In kubernetes a Replicaset is responsible for making sure a specific number of pods runs. An example for a reason when that's not is the case, is when the image cannot be pulled, the pod fails to initialize or no resources are available on the cluster\nThis alert is raised when running == 0 and desired > 0"
+  default = "In kubernetes a Replicaset is responsible for making sure a specific number of pods runs. An example for a reason when that's not is the case, is when the image cannot be pulled, the pod fails to initialize or no resources are available on the cluster\nThis alert is raised when running == 0 and desired > 1"
 }
 
 variable "replicaset_unavailable_filter_override" {
@@ -38,3 +38,28 @@ variable "replicaset_unavailable_alerting_enabled" {
   type    = bool
   default = true
 }
+
+variable "replicaset_unavailable_no_data_timeframe" {
+  type    = number
+  default = null
+}
+
+variable "replicaset_unavailable_notify_no_data" {
+  type    = bool
+  default = false
+}
+
+variable "replicaset_unavailable_ok_threshold" {
+  type    = number
+  default = null
+}
+
+variable "replicaset_unavailable_name_prefix" {
+  type    = string
+  default = ""
+}
+
+variable "replicaset_unavailable_name_suffix" {
+  type    = string
+  default = ""
+}
diff --git a/replicaset-unavailable.tf b/replicaset-unavailable.tf
@@ -3,14 +3,16 @@ locals {
     var.replicaset_unavailable_filter_override,
     var.filter_str
   )
+  rs_pods_ready   = "min:kubernetes_state.replicaset.replicas_ready{${local.replicaset_unavailable_filter}} by {kube_replica_set,cluster_name}"
+  rs_pods_desired = "min:kubernetes_state.replicaset.replicas_desired{${local.replicaset_unavailable_filter}} by {kube_replica_set,cluster_name}"
 }
 
 module "replicaset_unavailable" {
-  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"
+  source = "git@github.com:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"
 
-  name  = "Replicaset Unavailable"
-  # This (ab)uses a division by zero to
-  query = "max(${var.replicaset_unavailable_evaluation_period}):min:kubernetes_state.replicaset.replicas_ready{${local.replicaset_unavailable_filter}} by {kube_replica_set,cluster_name} / min:kubernetes_state.replicaset.replicas_desired{${local.replicaset_unavailable_filter}} by {kube_replica_set,cluster_name} <= 0"
+  name = "Replicaset Unavailable"
+  # This (ab)uses a division by zero to make sure we don't get alerts when nr of desired pods < 2
+  query = "max(${var.replicaset_unavailable_evaluation_period}):( ${local.rs_pods_ready} ) / ${local.rs_pods_desired} / ( ${local.rs_pods_desired} - 1 ) <= 0"
 
   enabled          = var.replicaset_unavailable_enabled
   alerting_enabled = var.replicaset_unavailable_alerting_enabled
@@ -30,4 +32,5 @@ module "replicaset_unavailable" {
   require_full_window = true
 
   critical_threshold = 0
+  locked             = var.locked
 }
diff --git a/variables.tf b/variables.tf
@@ -23,3 +23,8 @@ variable "additional_tags" {
 variable "filter_str" {
   type = string
 }
+
+variable "locked" {
+  type    = bool
+  default = true
+}

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ locals {`
`6`	`6`	`}`
`7`	`7`
`8`	`8`	`module "daemonset_incomplete" {`
`9`		`- source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"`
	`9`	`+ source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"`
`10`	`10`
`11`	`11`	`name = "Daemonset Incomplete"`
`12`	`12`	`query = "min(${var.daemonset_incomplete_evaluation_period}):max:kubernetes_state.daemonset.scheduled{${local.daemonset_incomplete_filter}} by {daemonset,cluster_name} - min:kubernetes_state.daemonset.ready{${local.daemonset_incomplete_filter}} by {daemonset,cluster_name} > 0"`
`@@ -29,4 +29,5 @@ module "daemonset_incomplete" {`
`29`	`29`	`require_full_window = true`
`30`	`30`
`31`	`31`	`critical_threshold = var.daemonset_incomplete_critical`
	`32`	`+ locked = var.locked`
`32`	`33`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,13 +1,13 @@`
`1`	`1`	`locals {`
`2`	`2`	`hpa_status_filter = (`
`3`	`3`	`var.hpa_status_filter_override != ""`
`4`		`- ? var.hpa_status_filter_override`
`5`		`- : var.filter_str`
	`4`	`+ ? var.hpa_status_filter_override`
	`5`	`+ : var.filter_str`
`6`	`6`	`)`
`7`	`7`	`}`
`8`	`8`
`9`	`9`	`module "hpa_status" {`
`10`		`- source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"`
	`10`	`+ source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"`
`11`	`11`
`12`	`12`	`name = "HPA Status not OK"`
`13`	`13`	`query = "avg(${var.hpa_status_evaluation_period}):avg:kubernetes_state.hpa.condition{${local.hpa_status_filter}} by {hpa,kube_namespace,status,condition} < 1"`
`@@ -30,4 +30,5 @@ module "hpa_status" {`
`30`	`30`	`require_full_window = false`
`31`	`31`
`32`	`32`	`critical_threshold = 1`
	`33`	`+ locked = var.locked`
`33`	`34`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ locals {`
`6`	`6`	`}`
`7`	`7`
`8`	`8`	`module "node_memory_used_percent" {`
`9`		`- source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"`
	`9`	`+ source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"`
`10`	`10`
`11`	`11`	`name = "Memory Used Percent"`
`12`	`12`	`query = "avg(${var.node_memory_used_percent_evaluation_period}):( 100 * max:kubernetes.memory.usage{${local.node_memory_used_percent_filter}} by {host,cluster_name} ) / max:kubernetes.memory.capacity{${local.node_memory_used_percent_filter}} by {host,cluster_name} > ${var.node_memory_used_percent_critical}"`
`@@ -30,4 +30,5 @@ module "node_memory_used_percent" {`
`30`	`30`
`31`	`31`	`critical_threshold = var.node_memory_used_percent_critical`
`32`	`32`	`warning_threshold = var.node_memory_used_percent_warning`
	`33`	`+ locked = var.locked`
`33`	`34`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ locals {`
`6`	`6`	`}`
`7`	`7`
`8`	`8`	`module "node_status" {`
`9`		`- source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.2"`
	`9`	`+ source = "[email protected]:kabisa/terraform-datadog-generic-monitor.git?ref=0.5"`
`10`	`10`
`11`	`11`	`name = "Node Status not OK"`
`12`	`12`	`query = "avg(${var.node_status_evaluation_period}):avg:kubernetes_state.node.status{${local.node_status_filter}} by {cluster_name,node} < 1"`
`@@ -29,4 +29,5 @@ module "node_status" {`
`29`	`29`	`require_full_window = false`
`30`	`30`
`31`	`31`	`critical_threshold = 1`
	`32`	`+ locked = var.locked`
`32`	`33`	`}`