feat: Update BigQuery workflow failure alert display name and notification rate limit, and add a new alert for successful BigQuery workflow completion.

Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>

Author: max-ostapenko

infra/monitoring.tf

@@ -62,7 +62,7 @@ EOF
 
 resource "google_monitoring_alert_policy" "dataform_workflow" {
   combiner              = "OR"
-  display_name          = "Dataform Workflow Invocation Failed"
+  display_name          = "BigQuery Workflow Failed"
   enabled               = true
   notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"]
   project               = var.project
@@ -74,7 +74,7 @@ resource "google_monitoring_alert_policy" "dataform_workflow" {
   alert_strategy {
     notification_prompts = ["OPENED"]
     notification_rate_limit {
-      period = "3600s"
+      period = "21600s"
     }
     auto_close = "604800s"
   }
@@ -91,3 +91,34 @@ EOF
     }
   }
 }
+
+resource "google_monitoring_alert_policy" "dataform_workflow_complete" {
+  combiner              = "OR"
+  display_name          = "BigQuery Workflow Complete (CrUX or crawl)"
+  enabled               = true
+  notification_channels = ["projects/${var.project}/notificationChannels/5647028675917298338"]
+  project               = var.project
+  user_labels           = {}
+  alert_strategy {
+    notification_prompts = ["OPENED"]
+    notification_rate_limit {
+      period = "1800s"
+    }
+    auto_close = "1800s"
+  }
+  conditions {
+    display_name = "Log match condition"
+    condition_matched_log {
+      filter           = <<EOF
+resource.type="dataform.googleapis.com/Repository"
+resource.labels.repository_id="crawl-data"
+jsonPayload.@type="type.googleapis.com/google.cloud.dataform.logging.v1.WorkflowInvocationCompletionLogEntry"
+jsonPayload.terminalState="SUCCEEDED"
+EOF
+      label_extractors = {}
+    }
+  }
+  documentation {
+    content = "See details here: https://console.cloud.google.com/bigquery/dataform/locations/us-central1/repositories/crawl-data/details/workflows\n\nCrUX Firestore exports may still require up to 1 hour to finish."
+  }
+}