feat: disable write back informer by default (#15079)

Signed-off-by: Eduardo Rodrigues <[email protected]>

Author: eduardodbr

.features/pending/disable-write-back-informer.md

@@ -0,0 +1,7 @@
+Description: Disable write back informer by default
+Author: [Eduardo Rodrigues](https://github.com/eduardodbr)
+Component: General
+Issues: 12352
+
+Update the controller’s default behavior to disable the write-back informer. We’ve seen several cases of unexpected behavior that appear to be caused by the write-back mechanism, and Kubernetes docs recommend avoiding writes to the informer store. Although turning it off may increase the frequency of 409 Conflict errors, it should help reduce unpredictable controller behavior.
+

docs/environment-variables.md

@@ -32,7 +32,7 @@ This document outlines environment variables that can be used to customize behav
 | `EXPRESSION_TEMPLATES`                   | `bool`              | `true`                                                                                      | Escape hatch to disable expression templates.                                                                                                                                                                                                                            |
 | `EVENT_AGGREGATION_WITH_ANNOTATIONS`     | `bool`              | `false`                                                                                     | Whether event annotations will be used when aggregating events.                                                                                                                                                                                                          |
 | `GZIP_IMPLEMENTATION`                    | `string`            | `PGZip`                                                                                     | The implementation of compression/decompression. Currently only "`PGZip`" and "`GZip`" are supported.                                                                                                                                                                    |
-| `INFORMER_WRITE_BACK`                    | `bool`              | `true`                                                                                      | Whether to write back to informer instead of catching up.                                                                                                                                                                                                                |
+| `INFORMER_WRITE_BACK`                    | `bool`              | `false`                                                                                      | Whether to write back to informer instead of catching up.                                                                                                                                                                                                                |
 | `HEALTHZ_AGE`                            | `time.Duration`     | `5m`                                                                                        | How old a un-reconciled workflow is to report unhealthy.                                                                                                                                                                                                                 |
 | `INDEX_WORKFLOW_SEMAPHORE_KEYS`          | `bool`              | `true`                                                                                      | Whether or not to index semaphores.                                                                                                                                                                                                                                      |
 | `LEADER_ELECTION_IDENTITY`               | `string`            | Controller's `metadata.name`                                                                | The ID used for workflow controllers to elect a leader.                                                                                                                                                                                                                  |

workflow/controller/operator.go

@@ -798,14 +798,15 @@ func (woc *wfOperationCtx) persistUpdates(ctx context.Context) {
 	woc.log.WithFields(logging.Fields{"resourceVersion": woc.wf.ResourceVersion, "phase": woc.wf.Status.Phase}).Info(ctx, "Workflow update successful")
 
 	switch os.Getenv("INFORMER_WRITE_BACK") {
-	// By default we write back (as per v2.11), this does not reduce errors, but does reduce
+	// this does not reduce errors, but does reduce
 	// conflicts and therefore we log fewer warning messages.
-	case "", "true":
+	case "true":
 		if err := woc.writeBackToInformer(); err != nil {
 			woc.markWorkflowError(ctx, err)
 			return
 		}
-	case "false":
+	// no longer write back to informer cache as default (as per v4.0)
+	case "", "false":
 		time.Sleep(1 * time.Second)
 	}