chore: improve example test runner to check for new labels #15002

Since we have changed how we identify if an example runs correctly as
test this commit applies to the new protocol in the test runner.

Signed-off-by: Gianluca Arbezzano <[email protected]>

Author: gianarb

.github/workflows/ci-build.yaml

@@ -262,9 +262,11 @@ jobs:
             profile: minimal
             use-api: false
             retries: 0
+          # Example tests are not retryable withou a controller restart
           - test: test-examples
             profile: minimal
             use-api: false
+            retries: 0
           - test: test-plugins
             profile: plugins
             use-api: false

test/e2e/examples_test.go

@@ -3,6 +3,7 @@
 package e2e
 
 import (
+	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
@@ -30,14 +31,33 @@ func (s *ExamplesSuite) TestExampleWorkflows() {
 			s.T().Fatalf("Error parsing %s: %v", path, err)
 		}
 		for _, wf := range wfs {
-			if _, ok := wf.GetLabels()["workflows.argoproj.io/test"]; ok {
-				s.T().Logf("Found example workflow at %s with test label\n", path)
+			isTestBroken := false
+			isEvironmentNotReady := false
+			isTestBrokenRaw, noTestBrokenLabelExists := wf.GetLabels()["workflows.argoproj.io/no-test-broken"]
+			if noTestBrokenLabelExists {
+				isTestBroken, err = strconv.ParseBool(isTestBrokenRaw)
+				if err != nil {
+					s.T().Fatalf("Error parsing annotation \"workflows.argoproj.io/no-test-broken\": %v", err)
+				}
+			}
+			isEvironmentNotReadyRaw, noTestBrokenEnvironmentLabelExists := wf.GetLabels()["workflows.argoproj.io/no-test-environment"]
+			if noTestBrokenEnvironmentLabelExists {
+				isEvironmentNotReady, err = strconv.ParseBool(isEvironmentNotReadyRaw)
+				if err != nil {
+					s.T().Fatalf("Error parsing annotation \"workflows.argoproj.io/no-test-environment\": %v", err)
+				}
+			}
+			if isTestBroken || isEvironmentNotReady {
+				continue
+			}
+			s.T().Run(path, func(t *testing.T) {
+				s.T().Logf("Found example workflow at %s\n", path)
 				s.Given().
 					ExampleWorkflow(&wf).
 					When().
 					SubmitWorkflow().
 					WaitForWorkflow(fixtures.ToBeSucceeded)
-			}
+			})
 		}
 		return nil
 	})

test/e2e/fixtures/given.go

@@ -136,7 +136,18 @@ func (g *Given) checkImages(wf interface{}, isExample bool) {
 			image == "argoproj/argosay:v1" ||
 			image == "argoproj/argosay:v2" ||
 			image == "quay.io/argoproj/argocli:latest" ||
-			(isExample && (image == "busybox" || image == "python:alpine3.6"))
+			(isExample && (image == "busybox" ||
+				image == "python:alpine3.6" ||
+				image == "golang:1.18" ||
+				image == "nginx:1.13" ||
+				image == "curlimages/curl:latest" ||
+				image == "node:9.1-alpine" ||
+				image == "docker:19.03.13" ||
+				image == "docker:19.03.13-dind" ||
+				image == "alpine/git:v2.26.2" ||
+				image == "alpine:3.6" ||
+				image == "stedolan/jq:latest" ||
+				image == "influxdb:1.2"))
 	}
 	for _, t := range templates {
 		container := t.Container