test: Simplify E2E test tear-down

hack/test_report.go

@@ -4,6 +4,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"strconv"
 	"strings"
 )
@@ -40,6 +41,10 @@ func testReport() {
 		for _, c := range s.TestCases {
 			if c.Failure.Text != "" {
 				x := newFailureText(s.Name, c.Failure.Text)
+				if x.file == "" {
+					_, _ = fmt.Fprintf(os.Stderr, "could not parse "+c.Failure.Text)
+					continue
+				}
 				// https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message
 				// Replace ‘/n’ with ‘%0A’ for multiple strings output.
 				_, _ = fmt.Printf("::error file=%s,line=%v,col=0::%s\n", x.file, x.line, x.message)
@@ -61,12 +66,15 @@ func trimStdoutLines(text string) string {
 			return strings.Join(split[i:], "\n")
 		}
 	}
-	panic(text)
+	return text
 }
 
 func newFailureText(suite, text string) failureText {
 	text = trimStdoutLines(text)
 	parts := strings.SplitN(text, ":", 3)
+	if len(parts) != 3 {
+		return failureText{}
+	}
 	file := strings.TrimPrefix(suite, "github.com/argoproj/argo/") + "/" + parts[0]
 	line, _ := strconv.Atoi(parts[1])
 	message := strings.ReplaceAll(strings.TrimSpace(parts[2]), "\n", "%0A")

pkg/apis/workflow/register.go

@@ -14,6 +14,7 @@ const (
 	WorkflowTemplatePlural           string = "workflowtemplates"
 	WorkflowTemplateShortName        string = "wftmpl"
 	WorkflowTemplateFullName         string = WorkflowTemplatePlural + "." + Group
+	WorkflowEventBindingPlural       string = "workfloweventbindings"
 	CronWorkflowKind                 string = "CronWorkflow"
 	CronWorkflowSingular             string = "cronworkflow"
 	CronWorkflowPlural               string = "cronworkflows"

test/e2e/cli_test.go

@@ -31,9 +31,12 @@ func (s *CLISuite) BeforeTest(suiteName, testName string) {
 }
 
 func (s *CLISuite) testNeedsOffloading() {
-	skip := s.Persistence.IsEnabled() && os.Getenv("ARGO_SERVER") == ""
-	if skip {
-		s.T().Skip("test needs offloading, but not Argo Server available")
+	serverUnavailable := os.Getenv("ARGO_SERVER") == ""
+	if s.Persistence.IsEnabled() && serverUnavailable {
+		if !serverUnavailable {
+			s.T().Skip("test needs offloading, but the Argo Server is unavailable - if `testNeedsOffloading()` is the first line of your test test, you should move your test to `CliWithServerSuite`?")
+		}
+		s.T().Skip("test needs offloading, but offloading not enabled")
 	}
 }
 
@@ -278,7 +281,7 @@ func (s *CLISuite) TestRoot() {
 	})
 }
 
-func (s *CLISuite) TestWorkflowSuspendResume() {
+func (s *CLIWithServerSuite) TestWorkflowSuspendResume() {
 	s.testNeedsOffloading()
 	s.Given().
 		Workflow("@testdata/sleep-3s.yaml").
@@ -302,7 +305,7 @@ func (s *CLISuite) TestWorkflowSuspendResume() {
 		})
 }
 
-func (s *CLISuite) TestNodeSuspendResume() {
+func (s *CLIWithServerSuite) TestNodeSuspendResume() {
 	s.testNeedsOffloading()
 	s.Given().
 		Workflow("@testdata/node-suspend.yaml").
@@ -582,7 +585,7 @@ func (s *CLISuite) TestWorkflowLint() {
 	})
 }
 
-func (s *CLISuite) TestWorkflowRetry() {
+func (s *CLIWithServerSuite) TestWorkflowRetry() {
 	s.testNeedsOffloading()
 	var retryTime corev1.Time
 
@@ -635,7 +638,7 @@ func (s *CLISuite) TestWorkflowTerminate() {
 		})
 }
 
-func (s *CLISuite) TestWorkflowWait() {
+func (s *CLIWithServerSuite) TestWorkflowWait() {
 	s.testNeedsOffloading()
 	s.Given().
 		Workflow("@smoke/basic.yaml").
@@ -649,7 +652,7 @@ func (s *CLISuite) TestWorkflowWait() {
 		})
 }
 
-func (s *CLISuite) TestWorkflowWatch() {
+func (s *CLIWithServerSuite) TestWorkflowWatch() {
 	s.testNeedsOffloading()
 	s.Given().
 		Workflow("@smoke/basic.yaml").
@@ -948,58 +951,17 @@ func (s *CLISuite) TestWorkflowTemplateRefSubmit() {
 	})
 }
 
-func (s *CLISuite) TestWorkflowLevelSemaphore() {
-	semaphoreData := map[string]string{
-		"workflow": "1",
-	}
-	s.testNeedsOffloading()
-	s.Given().
-		Workflow("@testdata/semaphore-wf-level.yaml").
-		When().
-		CreateConfigMap("my-config", semaphoreData).
-		RunCli([]string{"submit", "testdata/semaphore-wf-level-1.yaml"}, func(t *testing.T, output string, err error) {
-			if assert.NoError(t, err) {
-				assert.Contains(t, output, "semaphore-wf-level-1")
-			}
-		}).
-		SubmitWorkflow().
-		RunCli([]string{"get", "semaphore-wf-level"}, func(t *testing.T, output string, err error) {
-			assert.Contains(t, output, "Waiting for")
-		}).
-		WaitForWorkflow(30 * time.Second).
-		DeleteConfigMap().
-		Then().
-		ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
-			assert.Equal(t, wfv1.NodeSucceeded, status.Phase)
-		})
-}
-
-func (s *CLISuite) TestTemplateLevelSemaphore() {
-	semaphoreData := map[string]string{
-		"template": "1",
-	}
-
-	s.testNeedsOffloading()
-	s.Given().
-		Workflow("@testdata/semaphore-tmpl-level.yaml").
-		When().
-		CreateConfigMap("my-config", semaphoreData).
-		SubmitWorkflow().
-		Wait(12*time.Second).
-		RunCli([]string{"get", "semaphore-tmpl-level"}, func(t *testing.T, output string, err error) {
-			assert.Contains(t, output, "Waiting for")
-		}).
-		WaitForWorkflow(20 * time.Second).
-		DeleteConfigMap()
-}
-
-func (s *CLISuite) TestRetryOmit() {
+func (s *CLIWithServerSuite) TestRetryOmit() {
 	s.testNeedsOffloading()
 	s.Given().
 		Workflow("@testdata/retry-omit.yaml").
 		When().
 		SubmitWorkflow().
-		WaitForWorkflow(20*time.Second).
+		WaitForWorkflowCondition(func(wf *wfv1.Workflow) bool {
+			return wf.Status.Nodes.Any(func(node wfv1.NodeStatus) bool {
+				return node.Phase == wfv1.NodeOmitted
+			})
+		}, "any node omitted", 20*time.Second).
 		Then().
 		ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
 			node := status.Nodes.FindByDisplayName("should-not-execute")
@@ -1014,7 +976,7 @@ func (s *CLISuite) TestRetryOmit() {
 		WaitForWorkflow(20 * time.Second)
 }
 
-func (s *CLISuite) TestResourceTemplateStopAndTerminate() {
+func (s *CLIWithServerSuite) TestResourceTemplateStopAndTerminate() {
 	s.testNeedsOffloading()
 	s.Run("ResourceTemplateStop", func() {
 		s.Given().

test/e2e/cli_with_server_test.go

@@ -152,49 +152,41 @@ func (s *CLIWithServerSuite) TestArchive() {
 }
 
 func (s *CLIWithServerSuite) TestWorkflowLevelSemaphore() {
-	semaphoreData := map[string]string{
-		"workflow": "1",
-	}
 	s.testNeedsOffloading()
 	s.Given().
 		Workflow("@testdata/semaphore-wf-level.yaml").
 		When().
-		CreateConfigMap("my-config", semaphoreData).
-		RunCli([]string{"submit", "testdata/semaphore-wf-level-1.yaml"}, func(t *testing.T, output string, err error) {
-			if assert.NoError(t, err) {
-				assert.Contains(t, output, "semaphore-wf-level-1")
-			}
-		}).
+		CreateConfigMap("my-config", map[string]string{"my-key": "1"}).
 		SubmitWorkflow().
-		Wait(1*time.Second).
-		RunCli([]string{"get", "semaphore-wf-level"}, func(t *testing.T, output string, err error) {
-			assert.Contains(t, output, "Pending")
-		}).
+		WaitForWorkflowToStart(15*time.Second).
+		// the second workflow will have to wait for the first to complete
+		SubmitWorkflow().
+		WaitForWorkflowCondition(func(wf *wfv1.Workflow) bool {
+			return wf.Status.Synchronization != nil
+		}, "is waiting on sync", 15*time.Second).
 		WaitForWorkflow(30 * time.Second).
-		DeleteConfigMap().
 		Then().
 		ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
 			assert.Equal(t, wfv1.NodeSucceeded, status.Phase)
 		})
 }
 
 func (s *CLIWithServerSuite) TestTemplateLevelSemaphore() {
-	semaphoreData := map[string]string{
-		"template": "1",
-	}
-
 	s.testNeedsOffloading()
 	s.Given().
 		Workflow("@testdata/semaphore-tmpl-level.yaml").
 		When().
-		CreateConfigMap("my-config", semaphoreData).
+		CreateConfigMap("my-config", map[string]string{"my-key": "1"}).
 		SubmitWorkflow().
-		Wait(1*time.Second).
-		RunCli([]string{"get", "semaphore-tmpl-level"}, func(t *testing.T, output string, err error) {
-			assert.Contains(t, output, "Waiting for")
-		}).
-		WaitForWorkflow(20 * time.Second).
-		DeleteConfigMap()
+		// we'll have one node waiting on another
+		WaitForWorkflowCondition(func(wf *wfv1.Workflow) bool {
+			return wf.Status.Synchronization != nil
+		}, "is waiting on sync", 15*time.Second).
+		WaitForWorkflow(30 * time.Second).
+		Then().
+		ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
+			assert.Equal(t, wfv1.NodeSucceeded, status.Phase)
+		})
 }
 
 func (s *CLIWithServerSuite) TestArgoSetOutputs() {