Refactor e2e tests

Makefile

@@ -62,16 +62,29 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+## Enable debug options.
+DEBUG =
+
 ## Tests parallelism.
-GINKGO_PROCS ?= 1
+GINKGO_PROCS ?= 2
+
+## Filter tests by labels.
+GINKGO_LABEL_FILTER =
 
 GINKGO_FLAGS += --vv
 GINKGO_FLAGS += --trace
 GINKGO_FLAGS += --procs="$(GINKGO_PROCS)"
 GINKGO_FLAGS += --timeout=1h
-GINKGO_FLAGS += --poll-progress-after=5m
+GINKGO_FLAGS += --poll-progress-after=2m
 GINKGO_FLAGS += --poll-progress-interval=1m
-GINKGO_FLAGS += --label-filter="$(GINKGO_LABEL_FILTER)"
+
+ifneq ($(GINKGO_LABEL_FILTER),)
+	GINKGO_FLAGS += --label-filter="$(GINKGO_LABEL_FILTER)"
+endif
+
+ifneq ($(DEBUG),)
+	GINKGO_FLAGS += --fail-fast
+endif
 
 ##@ General
 
@@ -92,7 +105,7 @@ all: lint build helm-chart test ## Default target: make build, run linters and u
 .PHONY: help
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
-	@awk 'BEGIN {FS = " \??= "; printf "\n\033[1m%s\033[0m\n", "Options:"} /^## .*/ {C=C substr($$0, 4) " "} /^[A-Z0-9_]* \??= .*/ && C { printf "  \033[36m%s\033[0m = %*s %s\n", $$1, -50+length($$1), $$2, C} /^[^#]/ { C="" }  END {  }  ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = " \??= *"; printf "\n\033[1m%s\033[0m\n", "Options:"} /^## .*/ {C=C substr($$0, 4) " "} /^[A-Z0-9_]* \??=.*/ && C { printf "  \033[36m%s\033[0m = %*s %s\n", $$1, -50+length($$1), $$2, C} /^[^#]/ { C="" }  END {  }  ' $(MAKEFILE_LIST)
 
 ##@ Development
 

pkg/testutil/spec_builders.go

@@ -20,7 +20,8 @@ const (
 	// It is short because of error:
 	// `Failed to create pod sandbox: failed to construct FQDN from pod hostname and cluster domain, FQDN
 	// <...> is too long (64 characters is the max, 67 characters requested)`.
-	RemoteResourceName = "tst-rmt"
+	// FIXME(khlebnikov): https://github.com/ytsaurus/ytsaurus-k8s-operator/issues/390
+	RemoteResourceName = "rmt"
 	// Images should be in sync with TEST_IMAGES variable in Makefile
 	// todo: come up with a more elegant solution
 	CoreImageNextVer = "ghcr.io/ytsaurus/ytsaurus:stable-24.1.0"
@@ -110,8 +111,7 @@ func CreateMinimalYtsaurusResource(namespace string) *ytv1.Ytsaurus {
 	}
 }
 
-func CreateBaseYtsaurusResource(namespace string) *ytv1.Ytsaurus {
-	ytsaurus := CreateMinimalYtsaurusResource(namespace)
+func WithBaseYtsaurusComponents(ytsaurus *ytv1.Ytsaurus) *ytv1.Ytsaurus {
 	ytsaurus = WithBootstrap(ytsaurus)
 	ytsaurus = WithScheduler(ytsaurus)
 	ytsaurus = WithControllerAgents(ytsaurus)
@@ -121,6 +121,12 @@ func CreateBaseYtsaurusResource(namespace string) *ytv1.Ytsaurus {
 	return ytsaurus
 }
 
+func CreateBaseYtsaurusResource(namespace string) *ytv1.Ytsaurus {
+	ytsaurus := CreateMinimalYtsaurusResource(namespace)
+	ytsaurus = WithBaseYtsaurusComponents(ytsaurus)
+	return ytsaurus
+}
+
 // TODO (l0kix2): merge with ytconfig build spec helpers.
 func WithDataNodes(ytsaurus *ytv1.Ytsaurus) *ytv1.Ytsaurus {
 	return WithDataNodesCount(ytsaurus, 3, nil)

test/e2e/suite_test.go

@@ -27,11 +27,13 @@ import (
 	"k8s.io/utils/ptr"
 
 	. "github.com/onsi/ginkgo/v2"
+	gtypes "github.com/onsi/ginkgo/v2/types"
 	. "github.com/onsi/gomega"
 	otypes "github.com/onsi/gomega/types"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
 
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	clientgoretry "k8s.io/client-go/util/retry"
@@ -42,7 +44,10 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
+	corev1 "k8s.io/api/core/v1"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	ytv1 "github.com/ytsaurus/ytsaurus-k8s-operator/api/v1"
 	//+kubebuilder:scaffold:imports
@@ -51,7 +56,7 @@ import (
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
-var k8sClient client.Client
+var k8sClient client.WithWatch
 var ctx context.Context
 var log = logf.Log
 
@@ -102,11 +107,19 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) []byte {
 	err = ytv1.AddToScheme(scheme)
 	Expect(err).NotTo(HaveOccurred())
 
-	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
+	k8sClient, err = client.NewWithWatch(cfg, client.Options{Scheme: scheme})
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())
 })
 
+func ShouldPreserveArtifacts() bool {
+	suiteConfig, _ := GinkgoConfiguration()
+	if !suiteConfig.FailFast {
+		return false
+	}
+	return CurrentSpecReport().State.Is(gtypes.SpecStateFailed | gtypes.SpecStateTimedout)
+}
+
 var _ = BeforeEach(func() {
 	var cancel context.CancelFunc
 	ctx, cancel = context.WithCancel(context.TODO())
@@ -116,6 +129,36 @@ var _ = BeforeEach(func() {
 	})
 })
 
+func LogObjectEvents(ctx context.Context, namespace string) func() {
+	watcher, err := k8sClient.Watch(ctx, &corev1.EventList{}, &client.ListOptions{
+		Namespace: namespace,
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	logEvent := func(event *corev1.Event) {
+		log.Info("Event",
+			"type", event.Type,
+			"kind", event.InvolvedObject.Kind,
+			"name", event.InvolvedObject.Name,
+			"reason", event.Reason,
+			"message", event.Message,
+		)
+	}
+
+	go func() {
+		for ev := range watcher.ResultChan() {
+			switch ev.Type {
+			case watch.Added, watch.Modified:
+				if event, ok := ev.Object.(*corev1.Event); ok {
+					logEvent(event)
+				}
+			}
+		}
+	}()
+
+	return watcher.Stop
+}
+
 func NewYtsaurusStatusTracker() func(*ytv1.Ytsaurus) bool {
 	prevStatus := ytv1.YtsaurusStatus{}
 	conditions := map[string]metav1.Condition{}
@@ -182,21 +225,35 @@ func NewYtsaurusStatusTracker() func(*ytv1.Ytsaurus) bool {
 	}
 }
 
-func GetYtsaurus(ctx context.Context, ytsaurus *ytv1.Ytsaurus) Assertion {
-	name := client.ObjectKeyFromObject(ytsaurus)
-	err := k8sClient.Get(ctx, name, ytsaurus)
+func GetObjectGVK(object client.Object) schema.GroupVersionKind {
+	gvks, _, err := k8sClient.Scheme().ObjectKinds(object)
 	Expect(err).ToNot(HaveOccurred())
-	return Expect(ytsaurus)
+	return gvks[0]
 }
 
-func UpdateYtsaurus(ctx context.Context, ytsaurus *ytv1.Ytsaurus) {
-	var current ytv1.Ytsaurus
-	name := client.ObjectKeyFromObject(ytsaurus)
+func CurrentlyObject[T client.Object](ctx context.Context, object T) Assertion {
+	key := client.ObjectKeyFromObject(object)
+	err := k8sClient.Get(ctx, key, object)
+	Expect(err).ToNot(HaveOccurred())
+	return Expect(object)
+}
+
+func EventuallyObject[T client.Object](ctx context.Context, object T, timeout time.Duration) AsyncAssertion {
+	key := client.ObjectKeyFromObject(object)
+	return Eventually(ctx, func(ctx context.Context) (T, error) {
+		err := k8sClient.Get(ctx, key, object)
+		return object, err
+	}, timeout, pollInterval)
+}
+
+func UpdateObject(ctx context.Context, object client.Object) {
+	current := object.DeepCopyObject().(client.Object)
+	key := client.ObjectKeyFromObject(object)
 	err := clientgoretry.RetryOnConflict(clientgoretry.DefaultRetry, func() error {
-		Expect(k8sClient.Get(ctx, name, &current)).To(Succeed())
+		Expect(k8sClient.Get(ctx, key, current)).To(Succeed())
 		// Fetch current resource version: any status update changes it too.
-		ytsaurus.SetResourceVersion(current.GetResourceVersion())
-		return k8sClient.Update(ctx, ytsaurus)
+		object.SetResourceVersion(current.GetResourceVersion())
+		return k8sClient.Update(ctx, object)
 	})
 	Expect(err).ToNot(HaveOccurred())
 }
@@ -276,3 +333,7 @@ func HaveClusterUpdatingComponents(components ...string) otypes.GomegaMatcher {
 	)
 	// FIXME(khlebnikov): Flow initial state is None which is weird.
 }
+
+func HaveRemoteNodeReleaseStatusRunning() otypes.GomegaMatcher {
+	return HaveField("Status.ReleaseStatus", ytv1.RemoteNodeReleaseStatusRunning)
+}