fix flaky UT

Signed-off-by: Michael Shitrit <mshitrit@redhat.com>

controllers/tests/config/selfnoderemediationconfig_controller_test.go

@@ -132,16 +132,17 @@ var _ = Describe("SNR Config Test", func() {
 				config.Spec.SafeTimeToAssumeNodeRebootedSeconds = 60
 			})
 			It("The Manager Reconciler and the DS should be modified with the new value", func() {
-				Eventually(func() error {
-					return k8sClient.Get(context.Background(), key, ds)
-				}, 10*time.Second, 250*time.Millisecond).Should(BeNil())
-
-				dsContainers := ds.Spec.Template.Spec.Containers
-				Expect(len(dsContainers)).To(BeNumerically("==", 1))
-				container := dsContainers[0]
-				envVars := getEnvVarMap(container.Env)
-				Expect(envVars["TIME_TO_ASSUME_NODE_REBOOTED"].Value).To(Equal("60"))
-				Expect(managerReconciler.SafeTimeCalculator.GetTimeToAssumeNodeRebooted()).To(Equal(time.Minute))
+				Eventually(func(g Gomega) bool {
+					ds = &appsv1.DaemonSet{}
+					g.Expect(k8sClient.Get(context.Background(), key, ds)).To(Succeed())
+					dsContainers := ds.Spec.Template.Spec.Containers
+					g.Expect(len(dsContainers)).To(BeNumerically("==", 1))
+					container := dsContainers[0]
+					envVars := getEnvVarMap(container.Env)
+					g.Expect(envVars["TIME_TO_ASSUME_NODE_REBOOTED"].Value).To(Equal("60"))
+					g.Expect(managerReconciler.SafeTimeCalculator.GetTimeToAssumeNodeRebooted()).To(Equal(time.Minute))
+					return true
+				}, 10*time.Second, 250*time.Millisecond).Should(BeTrue())
 			})
 
 		})
@@ -150,7 +151,10 @@ var _ = Describe("SNR Config Test", func() {
 			var oldDsVersion, currentDsVersion = "0", "1"
 
 			JustBeforeEach(func() {
-				Expect(k8sClient.Create(context.Background(), ds)).To(Succeed())
+				Eventually(func() error {
+					return k8sClient.Create(context.Background(), ds)
+				}, 2*time.Second, 250*time.Millisecond).Should(Succeed())
+
 				Eventually(func() error {
 					return k8sClient.Get(context.Background(), key, ds)
 				}, 2*time.Second, 250*time.Millisecond).Should(BeNil())

controllers/tests/controller/selfnoderemediation_controller_test.go

@@ -317,8 +317,6 @@ var _ = Describe("SNR Controller", func() {
 
 				verifyNoExecuteTaintExist()
 
-				verifyOutOfServiceTaintExist()
-
 				verifyEvent("Normal", "AddOutOfService", "Remediation process - add out-of-service taint to unhealthy node")
 
 				// simulate the out-of-service taint by Pod GC Controller
@@ -523,9 +521,9 @@ func verifyVaNotDeleted(vaName string) {
 }
 
 func verifyLastErrorKeepsApiError() {
-	By("Verify that LastError in SNR status has been kept kube-api error for VA")
+	By("Verify that LastError in SNR status has been kept")
 	snr := &v1alpha1.SelfNodeRemediation{}
-	ConsistentlyWithOffset(1, func() bool {
+	EventuallyWithOffset(1, func() bool {
 		snrNamespacedName := client.ObjectKey{Name: shared.UnhealthyNodeName, Namespace: snrNamespace}
 		if err := k8sClient.Client.Get(context.Background(), snrNamespacedName, snr); err != nil {
 			return false
@@ -595,13 +593,6 @@ func verifyOutOfServiceTaintRemoved() {
 	}, 10*time.Second, 200*time.Millisecond).Should(BeFalse())
 }
 
-func verifyOutOfServiceTaintExist() {
-	By("Verify that node has out-of-service taint")
-	Eventually(func() (bool, error) {
-		return isTaintExist(controllers.OutOfServiceTaint)
-	}, 10*time.Second, 200*time.Millisecond).Should(BeTrue())
-}
-
 func isTaintExist(taintToMatch *v1.Taint) (bool, error) {
 	node := &v1.Node{}
 	err := k8sClient.Reader.Get(context.TODO(), unhealthyNodeNamespacedName, node)
@@ -701,7 +692,7 @@ func deleteRemediations() {
 		snrs := &v1alpha1.SelfNodeRemediationList{}
 		err := k8sClient.List(context.Background(), snrs)
 		return err == nil && len(snrs.Items) == 0
-	}, 5*time.Second, 100*time.Millisecond).Should(BeTrue())
+	}, 7*time.Second, 100*time.Millisecond).Should(BeTrue())
 
 }
 func deleteSNR(snr *v1alpha1.SelfNodeRemediation) {
@@ -933,13 +924,15 @@ func clearEvents() {
 }
 
 func verifyEvent(eventType, reason, message string) {
-	isEventMatch := isEventOccurred(eventType, reason, message)
-	ExpectWithOffset(1, isEventMatch).To(BeTrue())
+	EventuallyWithOffset(1, func() bool {
+		return isEventOccurred(eventType, reason, message)
+	}, 5*time.Second, 250*time.Millisecond).Should(BeTrue())
 }
 
 func verifyNoEvent(eventType, reason, message string) {
-	isEventMatch := isEventOccurred(eventType, reason, message)
-	ExpectWithOffset(1, isEventMatch).To(BeFalse())
+	EventuallyWithOffset(1, func() bool {
+		return isEventOccurred(eventType, reason, message)
+	}, 5*time.Second, 250*time.Millisecond).Should(BeFalse())
 }
 
 func isEventOccurred(eventType string, reason string, message string) bool {

pkg/peerhealth/client_server_test.go

@@ -111,11 +111,12 @@ var _ = Describe("Checking health using grpc client and server", func() {
 			By("calling isHealthy")
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer (cancel)()
-			resp, err := phClient.IsHealthy(ctx, &HealthRequest{
-				NodeName: nodeName,
-			})
-			Expect(err).ToNot(HaveOccurred())
-			Expect(api.HealthCheckResponseCode(resp.Status)).To(Equal(api.Unhealthy))
+			Eventually(func() bool {
+				resp, err := phClient.IsHealthy(ctx, &HealthRequest{
+					NodeName: nodeName,
+				})
+				return err == nil && api.HealthCheckResponseCode(resp.Status) == api.Unhealthy
+			}, time.Second*5, time.Millisecond*250).Should(BeTrue())
 
 		})