FLUO: Add reboot test

This commit adds a reboot test for FLUO, the test verifies if the FLUO
can reboot a node running stateless applications.

Signed-off-by: Suraj Deshmukh <suraj@kinvolk.io>

ci/packet_fluo/packet_fluo-cluster.lokocfg.envsubst

@@ -35,6 +35,10 @@ EOF
   worker_pool "general" {
     count     = 1
     node_type = "c2.medium.x86"
+
+    labels = {
+      "fluo-test-pool" = "true"
+    }
   }
 
   worker_pool "storage" {

test/components/flatcar-linux-update-operator/fluo_reboot_test.go

@@ -0,0 +1,82 @@
+// Copyright 2020 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build packet_fluo
+// +build disruptivee2e
+
+package fluo //nolint:testpackage
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	testutil "github.com/kinvolk/lokomotive/test/components/util"
+)
+
+func TestReboot(t *testing.T) {
+	t.Parallel()
+
+	client := testutil.CreateKubeClient(t)
+
+	// Select a node from the general worker pool.
+	nodesList, err := client.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
+		LabelSelector: "fluo-test-pool=true",
+	})
+	if err != nil {
+		t.Fatalf("Listing nodes with label 'fluo-test-pool=true': %v", err)
+	}
+
+	nodes := nodesList.Items
+	if len(nodes) < 1 {
+		t.Fatalf("Wanted one or more nodes found none.")
+	}
+
+	// Select a node to remove annotation
+	chosenNode := nodes[0]
+	t.Logf("Chosen node to reboot: %s", chosenNode.Name)
+
+	// Remove annotation.
+	delete(chosenNode.Annotations, "flatcar-linux-update.v1.flatcar-linux.net/reboot-paused")
+
+	if _, err := client.CoreV1().Nodes().Update(context.Background(), &chosenNode, metav1.UpdateOptions{}); err != nil {
+		t.Fatalf("Removing node annotation 'flatcar-linux-update.v1.flatcar-linux.net/reboot-paused': %v", err)
+	}
+
+	// Wait for the node to reboot and keep checking if the following annotation is added:
+	// "flatcar-linux-update.v1.flatcar-linux.net/status": "UPDATE_STATUS_IDLE"
+	if err := wait.PollImmediate(testutil.RetryInterval, testutil.Timeout, func() (done bool, err error) {
+		node, err := client.CoreV1().Nodes().Get(context.Background(), chosenNode.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, fmt.Errorf("Getting node %q: %v", chosenNode.Name, err)
+		}
+
+		val, ok := node.Annotations["flatcar-linux-update.v1.flatcar-linux.net/status"]
+		if !ok {
+			return false, nil
+		}
+
+		if val == "UPDATE_STATUS_IDLE" {
+			return true, nil
+		}
+
+		return false, nil
+	}); err != nil {
+		t.Fatalf("Waiting for the node to add annotation %s: %v",
+			"flatcar-linux-update.v1.flatcar-linux.net/status: UPDATE_STATUS_IDLE", err)
+	}
+}