Merge pull request #1544 from endocode/dongsu/fleetctl-unit-action-fx…

…tests

functional: more fleetctl unit action tests

functional/platform/cluster.go

@@ -35,7 +35,9 @@ type Cluster interface {
 	// client operations
 	Fleetctl(m Member, args ...string) (string, string, error)
 	FleetctlWithInput(m Member, input string, args ...string) (string, string, error)
+	WaitForNUnits(Member, int) (map[string][]util.UnitState, error)
 	WaitForNActiveUnits(Member, int) (map[string][]util.UnitState, error)
+	WaitForNUnitFiles(Member, int) (map[string][]util.UnitFileState, error)
 	WaitForNMachines(Member, int) ([]string, error)
 }
 

functional/platform/nspawn.go

@@ -128,6 +128,47 @@ func (nc *nspawnCluster) FleetctlWithInput(m Member, input string, args ...strin
 	return util.RunFleetctlWithInput(input, args...)
 }
 
+// WaitForNUnits runs fleetctl list-units to verify the actual number of units
+// matched with the given expected number. It periodically runs list-units
+// waiting until list-units actually shows the expected units.
+func (nc *nspawnCluster) WaitForNUnits(m Member, expectedUnits int) (map[string][]util.UnitState, error) {
+	var nUnits int
+	retStates := make(map[string][]util.UnitState)
+	checkListUnits := func() bool {
+		outListUnits, _, err := nc.Fleetctl(m, "list-units", "--no-legend", "--full", "--fields", "unit,active,machine")
+		if err != nil {
+			return false
+		}
+		// NOTE: There's no need to check if outListUnits is expected to be empty,
+		// because ParseUnitStates() implicitly filters out such cases.
+		// However, in case of ParseUnitStates() going away, we should not
+		// forget about such special cases.
+		units := strings.Split(strings.TrimSpace(outListUnits), "\n")
+		allStates := util.ParseUnitStates(units)
+		nUnits = len(allStates)
+		if nUnits != expectedUnits {
+			return false
+		}
+
+		for _, state := range allStates {
+			name := state.Name
+			if _, ok := retStates[name]; !ok {
+				retStates[name] = []util.UnitState{}
+			}
+			retStates[name] = append(retStates[name], state)
+		}
+		return true
+	}
+
+	timeout, err := util.WaitForState(checkListUnits)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find %d units within %v (last found: %d)",
+			expectedUnits, timeout, nUnits)
+	}
+
+	return retStates, nil
+}
+
 func (nc *nspawnCluster) WaitForNActiveUnits(m Member, count int) (map[string][]util.UnitState, error) {
 	var nactive int
 	states := make(map[string][]util.UnitState)
@@ -165,6 +206,49 @@ func (nc *nspawnCluster) WaitForNActiveUnits(m Member, count int) (map[string][]
 	return states, nil
 }
 
+// WaitForNUnitFiles runs fleetctl list-unit-files to verify the actual number of units
+// matched with the given expected number. It periodically runs list-unit-files
+// waiting until list-unit-files actually shows the expected units.
+func (nc *nspawnCluster) WaitForNUnitFiles(m Member, expectedUnits int) (map[string][]util.UnitFileState, error) {
+	var nUnits int
+	retStates := make(map[string][]util.UnitFileState)
+
+	checkListUnitFiles := func() bool {
+		outListUnitFiles, _, err := nc.Fleetctl(m, "list-unit-files", "--no-legend", "--full", "--fields", "unit,dstate,state")
+		if err != nil {
+			return false
+		}
+		// NOTE: There's no need to check if outListUnits is expected to be empty,
+		// because ParseUnitFileStates() implicitly filters out such cases.
+		// However, in case of ParseUnitFileStates() going away, we should not
+		// forget about such special cases.
+		units := strings.Split(strings.TrimSpace(outListUnitFiles), "\n")
+		allStates := util.ParseUnitFileStates(units)
+		nUnits = len(allStates)
+		if nUnits != expectedUnits {
+			// retry until number of units matched
+			return false
+		}
+
+		for _, state := range allStates {
+			name := state.Name
+			if _, ok := retStates[name]; !ok {
+				retStates[name] = []util.UnitFileState{}
+			}
+			retStates[name] = append(retStates[name], state)
+		}
+		return true
+	}
+
+	timeout, err := util.WaitForState(checkListUnitFiles)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find %d units within %v (last found: %d)",
+			expectedUnits, timeout, nUnits)
+	}
+
+	return retStates, nil
+}
+
 func (nc *nspawnCluster) WaitForNMachines(m Member, count int) ([]string, error) {
 	var machines []string
 	timeout := 10 * time.Second

functional/unit_action_test.go

@@ -15,6 +15,8 @@
 package functional
 
 import (
+	"io/ioutil"
+	"path"
 	"strings"
 	"testing"
 
@@ -53,6 +55,9 @@ func TestUnitRunnable(t *testing.T) {
 	}
 }
 
+// TestUnitSubmit checks if a unit becomes submitted and destroyed successfully.
+// First it submits a unit, and destroys the unit, verifies it's destroyed,
+// finally submits the unit again.
 func TestUnitSubmit(t *testing.T) {
 	cluster, err := platform.NewNspawnCluster("smoke")
 	if err != nil {
@@ -69,47 +74,133 @@ func TestUnitSubmit(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	unitFile := "fixtures/units/hello.service"
+
 	// submit a unit and assert it shows up
-	if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
+	if _, _, err := cluster.Fleetctl(m, "submit", unitFile); err != nil {
 		t.Fatalf("Unable to submit fleet unit: %v", err)
 	}
-	stdout, _, err := cluster.Fleetctl(m, "list-units", "--no-legend")
+
+	// wait until the unit gets submitted up to 15 seconds
+	listUnitStates, err := cluster.WaitForNUnitFiles(m, 1)
 	if err != nil {
-		t.Fatalf("Failed to run list-units: %v", err)
+		t.Fatalf("Failed to run list-unit-files: %v", err)
 	}
-	units := strings.Split(strings.TrimSpace(stdout), "\n")
-	if len(units) != 1 {
-		t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout)
+
+	// given unit name must be there in list-unit-files
+	_, found := listUnitStates[path.Base(unitFile)]
+	if len(listUnitStates) != 1 || !found {
+		t.Fatalf("Expected %s to be unit file, got %v", path.Base(unitFile), listUnitStates)
 	}
 
 	// submitting the same unit should not fail
-	if _, _, err = cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
+	if _, _, err = cluster.Fleetctl(m, "submit", unitFile); err != nil {
 		t.Fatalf("Expected no failure when double-submitting unit, got this: %v", err)
 	}
 
 	// destroy the unit and ensure it disappears from the unit list
-	if _, _, err := cluster.Fleetctl(m, "destroy", "fixtures/units/hello.service"); err != nil {
+	if _, _, err := cluster.Fleetctl(m, "destroy", unitFile); err != nil {
 		t.Fatalf("Failed to destroy unit: %v", err)
 	}
-	stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend")
+	// wait until the unit gets destroyed up to 15 seconds
+	listUnitStates, err = cluster.WaitForNUnitFiles(m, 0)
 	if err != nil {
-		t.Fatalf("Failed to run list-units: %v", err)
+		t.Fatalf("Failed to run list-unit-files: %v", err)
 	}
-	if strings.TrimSpace(stdout) != "" {
-		t.Fatalf("Did not find 0 units in cluster: \n%s", stdout)
+	if len(listUnitStates) != 0 {
+		t.Fatalf("Expected nil unit file list, got %v", listUnitStates)
 	}
 
 	// submitting the unit after destruction should succeed
-	if _, _, err := cluster.Fleetctl(m, "submit", "fixtures/units/hello.service"); err != nil {
+	if _, _, err := cluster.Fleetctl(m, "submit", unitFile); err != nil {
 		t.Fatalf("Unable to submit fleet unit: %v", err)
 	}
-	stdout, _, err = cluster.Fleetctl(m, "list-units", "--no-legend")
+
+	// wait until the unit gets submitted up to 15 seconds
+	listUnitStates, err = cluster.WaitForNUnitFiles(m, 1)
+	if err != nil {
+		t.Fatalf("Failed to run list-unit-files: %v", err)
+	}
+
+	// given unit name must be there in list-unit-files
+	_, found = listUnitStates[path.Base(unitFile)]
+	if len(listUnitStates) != 1 || !found {
+		t.Fatalf("Expected %s to be unit file, got %v", path.Base(unitFile), listUnitStates)
+	}
+}
+
+// TestUnitLoad checks if a unit becomes loaded and unloaded successfully.
+// First it load a unit, and unloads the unit, verifies it's unloaded,
+// finally loads the unit again.
+func TestUnitLoad(t *testing.T) {
+	cluster, err := platform.NewNspawnCluster("smoke")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cluster.Destroy()
+
+	m, err := cluster.CreateMember()
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = cluster.WaitForNMachines(m, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	unitFile := "fixtures/units/hello.service"
+
+	// load a unit and assert it shows up
+	_, _, err = cluster.Fleetctl(m, "load", unitFile)
+	if err != nil {
+		t.Fatalf("Unable to load fleet unit: %v", err)
+	}
+
+	// wait until the unit gets loaded up to 15 seconds
+	listUnitStates, err := cluster.WaitForNUnits(m, 1)
+	if err != nil {
+		t.Fatalf("Failed to run list-units: %v", err)
+	}
+
+	// given unit name must be there in list-units
+	_, found := listUnitStates[path.Base(unitFile)]
+	if len(listUnitStates) != 1 || !found {
+		t.Fatalf("Expected %s to be unit, got %v", path.Base(unitFile), listUnitStates)
+	}
+
+	// unload the unit and ensure it disappears from the unit list
+	_, _, err = cluster.Fleetctl(m, "unload", unitFile)
+	if err != nil {
+		t.Fatalf("Failed to unload unit: %v", err)
+	}
+
+	// wait until the unit gets unloaded up to 15 seconds
+	listUnitStates, err = cluster.WaitForNUnits(m, 0)
 	if err != nil {
 		t.Fatalf("Failed to run list-units: %v", err)
 	}
-	units = strings.Split(strings.TrimSpace(stdout), "\n")
-	if len(units) != 1 {
-		t.Fatalf("Did not find 1 unit in cluster: \n%s", stdout)
+
+	// given unit name must be there in list-units
+	if len(listUnitStates) != 0 {
+		t.Fatalf("Expected nil unit list, got %v", listUnitStates)
+	}
+
+	// loading the unit after destruction should succeed
+	_, _, err = cluster.Fleetctl(m, "load", unitFile)
+	if err != nil {
+		t.Fatalf("Unable to load fleet unit: %v", err)
+	}
+
+	// wait until the unit gets loaded up to 15 seconds
+	listUnitStates, err = cluster.WaitForNUnits(m, 1)
+	if err != nil {
+		t.Fatalf("Failed to run list-units: %v", err)
+	}
+
+	// given unit name must be there in list-units
+	_, found = listUnitStates[path.Base(unitFile)]
+	if len(listUnitStates) != 1 || !found {
+		t.Fatalf("Expected %s to be unit, got %v", path.Base(unitFile), listUnitStates)
 	}
 }
 
@@ -224,3 +315,92 @@ func TestUnitSSHActions(t *testing.T) {
 		t.Errorf("Could not find expected string in journal output:\n%s", stdout)
 	}
 }
+
+// TestUnitCat simply compares body of a unit file with that of a unit fetched
+// from the remote cluster using "fleetctl cat".
+func TestUnitCat(t *testing.T) {
+	cluster, err := platform.NewNspawnCluster("smoke")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cluster.Destroy()
+
+	m, err := cluster.CreateMember()
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = cluster.WaitForNMachines(m, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// read a sample unit file to a buffer
+	unitFile := "fixtures/units/hello.service"
+	fileBuf, err := ioutil.ReadFile(unitFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fileBody := strings.TrimSpace(string(fileBuf))
+
+	// submit a unit and assert it shows up
+	_, _, err = cluster.Fleetctl(m, "submit", unitFile)
+	if err != nil {
+		t.Fatalf("Unable to submit fleet unit: %v", err)
+	}
+	// wait until the unit gets submitted up to 15 seconds
+	_, err = cluster.WaitForNUnitFiles(m, 1)
+	if err != nil {
+		t.Fatalf("Failed to run list-units: %v", err)
+	}
+
+	// cat the unit file and compare it with the original unit body
+	stdout, _, err := cluster.Fleetctl(m, "cat", path.Base(unitFile))
+	if err != nil {
+		t.Fatalf("Unable to submit fleet unit: %v", err)
+	}
+	catBody := strings.TrimSpace(stdout)
+
+	if strings.Compare(catBody, fileBody) != 0 {
+		t.Fatalf("unit body changed across fleetctl cat: \noriginal:%s\nnew:%s", fileBody, catBody)
+	}
+}
+
+// TestUnitStatus simply checks "fleetctl status hello.service" actually works.
+func TestUnitStatus(t *testing.T) {
+	cluster, err := platform.NewNspawnCluster("smoke")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cluster.Destroy()
+
+	m, err := cluster.CreateMember()
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = cluster.WaitForNMachines(m, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	unitFile := "fixtures/units/hello.service"
+
+	// Load a unit and print out status.
+	// Without loading a unit, it's impossible to run fleetctl status
+	_, _, err = cluster.Fleetctl(m, "load", unitFile)
+	if err != nil {
+		t.Fatalf("Unable to load a fleet unit: %v", err)
+	}
+
+	// wait until the unit gets loaded up to 15 seconds
+	_, err = cluster.WaitForNUnits(m, 1)
+	if err != nil {
+		t.Fatalf("Failed to run list-units: %v", err)
+	}
+
+	stdout, stderr, err := cluster.Fleetctl(m,
+		"--strict-host-key-checking=false", "status", path.Base(unitFile))
+	if !strings.Contains(stdout, "Loaded: loaded") {
+		t.Errorf("Could not find expected string in status output:\n%s\nstderr:\n%s",
+			stdout, stderr)
+	}
+}