Reduce the number of VMs on TestMultipleVMs_Isolated

The test is still unstable (see firecracker-microvm#581) and we couldn't spend much time
root-causing the issue.

Signed-off-by: Kazuyoshi Kato <katokazu@amazon.com>

runtime/service_integ_test.go

@@ -252,6 +252,32 @@ func createTapDevice(ctx context.Context, tapName string) error {
 func TestMultipleVMs_Isolated(t *testing.T) {
 	integtest.Prepare(t)
 
+	var err error
+
+	// numberOfVmsEnvName = NUMBER_OF_VMS ENV and is configurable from buildkite
+	numberOfVms := defaultNumberOfVms
+	if str := os.Getenv(numberOfVmsEnvName); str != "" {
+		numberOfVms, err = strconv.Atoi(str)
+		require.NoError(t, err, "failed to get NUMBER_OF_VMS env")
+	}
+	t.Logf("TestMultipleVMs_Isolated: will run up to %d VMs", numberOfVms)
+
+	// We should be able to run 10 VMs without any issues.
+	if numberOfVms <= 10 {
+		testMultipleVMs(t, 10)
+		return
+	}
+
+	// We have issues running 100 VMs (see #581).
+	// Incrementally increase the number of VMs to find the breaking point.
+	for n := 0; n <= numberOfVms; n += 10 {
+		t.Run(fmt.Sprintf("VMs=%d", n), func(t *testing.T) {
+			testMultipleVMs(t, n)
+		})
+	}
+}
+
+func testMultipleVMs(t *testing.T, count int) {
 	// This test starts multiple VMs and some may hit firecracker-containerd's
 	// default timeout. So overriding the timeout to wait longer.
 	// One hour should be enough to start a VM, regardless of the load of
@@ -261,14 +287,6 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 	netns, err := ns.GetCurrentNS()
 	require.NoError(t, err, "failed to get a namespace")
 
-	// numberOfVmsEnvName = NUMBER_OF_VMS ENV and is configurable from buildkite
-	numberOfVms := defaultNumberOfVms
-	if str := os.Getenv(numberOfVmsEnvName); str != "" {
-		numberOfVms, err = strconv.Atoi(str)
-		require.NoError(t, err, "failed to get NUMBER_OF_VMS env")
-	}
-	t.Logf("TestMultipleVMs_Isolated: will run %d vm's", numberOfVms)
-
 	tapPrefix := os.Getenv(tapPrefixEnvName)
 
 	cases := []struct {
@@ -313,7 +331,7 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 	// container ends up in the right VM by assigning each VM a network device with a unique mac address and having each container
 	// print the mac address it sees inside its VM.
 	vmEg, vmEgCtx := errgroup.WithContext(testCtx)
-	for i := 0; i < numberOfVms; i++ {
+	for i := 0; i < count; i++ {
 		caseTypeNumber := i % len(cases)
 		vmID := i
 		c := cases[caseTypeNumber]
@@ -375,6 +393,7 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 					createVMErr,
 				)
 			}
+			t.Logf("started VM %d", vmID)
 
 			containerEg, containerCtx := errgroup.WithContext(vmEgCtx)
 			for containerID := 0; containerID < int(containerCount); containerID++ {
@@ -433,11 +452,13 @@ func TestMultipleVMs_Isolated(t *testing.T) {
 			if err != nil {
 				return fmt.Errorf("unexpected error from the containers in VM %d: %w", vmID, err)
 			}
+			t.Logf("all containers in VM %d are stopped", vmID)
 
 			_, err = fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: strconv.Itoa(vmID), TimeoutSeconds: 5})
 			if err != nil {
 				return err
 			}
+			t.Logf("stopped VM %d", vmID)
 			return nil
 		}