Run all integration test in parallel

Makefile

@@ -5,7 +5,7 @@ COMMON_FLAGS := -X $(PROJECT)/pkg/odo/cli/version.GITCOMMIT=$(GITCOMMIT)
 BUILD_FLAGS := -ldflags="-w $(COMMON_FLAGS)"
 DEBUG_BUILD_FLAGS := -ldflags="$(COMMON_FLAGS)"
 FILES := odo dist
-TIMEOUT ?= 1800s
+TIMEOUT ?= 7200s
 
 # Env variable TEST_EXEC_NODES is used to pass spec execution type
 # (parallel or sequential) for ginkgo tests. To run the specs sequentially use
@@ -113,89 +113,114 @@ test:
 # Run generic integration tests
 .PHONY: test-generic
 test-generic:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo generic" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo generic" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run json outout tests
 .PHONY: test-json-format-output
 test-json-format-output:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odojsonoutput" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odojsonoutput" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run component e2e tests
 .PHONY: test-cmp-e2e
 test-cmp-e2e:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoCmpE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoCmpE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run component subcommands e2e tests
 .PHONY: test-cmp-sub-e2e
 test-cmp-sub-e2e:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoCmpSubE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoCmpSubE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run java e2e tests
 .PHONY: test-java-e2e
 test-java-e2e:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoJavaE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoJavaE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run source e2e tests
 .PHONY: test-source-e2e
 test-source-e2e:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoSourceE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoSourceE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run service catalog e2e tests
 .PHONY: test-service-e2e
 test-service-e2e:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoServiceE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoServiceE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/servicecatalog/ -timeout $(TIMEOUT)
 
 # Run link e2e tests
 .PHONY: test-link-e2e
 test-link-e2e:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoLinkE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoLinkE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/servicecatalog/ -timeout $(TIMEOUT)
 
 # Run watch e2e tests
 .PHONY: test-watch-e2e
 test-watch-e2e:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoWatchE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoWatchE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run storage command integration tests
 .PHONY: test-cmd-storage
 test-cmd-storage:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo storage command" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo storage command" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run odo app cmd tests
 .PHONY: test-cmd-app
 test-cmd-app:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoCmdApp" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoCmdApp" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run login e2e tests
 # This test shouldn't run spec in paralel because it will break the test behaviour
 # due to race condition in parallel run.
 .PHONY: test-odo-login-e2e
 test-odo-login-e2e:
-	ginkgo -v -nodes=1 -focus="odoLoginE2e" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=1 -focus="odoLoginE2e" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/loginlogout/ -timeout $(TIMEOUT)
 
 # Run config tests
 .PHONY: test-odo-config
 test-odo-config:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo config test" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo config test" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
-# Run all integration tests
+# Run command's integration tests irrespective of service catalog status in the cluster.
+# Service, link and login/logout command tests are not the part of this test run
 .PHONY: test-integration
 test-integration:
-	go test -v github.com/openshift/odo/tests/integration -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+
+# Run command's integration tests which are depend on service catalog enabled cluster.
+# Only service and link command tests are the part of this test run
+.PHONY: test-integration-service-catalog
+test-integration-service-catalog:
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/servicecatalog -timeout $(TIMEOUT)
 
 # Run url integreation tests
 .PHONY: test-odo-url-int
 test-odo-url-int:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoURLIntegration" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odoURLIntegration" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run push command e2e
 .PHONY: test-cmd-push
 test-cmd-push:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo push command tests" slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -focus="odo push command tests" \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/integration/ -timeout $(TIMEOUT)
 
 # Run e2e test scenarios
 .PHONY: test-e2e-scenarios
 test-e2e-scenarios:
-	ginkgo -v -nodes=$(TEST_EXEC_NODES) slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -randomizeAllSpecs  tests/e2escenarios/ -timeout $(TIMEOUT)
+	ginkgo -v -nodes=$(TEST_EXEC_NODES) -randomizeAllSpecs \
+	slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) tests/e2escenarios/ -timeout $(TIMEOUT)
 
 # this test shouldn't be in paralel -  it will effect the results
 .PHONY: test-benchmark

docs/development.adoc

@@ -268,6 +268,10 @@ $ make test-source-e2e
 
 * To run the tests sequentially or on single ginkgo test node use enviornment variable `TEST_EXEC_NODES`:
 +
+
+NOTE: However the env variable `TEST_EXEC_NODES` can be used to increase or decrease any number of test node for parallel run depending on the resource allocated to the OpenShift cluster. Make sure that the `TEST_EXEC_NODES` value cannot be `0` or any `negative` value.
++
+
 Run component command integration tests
 +
 ----
@@ -354,14 +358,33 @@ $ make test-odo-login-e2e
 
 NOTE: `make test-odo-login-e2e` doesn't honour environment variable `TEST_EXEC_NODES`. So by default it runs login and logout command integration test suite on a single ginkgo test node sequentially to avoid race conditions in a parallel run.
 
-* For the entire integration test suite:
+Entire integration tests suite can be run in parallel and sequentially separately:
+
+* To run entire integration tests spec in parallel (default: 2 ginkgo test node), on a test cluster:
 +
+----
+$ make test-integration-service-catalog
+----
++
+
 ----
 $ make test-integration
 ----
 +
 
-NOTE: `make test-integration` doesn't honour enviornment variable `TEST_EXEC_NODES`. So by default it runs the entire integration test suite on a single ginkgo test node sequentially.
+* To run entire integration tests spec sequentially or on single ginkgo test node use enviornment variable `TEST_EXEC_NODES`:
++
+----
+$ TEST_EXEC_NODES=1 make test-integration-service-catalog
+----
++
+
+----
+$ TEST_EXEC_NODES=1 make test-integration
+----
++
+
+NOTE: `make test-integration-service-catalog` includes only service and link tests and can be successfully validated only if the cluster has service catalog enabled. Whereas `make test-integration` includes tests except service, link, login tests and runs rest of the tests irrespective of service catalog status in the cluster.
 
 Similarly e2e (end to end) tests can be run in parallel and sequentially separately.
 

pkg/preference/preference.go

@@ -112,18 +112,18 @@ func NewPreference() Preference {
 // NewPreferenceInfo gets the PreferenceInfo from preference file and creates the preference file in case it's
 // not present
 func NewPreferenceInfo() (*PreferenceInfo, error) {
-	configFile, err := getPreferenceFile()
-	glog.V(4).Infof("The configFile is %+v", configFile)
+	preferenceFile, err := getPreferenceFile()
+	glog.V(4).Infof("The preference file is %+v", preferenceFile)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to get odo config file")
+		return nil, errors.Wrap(err, "unable to get odo preference file")
 	}
 
 	c := PreferenceInfo{
 		Preference: NewPreference(),
-		Filename:   configFile,
+		Filename:   preferenceFile,
 	}
-	// if the config file doesn't exist then we dont worry about it and return
-	if _, err = os.Stat(configFile); os.IsNotExist(err) {
+	// if the preference file doesn't exist then we dont worry about it and return
+	if _, err = os.Stat(preferenceFile); os.IsNotExist(err) {
 		return &c, nil
 	}
 

tests/e2escenarios/core_beta_flow.go

@@ -33,9 +33,9 @@ var _ = Describe("Core beta flow", func() {
 		oc = helper.NewOcRunner("oc")
 		odo = "odo"
 
-		project = helper.CreateRandProject()
 		context = helper.CreateNewContext()
 		os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "config.yaml"))
+		project = helper.CreateRandProject()
 	})
 
 	AfterEach(func() {

tests/integration/cmd_app_test.go

@@ -23,9 +23,9 @@ var _ = Describe("odoCmdApp", func() {
 	var _ = BeforeEach(func() {
 		SetDefaultEventuallyTimeout(10 * time.Minute)
 		SetDefaultConsistentlyDuration(30 * time.Second)
-		project = helper.CreateRandProject()
 		context = helper.CreateNewContext()
 		os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "config.yaml"))
+		project = helper.CreateRandProject()
 		originalDir = helper.Getwd()
 	})
 
@@ -61,6 +61,11 @@ var _ = Describe("odoCmdApp", func() {
 		})
 
 		Context("when running app command without app parameter in directory that contains .odo config directory", func() {
+
+			JustAfterEach(func() {
+				helper.Chdir(originalDir)
+			})
+
 			It("should successfuly execute list, describe and delete along with machine readable output", func() {
 				helper.CopyExample(filepath.Join("source", "nodejs"), context)
 				helper.CmdShouldPass("odo", "component", "create", "nodejs", cmpName, "--app", appName, "--project", project, "--context", context)
@@ -80,7 +85,6 @@ var _ = Describe("odoCmdApp", func() {
 				Expect(desiredDesAppJSON).Should(MatchJSON(actualDesAppJSON))
 
 				helper.CmdShouldPass("odo", "app", "delete", "-f")
-				helper.Chdir(originalDir)
 			})
 		})
 

tests/integration/cmd_pref_config_test.go

@@ -80,9 +80,9 @@ var _ = Describe("odo config test", func() {
 
 	Context("when creating odo local config in the same config dir", func() {
 		JustBeforeEach(func() {
-			project = helper.CreateRandProject()
 			context = helper.CreateNewContext()
 			os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "preference.yaml"))
+			project = helper.CreateRandProject()
 			originalDir = helper.Getwd()
 			helper.Chdir(context)
 		})
@@ -161,9 +161,9 @@ var _ = Describe("odo config test", func() {
 
 	Context("when creating odo local config with context flag", func() {
 		JustBeforeEach(func() {
-			project = helper.CreateRandProject()
 			context = helper.CreateNewContext()
 			os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "preference.yaml"))
+			project = helper.CreateRandProject()
 		})
 		JustAfterEach(func() {
 			helper.DeleteProject(project)
@@ -239,9 +239,9 @@ var _ = Describe("odo config test", func() {
 
 	Context("when creating odo local config with env variables", func() {
 		JustBeforeEach(func() {
-			project = helper.CreateRandProject()
 			context = helper.CreateNewContext()
 			os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "preference.yaml"))
+			project = helper.CreateRandProject()
 		})
 		JustAfterEach(func() {
 			helper.DeleteProject(project)
@@ -266,9 +266,9 @@ var _ = Describe("odo config test", func() {
 
 	Context("when viewing local config without logging into the OpenShift cluster", func() {
 		JustBeforeEach(func() {
-			project = helper.CreateRandProject()
 			context = helper.CreateNewContext()
 			os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "preference.yaml"))
+			project = helper.CreateRandProject()
 		})
 		JustAfterEach(func() {
 			helper.DeleteProject(project)

tests/integration/cmd_storage_test.go

@@ -18,9 +18,9 @@ var _ = Describe("odo storage command", func() {
 	var _ = BeforeEach(func() {
 		SetDefaultEventuallyTimeout(10 * time.Minute)
 		SetDefaultConsistentlyDuration(30 * time.Second)
-		project = helper.CreateRandProject()
 		context = helper.CreateNewContext()
 		os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "config.yaml"))
+		project = helper.CreateRandProject()
 		oc = helper.NewOcRunner("oc")
 	})
 

tests/integration/cmd_watch_test.go

@@ -21,9 +21,9 @@ var _ = Describe("odoWatchE2e", func() {
 	BeforeEach(func() {
 		SetDefaultEventuallyTimeout(10 * time.Minute)
 		SetDefaultConsistentlyDuration(30 * time.Second)
-		project = helper.CreateRandProject()
 		context = helper.CreateNewContext()
 		os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "config.yaml"))
+		project = helper.CreateRandProject()
 	})
 
 	// Clean up after the test