Merge pull request #63 from hmrc/BDOG-2534

BDOG-2534 Fix large profiles failing to start

README.md

@@ -7,21 +7,26 @@ It's based on the the original [service-manager](https://github.com/hmrc/service
 ### Installing From Binary
 1. Run the following command in your terminal for your operating system/cpu:
 
-**Linux**
+**Linux Intel**
 ```base
-curl -L -O https://github.com/hmrc/sm2/releases/download/v1.0.6/sm2-1.0.6-linux-intel.zip && unzip sm2-1.0.6-linux-intel.zip && rm sm2-1.0.6-linux-intel.zip
+curl -L -O https://github.com/hmrc/sm2/releases/download/v1.0.7/sm2-1.0.6-linux-intel.zip && unzip sm2-1.0.6-linux-intel.zip && rm sm2-1.0.6-linux-intel.zip
+```
+
+**Linux Arm64**
+```base
+curl -L -O https://github.com/hmrc/sm2/releases/download/v1.0.7/sm2-1.0.6-linux-arm64.zip && unzip sm2-1.0.6-linux-arm64.zip && rm sm2-1.0.6-linux-arm64.zip
 ```
 
 **OSX/Apple (latest M1/M2 cpus)**
 
 ```base
-curl -L -O https://github.com/hmrc/sm2/releases/download/v1.0.6/sm2-1.0.6-apple-arm64.zip && unzip sm2-1.0.6-apple-arm64.zip && rm sm2-1.0.6-apple-arm64.zip
+curl -L -O https://github.com/hmrc/sm2/releases/download/v1.0.7/sm2-1.0.6-apple-arm64.zip && unzip sm2-1.0.6-apple-arm64.zip && rm sm2-1.0.6-apple-arm64.zip
 ```
 
 **OSX/Apple (older Intel cpus)**
 
 ```base
-curl -L -O https://github.com/hmrc/sm2/releases/download/v1.0.6/sm2-1.0.6-apple-intel.zip && unzip sm2-1.0.6-apple-intel.zip && rm sm2-1.0.6-apple-intel.zip
+curl -L -O https://github.com/hmrc/sm2/releases/download/v1.0.7/sm2-1.0.6-apple-intel.zip && unzip sm2-1.0.6-apple-intel.zip && rm sm2-1.0.6-apple-intel.zip
 ```
 
 If everything has worked you should have an executable called `sm2`.
@@ -88,6 +93,13 @@ Alternatively you can start more than one service at once by typing multiple ser
 $ sm2 -start SERVICE_ONE SERVICE_TWO
 ```
 
+#### Starting a large group of services
+Starting a large group of services can overload the cpu of a machine and lead to services failing to start.
+If this happens use the following command to start the services at a slower pace.
+```shell
+$ sm2 --start LARGE_PROFILE_NAME --workers 1 --delay-seconds 5
+```
+The workers argument starts one service at a time and the DelaySeconds argument adds a 5 second delay inbetween services.
 
 ### Starting specific versions
 If you need to run a specific version of a service you can do so by adding a colon followed by the version number to the service name, e.g.

cli/cli.go

@@ -44,6 +44,7 @@ type UserOption struct {
 	Verify        bool                // checks if a given service or profile is running
 	Wait          int                 // waits given number of secs after starting services for then to respond to pings
 	Workers       int                 // sets the number of concurrent downloads/service starts
+	DelaySeconds  int                 // sets the pause in seconds between starting services
 }
 
 func Parse(args []string) (*UserOption, error) {
@@ -115,8 +116,8 @@ func fixupInvalidFlags(args []string) []string {
 }
 
 /*
-   Parses extra args for all the services. Expected format is:
-   {"SERVICE_NAME":["-DFoo=Bar","SOMETHING"],"SERVICE_TWO":["APPEND_THIS"]}
+Parses extra args for all the services. Expected format is:
+{"SERVICE_NAME":["-DFoo=Bar","SOMETHING"],"SERVICE_TWO":["APPEND_THIS"]}
 */
 func parseAppendArgs(jsonArgs string) (map[string][]string, error) {
 
@@ -183,6 +184,7 @@ func buildFlagSet(opts *UserOption) *flag.FlagSet {
 	flagset.BoolVar(&opts.Verify, "verify", false, "for scripts, checks if a service/profile is running")
 	flagset.IntVar(&opts.Wait, "wait", 0, "used with --start, waits a specified number of seconds for the services to become available before exiting (use with --start)")
 	flagset.IntVar(&opts.Workers, "workers", defaultWorkers(), "how many services should be downloaded at the same time (use with --start)")
+	flagset.IntVar(&opts.DelaySeconds, "delay-seconds", 0, "how long to pause, in seconds, after starting a service before starting another")
 
 	return flagset
 }

servicemanager/startfromsource.go

@@ -43,7 +43,9 @@ func (sm *ServiceManager) StartFromSource(serviceName string) error {
 		return err
 	}
 
-	return sm.Ledger.SaveStateFile(installDir, state)
+	err = sm.Ledger.SaveStateFile(installDir, state)
+	sm.pauseTillHealthy(state.HealthcheckUrl)
+	return err
 }
 
 func (sm *ServiceManager) installFromGit(installDir string, gitUrl string, service Service) (ledger.InstallFile, error) {
@@ -96,17 +98,18 @@ func (sm ServiceManager) sbtBuildAndRun(srcDir string, service Service) (ledger.
 		return state, err
 	}
 
+	healthcheckUrl := findHealthcheckUrl(service, state.Port)
 	state = ledger.StateFile{
-		Service:  service.Id,
-		Artifact: service.Binary.Artifact,
-		Version:  SOURCE,
-		Path:     srcDir,
-		Started:  time.Now(),
-		Pid:      cmd.Process.Pid,
-		Port:     port,
-		Args:     args,
+		Service:        service.Id,
+		Artifact:       service.Binary.Artifact,
+		Version:        SOURCE,
+		Path:           srcDir,
+		Started:        time.Now(),
+		Pid:            cmd.Process.Pid,
+		Port:           port,
+		Args:           args,
+		HealthcheckUrl: healthcheckUrl,
 	}
-
 	return state, nil
 }
 

servicemanager/startservice.go

@@ -90,9 +90,20 @@ func (sm *ServiceManager) StartService(serviceAndVersion ServiceAndVersion) erro
 		return err
 	}
 	state.HealthcheckUrl = healthcheckUrl
-
 	// and finally, we record out success
-	return sm.Ledger.SaveStateFile(installDir, state)
+	err = sm.Ledger.SaveStateFile(installDir, state)
+	sm.pauseTillHealthy(healthcheckUrl)
+	return err
+}
+
+func (sm *ServiceManager) pauseTillHealthy(healthcheckUrl string) {
+	if sm.Commands.DelaySeconds > 0 {
+		count := 0
+		for count < sm.Commands.DelaySeconds*2 && !sm.CheckHealth(healthcheckUrl) {
+			count++
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
 }
 
 func (sm *ServiceManager) installService(installDir string, serviceId string, group string, artifact string, version string) (ledger.InstallFile, error) {