From 715e84ca39d07db3bb305ea44c1f225d199e6950 Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Tue, 19 Feb 2019 15:34:42 +0100 Subject: [PATCH 01/20] Initial commit (issue #90) --- api/blueprint/bootstrapping_api.go | 93 +++++ api/blueprint/bootstrapping_api_mocked.go | 3 + api/blueprint/bootstrapping_api_test.go | 16 + api/types/bootstrapping.go | 28 ++ bootstrapping/bootstrapping.go | 428 ++++++++++++++++++++++ bootstrapping/subcommands.go | 27 ++ cmd/bootstrapping_cmd.go | 29 ++ main.go | 9 + utils/utils.go | 42 +++ utils/webservice.go | 23 +- utils/webservice_mock.go | 2 +- 11 files changed, 690 insertions(+), 10 deletions(-) create mode 100644 api/blueprint/bootstrapping_api.go create mode 100644 api/blueprint/bootstrapping_api_mocked.go create mode 100644 api/blueprint/bootstrapping_api_test.go create mode 100644 api/types/bootstrapping.go create mode 100644 bootstrapping/bootstrapping.go create mode 100644 bootstrapping/subcommands.go create mode 100644 cmd/bootstrapping_cmd.go diff --git a/api/blueprint/bootstrapping_api.go b/api/blueprint/bootstrapping_api.go new file mode 100644 index 0000000..fe45013 --- /dev/null +++ b/api/blueprint/bootstrapping_api.go @@ -0,0 +1,93 @@ +package blueprint + +import ( + "encoding/json" + "fmt" + + log "github.com/Sirupsen/logrus" + "github.com/ingrammicro/concerto/api/types" + "github.com/ingrammicro/concerto/utils" +) + + +// BootstrappingService manages bootstrapping operations +type BootstrappingService struct { + concertoService utils.ConcertoService +} + +// NewBootstrappingService returns a bootstrapping service +func NewBootstrappingService(concertoService utils.ConcertoService) (*BootstrappingService, error) { + if concertoService == nil { + return nil, fmt.Errorf("must initialize ConcertoService before using it") + } + + return &BootstrappingService{ + concertoService: concertoService, + }, nil + +} + +// GetBootstrappingConfiguration returns the list of policy files as a JSON response with the desired configuration changes +func (bs *BootstrappingService) GetBootstrappingConfiguration() (bootstrappingConfigurations *types.BootstrappingConfiguration, status int, err error) { + log.Debug("GetBootstrappingConfiguration") + + data, status, err := bs.concertoService.Get("/blueprint/configuration") + if err != nil { + return nil, status, err + } + + if err = utils.CheckStandardStatus(status, data); err != nil { + return nil, status, err + } + + if err = json.Unmarshal(data, &bootstrappingConfigurations); err != nil { + return nil, status, err + } + + return bootstrappingConfigurations, status, nil +} + +// ReportBootstrappingAppliedConfiguration +func (bs *BootstrappingService) ReportBootstrappingAppliedConfiguration(BootstrappingAppliedConfigurationVector *map[string]interface{}) (err error) { + log.Debug("ReportBootstrappingAppliedConfiguration") + + data, status, err := bs.concertoService.Put("/blueprint/applied_configuration", BootstrappingAppliedConfigurationVector) + if err != nil { + return err + } + + if err = utils.CheckStandardStatus(status, data); err != nil { + return err + } + + return nil +} + +// ReportBootstrappingLog reports a policy files application result +func (bs *BootstrappingService) ReportBootstrappingLog(BootstrappingContinuousReportVector *map[string]interface{}) (command *types.BootstrappingContinuousReport, status int, err error) { + log.Debug("ReportBootstrappingLog") + + data, status, err := bs.concertoService.Post("/blueprint/bootstrap_logs", BootstrappingContinuousReportVector) + if err != nil { + return nil, status, err + } + + if err = json.Unmarshal(data, &command); err != nil { + return nil, status, err + } + + return command, status, nil +} + + +// +func (bs *BootstrappingService) DownloadPolicyFile(url string, filePath string) (realFileName string, status int, err error) { + log.Debug("DownloadPolicyFile") + + realFileName, status, err = bs.concertoService.GetFile(url, "", filePath) + if err != nil { + return realFileName, status, err + } + + return realFileName, status, nil +} \ No newline at end of file diff --git a/api/blueprint/bootstrapping_api_mocked.go b/api/blueprint/bootstrapping_api_mocked.go new file mode 100644 index 0000000..28bf3c3 --- /dev/null +++ b/api/blueprint/bootstrapping_api_mocked.go @@ -0,0 +1,3 @@ +package blueprint + +// TODO diff --git a/api/blueprint/bootstrapping_api_test.go b/api/blueprint/bootstrapping_api_test.go new file mode 100644 index 0000000..74ba96c --- /dev/null +++ b/api/blueprint/bootstrapping_api_test.go @@ -0,0 +1,16 @@ +package blueprint + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewBootstrappingServiceNil(t *testing.T) { + assert := assert.New(t) + rs, err := NewBootstrappingService(nil) + assert.Nil(rs, "Uninitialized service should return nil") + assert.NotNil(err, "Uninitialized service should return error") +} + +// TODO diff --git a/api/types/bootstrapping.go b/api/types/bootstrapping.go new file mode 100644 index 0000000..76e9918 --- /dev/null +++ b/api/types/bootstrapping.go @@ -0,0 +1,28 @@ +package types + +import ( + "encoding/json" +) + +type BootstrappingConfiguration struct { + PolicyFiles []BootstrappingPolicyFile `json:"policyfiles,omitempty" header:"POLICY FILES" show:"nolist"` + Attributes *json.RawMessage `json:"attributes,omitempty" header:"ATTRIBUTES" show:"nolist"` + AttributeRevisionID string `json:"attribute_revision_id,omitempty" header:"ATTRIBUTE REVISION ID"` +} + +type BootstrappingPolicyFile struct { + ID string `json:"id,omitempty" header:"ID"` + RevisionID string `json:"revision_id,omitempty" header:"REVISION ID"` + DownloadURL string `json:"download_url,omitempty" header:"DOWNLOAD URL"` +} + +type BootstrappingContinuousReport struct { + Stdout string `json:"stdout" header:"STDOUT"` +} + +type BootstrappingAppliedConfiguration struct { + StartedAt string `json:"started_at,omitempty" header:"STARTED AT"` + FinishedAt string `json:"finished_at,omitempty" header:"FINISHED AT"` + PolicyFileRevisionIDs string `json:"policyfile_revision_ids,omitempty" header:"POLICY FILE REVISION IDS" show:"nolist"` + AttributeRevisionID string `json:"attribute_revision_id,omitempty" header:"ATTRIBUTE REVISION ID"` +} diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go new file mode 100644 index 0000000..2128403 --- /dev/null +++ b/bootstrapping/bootstrapping.go @@ -0,0 +1,428 @@ +package bootstrapping + +import ( + "context" + "encoding/json" + "io/ioutil" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "time" + "math/rand" + + log "github.com/Sirupsen/logrus" + "github.com/codegangsta/cli" + "github.com/ingrammicro/concerto/api/blueprint" + "github.com/ingrammicro/concerto/api/types" + "github.com/ingrammicro/concerto/cmd" + "github.com/ingrammicro/concerto/utils" + "github.com/ingrammicro/concerto/utils/format" + "fmt" +) + +const ( + // DefaultTimingInterval Default period for looping + DefaultTimingInterval = 600 // 600 seconds = 10 minutes + DefaultRandomMaxThreshold = 6 // minutes + + // ProcessIDFile + ProcessIDFile = "imco-bootstrapping.pid" + + RetriesNumber = 5 + RetriesFactor = 3 + DefaultThresholdTime = 10 +) + +type bootstrappingStatus struct { + startedAt string + finishedAt string + policiesStatus []policyStatus + attributes attributesStatus +} +type attributesStatus struct { + revisionID string + filename string + filePath string + rawData *json.RawMessage +} + +type policyStatus struct { + id string + revisionID string + name string + filename string + tarballURL string + queryURL string + tarballPath string + folderPath string + + downloaded bool + uncompressed bool + executed bool + logged bool +} + +// Handle signals +func handleSysSignals(cancelFunc context.CancelFunc) { + log.Debug("handleSysSignals") + + gracefulStop := make(chan os.Signal, 1) + signal.Notify(gracefulStop, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL) + log.Debug("Ending, signal detected:", <-gracefulStop) + cancelFunc() +} + +// Returns the full path to the tmp folder joined with pid management file name +func getProcessIDFilePath() string { + return strings.Join([]string{os.TempDir(), string(os.PathSeparator), ProcessIDFile}, "") +} + +// Returns the full path to the tmp folder +func getProcessingFolderFilePath() string { + dir := strings.Join([]string{os.TempDir(), string(os.PathSeparator), "imco", string(os.PathSeparator)}, "") + os.Mkdir(dir, 0777) + return dir +} + +// Start the bootstrapping process +func start(c *cli.Context) error { + log.Debug("start") + + formatter := format.GetFormatter() + if err := utils.SetProcessIdToFile(getProcessIDFilePath()); err != nil { + formatter.PrintFatal("cannot create the pid file", err) + } + + timingInterval := c.Int64("time") + if !(timingInterval > 0) { + timingInterval = DefaultTimingInterval + } + // Adds a random value to the given timing interval! + // Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance) + timingInterval = timingInterval + int64(rand.New(rand.NewSource(time.Now().UnixNano())).Intn(DefaultRandomMaxThreshold)*60) + log.Debug("time interval:", timingInterval) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go handleSysSignals(cancel) + + bootstrappingRoutine(ctx, c, timingInterval) + + return nil +} + +// Stop the bootstrapping process +func stop(c *cli.Context) error { + log.Debug("cmdStop") + + formatter := format.GetFormatter() + if err := utils.StopProcess(getProcessIDFilePath()); err != nil { + formatter.PrintFatal("cannot stop the bootstrapping process", err) + } + + log.Info("Bootstrapping routine successfully stopped") + return nil +} + +// Main bootstrapping background routine +func bootstrappingRoutine(ctx context.Context, c *cli.Context, timingInterval int64) { + log.Debug("bootstrappingRoutine") + + //formatter := format.GetFormatter() + bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c) + commandProcessed := make(chan bool, 1) + + // initialization + currentTicker := time.NewTicker(time.Duration(timingInterval) * time.Second) + for { + go processingCommandRoutine(bootstrappingSvc, formatter, commandProcessed) + + log.Debug("Waiting...", currentTicker) + + select { + //case <-commandProcessed: + // isRunningCommandRoutine = false + // log.Debug("commandProcessed") + case <-currentTicker.C: + log.Debug("ticker") + case <-ctx.Done(): + log.Debug(ctx.Err()) + log.Debug("closing bootstrapping") + return + } + } +} + +// Subsidiary routine for commands processing +func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, commandProcessed chan bool) { + log.Debug("processingCommandRoutine") + + // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes + bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration() + if err != nil { + formatter.PrintError("Couldn't receive bootstrapping data", err) + } else { + if status == 200 { + bsStatus := new(bootstrappingStatus) + directoryPath := getProcessingFolderFilePath() + + // proto structures + if err := initializePrototype(directoryPath, bsConfiguration, bsStatus); err != nil { + formatter.PrintError("Cannot initialize the policy files prototypes", err) + } + + // TODO Currently as a step previous to process tarballs policies but this can be done as a part or processing, and using defer for removing files (tgz & folder!?) + // For every policyFile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... + if err := downloadPolicyFiles(bootstrappingSvc, bsStatus); err != nil { + formatter.PrintError("Cannot download the policy files", err) + } + + //... and clean off any tarball that is no longer needed. + if err := cleanObsoletePolicyFiles(directoryPath, bsStatus); err != nil { + formatter.PrintError("Cannot clean obsolete policy files", err) + } + + // Store the attributes as JSON in a file with name `attrs-.json` + if err := saveAttributes(bsStatus); err != nil { + formatter.PrintError("Cannot save policy files attributes ", err) + } + + // Process tarballs policies + if err := processPolicyFiles(bootstrappingSvc, bsStatus); err != nil { + formatter.PrintError("Cannot process policy files ", err) + } + + // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to + reportAppliedConfiguration(bootstrappingSvc, bsStatus) + } + } + + // TODO + commandProcessed <- true +} + +func initializePrototype(directoryPath string, bsConfiguration *types.BootstrappingConfiguration, bsStatus *bootstrappingStatus) error { + log.Debug("initializePrototype") + + log.Debug("Initializing bootstrapping structures") + + bsStatus.startedAt = time.Now().UTC().String() // TODO UTC? + + // Attributes + bsStatus.attributes.revisionID = bsConfiguration.AttributeRevisionID + bsStatus.attributes.filename = strings.Join([]string{"attrs-", bsStatus.attributes.revisionID, ".json"}, "") + bsStatus.attributes.filePath = strings.Join([]string{directoryPath, bsStatus.attributes.filename}, "") + bsStatus.attributes.rawData = bsConfiguration.Attributes + + // Policies + for _, policyFile := range bsConfiguration.PolicyFiles { + policyStatus := new(policyStatus) + policyStatus.id = policyFile.ID + policyStatus.revisionID = policyFile.RevisionID + policyStatus.name = strings.Join([]string{policyFile.ID, "-", policyFile.RevisionID}, "") + policyStatus.filename = strings.Join([]string{policyStatus.name, ".tgz"}, "") + policyStatus.tarballURL = policyFile.DownloadURL + + url, err := url.Parse(policyStatus.tarballURL) + if err != nil { + // TODO should it be an error? + return err + } + policyStatus.queryURL = strings.Join([]string{url.Path[1:], url.RawQuery}, "?") + + policyStatus.tarballPath = strings.Join([]string{directoryPath, policyStatus.filename}, "") + policyStatus.folderPath = strings.Join([]string{directoryPath, policyStatus.name}, "") + + bsStatus.policiesStatus = append(bsStatus.policiesStatus, *policyStatus) + } + log.Debug(bsStatus) + return nil +} + +// downloadPolicyFiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... +func downloadPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsStatus *bootstrappingStatus) error { + log.Debug("downloadPolicyFiles") + + for _, policyStatus := range bsStatus.policiesStatus { + log.Debug("Downloading: ", policyStatus.tarballURL) + _, status, err := bootstrappingSvc.DownloadPolicyFile(policyStatus.queryURL, policyStatus.tarballPath) + if err != nil { + return err + } + if status == 200 { + policyStatus.downloaded = true + log.Debug("Uncompressing: ", policyStatus.tarballPath) + if err = utils.Untar(policyStatus.tarballPath, policyStatus.folderPath); err != nil { + return err + } + policyStatus.uncompressed = true + } else { + // TODO should it be an error? + log.Error("Cannot download the policy file: ", policyStatus.filename) + } + } + return nil +} + +// cleanObsoletePolicyFiles cleans off any tarball that is no longer needed. +func cleanObsoletePolicyFiles(directoryPath string, bsStatus *bootstrappingStatus) error { + log.Debug("cleanObsoletePolicyFiles") + + // builds an array of currently processable files at this looping time + currentlyProcessableFiles := []string{bsStatus.attributes.filename} // saved attributes file name + for _, policyStatus := range bsStatus.policiesStatus { + currentlyProcessableFiles = append(currentlyProcessableFiles, policyStatus.filename) // Downloaded tgz file names + currentlyProcessableFiles = append(currentlyProcessableFiles, policyStatus.name) // Uncompressed folder names + } + + // evaluates working folder + files, err := ioutil.ReadDir(directoryPath) + if err != nil { + // TODO should it be an error? + log.Warn("Cannot read directory: ", directoryPath, err) + } + + // removes files not regarding to any of current policy files + for _, f := range files { + if !utils.Contains(currentlyProcessableFiles, f.Name()) { + log.Debug("Removing: ", f.Name()) + if err := utils.RemoveFileInfo(f, strings.Join([]string{directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { + // TODO should it be an error? + log.Warn("Cannot remove: ", f.Name(), err) + } + } + } + return nil // TODO should it be managed as error? +} + +// saveAttributes stores the attributes as JSON in a file with name `attrs-.json` +func saveAttributes(bsStatus *bootstrappingStatus) error { + log.Debug("saveAttributes") + + attrs, err := json.Marshal(bsStatus.attributes.rawData) + if err != nil { + return err + } + if err := ioutil.WriteFile(bsStatus.attributes.filePath, attrs, 0600); err != nil { + return err + } + return nil +} + + +//For every policy file, apply them doing the following: +// * Extract the tarball to a temporal work directory DIR +// * Run `cd DIR; chef-client -z -j path/to/attrs-.json` while sending the stderr and stdout in bunches of 10 lines to the +// platform via `POST /blueprint/bootstrap_logs` (this resource is a copy of POST /command_polling/bootstrap_logs used in the command_polling command). +// If the command returns with a non-zero value, stop applying policy files and continue with the next step. + +// TODO On the first iteration that applies successfully all policy files (runs all `chef-client -z` commands obtaining 0 return codes) only, run the boot scripts for the server by executing the `scripts boot` sub-command (as an external process). +// TODO Just a POC, an starging point. To be completed... +func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsStatus *bootstrappingStatus) error { + log.Debug("processPolicyFiles") + + // Run `cd DIR; chef-client -z -j path/to/attrs-.json` while sending the stderr and stdout in bunches of + // 10 lines to the platform via `POST /blueprint/bootstrap_logs` (this resource is a copy of POST /command_polling/bootstrap_logs used in + // the command_polling command). If the command returns with a non-zero value, stop applying policyfiles and continue with the next step. + for _, policyStatus := range bsStatus.policiesStatus { + log.Warn(policyStatus.folderPath) + + // TODO cd ; chef-client -z -j ` + command := "ping -c 100 8.8.8.8" + + // cli command threshold flag + thresholdTime := DefaultThresholdTime + log.Debug("Time threshold: ", thresholdTime) + + // Custom method for chunks processing + fn := func(chunk string) error { + log.Debug("sendChunks") + err := retry(RetriesNumber, time.Second, func() error { + log.Debug("Sending: ", chunk) + + commandIn := map[string]interface{}{ + "stdout": chunk, + } + + _, statusCode, err := bootstrappingSvc.ReportBootstrappingLog(&commandIn) + switch { + // 0<100 error cases?? + case statusCode == 0: + return fmt.Errorf("communication error %v %v", statusCode, err) + case statusCode >= 500: + return fmt.Errorf("server error %v %v", statusCode, err) + case statusCode >= 400: + return fmt.Errorf("client error %v %v", statusCode, err) + default: + return nil + } + }) + + if err != nil { + return fmt.Errorf("cannot send the chunk data, %v", err) + } + return nil + } + + // TODO This method was implemented in some moment based on nLines, nTime, bBytes? Currently only working with thresholdTime + exitCode, err := utils.RunContinuousCmd(fn, command, thresholdTime) + if err != nil { + log.Error("cannot process continuous report command", err) + } + + log.Info("completed: ", exitCode) + } + return nil +} + +func retry(attempts int, sleep time.Duration, fn func() error) error { + log.Debug("retry") + + if err := fn(); err != nil { + if attempts--; attempts > 0 { + log.Debug("Waiting to retry: ", sleep) + time.Sleep(sleep) + return retry(attempts, RetriesFactor*sleep, fn) + } + return err + } + return nil +} + +// reportAppliedConfiguration Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request +//The `policy file_revision_ids` field should have revision ids set only for those policy files successfully applied on the iteration, that is, +// it should not have any values set for those failing and those skipped because of a previous one failing. +func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsStatus *bootstrappingStatus) error { + log.Debug("reportAppliedConfiguration") + + bsStatus.finishedAt = time.Now().UTC().String() // TODO UTC? + + var policyfileRevisionIDs string + for _, policyStatus := range bsStatus.policiesStatus { + if policyStatus.executed { // only for policies successfully applied + appliedPolicyMap := map[string]string{policyStatus.id: policyStatus.revisionID} + appliedPolicyBytes, err := json.Marshal(appliedPolicyMap) + if err != nil { + // TODO should it be an error? + return err + } + policyfileRevisionIDs = strings.Join([]string{policyfileRevisionIDs, string(appliedPolicyBytes)}, "") + } + } + + payload := map[string]interface{}{ + "started_at": bsStatus.startedAt, + "finished_at": bsStatus.finishedAt, + "policyfile_revision_ids": policyfileRevisionIDs, + "attribute_revision_id": bsStatus.attributes.revisionID, + } + err := bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload) + if err != nil { + // TODO should it be an error? + return err + } + return nil +} diff --git a/bootstrapping/subcommands.go b/bootstrapping/subcommands.go new file mode 100644 index 0000000..69a45d0 --- /dev/null +++ b/bootstrapping/subcommands.go @@ -0,0 +1,27 @@ +package bootstrapping + +import ( + "github.com/codegangsta/cli" +) + +func SubCommands() []cli.Command { + return []cli.Command{ + { + Name: "start", + Usage: "Starts a bootstrapping routine to check and execute required activities", + Action: start, + Flags: []cli.Flag{ + cli.Int64Flag{ + Name: "time, t", + Usage: "bootstrapping time interval (seconds)", + Value: DefaultTimingInterval, + }, + }, + }, + { + Name: "stop", + Usage: "Stops the running bootstrapping process", + Action: stop, + }, + } +} diff --git a/cmd/bootstrapping_cmd.go b/cmd/bootstrapping_cmd.go new file mode 100644 index 0000000..9394b3f --- /dev/null +++ b/cmd/bootstrapping_cmd.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "github.com/codegangsta/cli" + "github.com/ingrammicro/concerto/api/blueprint" + "github.com/ingrammicro/concerto/utils" + "github.com/ingrammicro/concerto/utils/format" +) + +// WireUpBootstrapping prepares common resources to send request to API +func WireUpBootstrapping(c *cli.Context) (ds *blueprint.BootstrappingService, f format.Formatter) { + + f = format.GetFormatter() + + config, err := utils.GetConcertoConfig() + if err != nil { + f.PrintFatal("Couldn't wire up config", err) + } + hcs, err := utils.NewHTTPConcertoService(config) + if err != nil { + f.PrintFatal("Couldn't wire up concerto service", err) + } + ds, err = blueprint.NewBootstrappingService(hcs) + if err != nil { + f.PrintFatal("Couldn't wire up serverPlan service", err) + } + + return ds, f +} \ No newline at end of file diff --git a/main.go b/main.go index cd170f5..fb8383e 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + //"context" "fmt" "os" "sort" @@ -11,6 +12,7 @@ import ( "github.com/ingrammicro/concerto/blueprint/scripts" "github.com/ingrammicro/concerto/blueprint/services" "github.com/ingrammicro/concerto/blueprint/templates" + "github.com/ingrammicro/concerto/bootstrapping" "github.com/ingrammicro/concerto/brownfield" cl_prov "github.com/ingrammicro/concerto/cloud/cloud_providers" "github.com/ingrammicro/concerto/cloud/generic_images" @@ -68,6 +70,13 @@ var ServerCommands = []cli.Command{ cmdpolling.SubCommands(), ), }, + { + Name: "bootstrap", + Usage: "Manages bootstrapping commands", + Subcommands: append( + bootstrapping.SubCommands(), + ), + }, } var BlueprintCommands = []cli.Command{ diff --git a/utils/utils.go b/utils/utils.go index 8b8b2a6..ece700e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -13,6 +13,8 @@ import ( "github.com/codegangsta/cli" + "os/exec" + log "github.com/Sirupsen/logrus" ) @@ -60,6 +62,21 @@ func Unzip(archive, target string) error { return nil } +// TODO using cmd := exec.CommandContext(ctx,... +func Untar(source, target string) error { + + if err := os.MkdirAll(target, 0600); err != nil { + return err + } + + cmd := exec.Command("tar", "-xzf", source, "-C", target) + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + func ScrapeErrorMessage(message string, regExpression string) string { re, err := regexp.Compile(regExpression) @@ -218,3 +235,28 @@ func Subset(s1, s2 []string) bool { } return true } + +func RemoveFileInfo(fileInfo os.FileInfo, fileInfoName string) error { + if fileInfo.IsDir() { + d, err := os.Open(fileInfoName) + if err != nil { + return err + } + defer d.Close() + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + for _, name := range names { + err = os.RemoveAll(filepath.Join(fileInfoName, name)) + if err != nil { + return err + } + } + } + + if err := os.Remove(fileInfoName); err != nil { + return err + } + return nil +} diff --git a/utils/webservice.go b/utils/webservice.go index c356184..a30cab8 100644 --- a/utils/webservice.go +++ b/utils/webservice.go @@ -20,7 +20,7 @@ type ConcertoService interface { Put(path string, payload *map[string]interface{}) ([]byte, int, error) Delete(path string) ([]byte, int, error) Get(path string) ([]byte, int, error) - GetFile(path string, directoryPath string) (string, int, error) + GetFile(path string, directoryPath string, fileName string) (string, int, error) } // HTTPConcertoservice web service manager. @@ -198,7 +198,7 @@ func (hcs *HTTPConcertoservice) Get(path string) ([]byte, int, error) { } // GetFile sends GET request to Concerto API and receives a file -func (hcs *HTTPConcertoservice) GetFile(path string, directoryPath string) (string, int, error) { +func (hcs *HTTPConcertoservice) GetFile(path string, directoryPath string, fileName string) (string, int, error) { url, _, err := hcs.prepareCall(path, nil) if err != nil { @@ -214,14 +214,19 @@ func (hcs *HTTPConcertoservice) GetFile(path string, directoryPath string) (stri defer response.Body.Close() log.Debugf("Status code:%d message:%s", response.StatusCode, response.Status) - r, err := regexp.Compile("filename=\\\"([^\\\"]*){1}\\\"") - if err != nil { - return "", response.StatusCode, err - } + realFileName := "" + if directoryPath != "" && fileName == "" { + r, err := regexp.Compile("filename=\\\"([^\\\"]*){1}\\\"") + if err != nil { + return "", response.StatusCode, err + } - // TODO check errors - fileName := r.FindStringSubmatch(response.Header.Get("Content-Disposition"))[1] - realFileName := fmt.Sprintf("%s/%s", directoryPath, fileName) + // TODO check errors + fileName = r.FindStringSubmatch(response.Header.Get("Content-Disposition"))[1] + realFileName = fmt.Sprintf("%s/%s", directoryPath, fileName) + } else { + realFileName = fileName + } output, err := os.Create(realFileName) if err != nil { diff --git a/utils/webservice_mock.go b/utils/webservice_mock.go index a70f810..aa9adeb 100644 --- a/utils/webservice_mock.go +++ b/utils/webservice_mock.go @@ -34,7 +34,7 @@ func (m *MockConcertoService) Get(path string) ([]byte, int, error) { } // GetFile sends GET request to Concerto API and receives a file -func (m *MockConcertoService) GetFile(path string, directoryPath string) (string, int, error) { +func (m *MockConcertoService) GetFile(path string, directoryPath string, fileName string) (string, int, error) { args := m.Called(path, directoryPath) return args.String(0), args.Int(1), args.Error(2) } From be2f4f2690cadecb5d208533eed120c10234499d Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Mon, 4 Mar 2019 16:19:30 +0100 Subject: [PATCH 02/20] Renamed structs (issue #90) --- bootstrapping/bootstrapping.go | 151 ++++++++++++++++----------------- 1 file changed, 72 insertions(+), 79 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 2128403..441b787 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -35,24 +35,24 @@ const ( DefaultThresholdTime = 10 ) -type bootstrappingStatus struct { - startedAt string - finishedAt string - policiesStatus []policyStatus - attributes attributesStatus +type bootstrappingProcess struct { + startedAt string + finishedAt string + policyFiles []policyFile + attributes attributes } -type attributesStatus struct { +type attributes struct { revisionID string - filename string + fileName string filePath string rawData *json.RawMessage } -type policyStatus struct { +type policyFile struct { id string revisionID string name string - filename string + fileName string tarballURL string queryURL string tarballPath string @@ -133,19 +133,15 @@ func bootstrappingRoutine(ctx context.Context, c *cli.Context, timingInterval in //formatter := format.GetFormatter() bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c) - commandProcessed := make(chan bool, 1) // initialization currentTicker := time.NewTicker(time.Duration(timingInterval) * time.Second) for { - go processingCommandRoutine(bootstrappingSvc, formatter, commandProcessed) + go processingCommandRoutine(bootstrappingSvc, formatter) log.Debug("Waiting...", currentTicker) select { - //case <-commandProcessed: - // isRunningCommandRoutine = false - // log.Debug("commandProcessed") case <-currentTicker.C: log.Debug("ticker") case <-ctx.Done(): @@ -157,7 +153,7 @@ func bootstrappingRoutine(ctx context.Context, c *cli.Context, timingInterval in } // Subsidiary routine for commands processing -func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, commandProcessed chan bool) { +func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter) { log.Debug("processingCommandRoutine") // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes @@ -166,116 +162,114 @@ func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, formatter.PrintError("Couldn't receive bootstrapping data", err) } else { if status == 200 { - bsStatus := new(bootstrappingStatus) + bsProcess := new(bootstrappingProcess) directoryPath := getProcessingFolderFilePath() // proto structures - if err := initializePrototype(directoryPath, bsConfiguration, bsStatus); err != nil { + if err := initializePrototype(directoryPath, bsConfiguration, bsProcess); err != nil { formatter.PrintError("Cannot initialize the policy files prototypes", err) } // TODO Currently as a step previous to process tarballs policies but this can be done as a part or processing, and using defer for removing files (tgz & folder!?) // For every policyFile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... - if err := downloadPolicyFiles(bootstrappingSvc, bsStatus); err != nil { + if err := downloadPolicyFiles(bootstrappingSvc, bsProcess); err != nil { formatter.PrintError("Cannot download the policy files", err) } //... and clean off any tarball that is no longer needed. - if err := cleanObsoletePolicyFiles(directoryPath, bsStatus); err != nil { + if err := cleanObsoletePolicyFiles(directoryPath, bsProcess); err != nil { formatter.PrintError("Cannot clean obsolete policy files", err) } // Store the attributes as JSON in a file with name `attrs-.json` - if err := saveAttributes(bsStatus); err != nil { + if err := saveAttributes(bsProcess); err != nil { formatter.PrintError("Cannot save policy files attributes ", err) } // Process tarballs policies - if err := processPolicyFiles(bootstrappingSvc, bsStatus); err != nil { + if err := processPolicyFiles(bootstrappingSvc, bsProcess); err != nil { formatter.PrintError("Cannot process policy files ", err) } // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to - reportAppliedConfiguration(bootstrappingSvc, bsStatus) + reportAppliedConfiguration(bootstrappingSvc, bsProcess) } } - - // TODO - commandProcessed <- true } -func initializePrototype(directoryPath string, bsConfiguration *types.BootstrappingConfiguration, bsStatus *bootstrappingStatus) error { +func initializePrototype(directoryPath string, bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) error { log.Debug("initializePrototype") log.Debug("Initializing bootstrapping structures") - bsStatus.startedAt = time.Now().UTC().String() // TODO UTC? + bsProcess.startedAt = time.Now().UTC().String() // Attributes - bsStatus.attributes.revisionID = bsConfiguration.AttributeRevisionID - bsStatus.attributes.filename = strings.Join([]string{"attrs-", bsStatus.attributes.revisionID, ".json"}, "") - bsStatus.attributes.filePath = strings.Join([]string{directoryPath, bsStatus.attributes.filename}, "") - bsStatus.attributes.rawData = bsConfiguration.Attributes + bsProcess.attributes.revisionID = bsConfiguration.AttributeRevisionID + bsProcess.attributes.fileName = strings.Join([]string{"attrs-", bsProcess.attributes.revisionID, ".json"}, "") + bsProcess.attributes.filePath = strings.Join([]string{directoryPath, bsProcess.attributes.fileName}, "") + bsProcess.attributes.rawData = bsConfiguration.Attributes // Policies - for _, policyFile := range bsConfiguration.PolicyFiles { - policyStatus := new(policyStatus) - policyStatus.id = policyFile.ID - policyStatus.revisionID = policyFile.RevisionID - policyStatus.name = strings.Join([]string{policyFile.ID, "-", policyFile.RevisionID}, "") - policyStatus.filename = strings.Join([]string{policyStatus.name, ".tgz"}, "") - policyStatus.tarballURL = policyFile.DownloadURL - - url, err := url.Parse(policyStatus.tarballURL) + for _, bsConfPolicyFile := range bsConfiguration.PolicyFiles { + policyFile := new(policyFile) + policyFile.id = bsConfPolicyFile.ID + policyFile.revisionID = bsConfPolicyFile.RevisionID + + policyFile.name = strings.Join([]string{policyFile.id, "-", policyFile.revisionID}, "") + policyFile.fileName = strings.Join([]string{policyFile.name, ".tgz"}, "") + policyFile.tarballURL = bsConfPolicyFile.DownloadURL + + url, err := url.Parse(policyFile.tarballURL) if err != nil { // TODO should it be an error? return err } - policyStatus.queryURL = strings.Join([]string{url.Path[1:], url.RawQuery}, "?") + policyFile.queryURL = strings.Join([]string{url.Path[1:], url.RawQuery}, "?") - policyStatus.tarballPath = strings.Join([]string{directoryPath, policyStatus.filename}, "") - policyStatus.folderPath = strings.Join([]string{directoryPath, policyStatus.name}, "") + policyFile.tarballPath = strings.Join([]string{directoryPath, policyFile.fileName}, "") + policyFile.folderPath = strings.Join([]string{directoryPath, policyFile.name}, "") - bsStatus.policiesStatus = append(bsStatus.policiesStatus, *policyStatus) + bsProcess.policyFiles = append(bsProcess.policyFiles, *policyFile) } - log.Debug(bsStatus) + log.Debug(bsProcess) return nil } // downloadPolicyFiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... -func downloadPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsStatus *bootstrappingStatus) error { +func downloadPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { log.Debug("downloadPolicyFiles") - for _, policyStatus := range bsStatus.policiesStatus { - log.Debug("Downloading: ", policyStatus.tarballURL) - _, status, err := bootstrappingSvc.DownloadPolicyFile(policyStatus.queryURL, policyStatus.tarballPath) + for _, bsPolicyFile := range bsProcess.policyFiles { + log.Debug("Downloading: ", bsPolicyFile.tarballURL) + _, status, err := bootstrappingSvc.DownloadPolicyFile(bsPolicyFile.queryURL, bsPolicyFile.tarballPath) if err != nil { return err } if status == 200 { - policyStatus.downloaded = true - log.Debug("Uncompressing: ", policyStatus.tarballPath) - if err = utils.Untar(policyStatus.tarballPath, policyStatus.folderPath); err != nil { + bsPolicyFile.downloaded = true + log.Debug("Uncompressing: ", bsPolicyFile.tarballPath) + if err = utils.Untar(bsPolicyFile.tarballPath, bsPolicyFile.folderPath); err != nil { return err } - policyStatus.uncompressed = true + bsPolicyFile.uncompressed = true } else { // TODO should it be an error? - log.Error("Cannot download the policy file: ", policyStatus.filename) + log.Error("Cannot download the policy file: ", bsPolicyFile.fileName) } } return nil } // cleanObsoletePolicyFiles cleans off any tarball that is no longer needed. -func cleanObsoletePolicyFiles(directoryPath string, bsStatus *bootstrappingStatus) error { +func cleanObsoletePolicyFiles(directoryPath string, bsProcess *bootstrappingProcess) error { log.Debug("cleanObsoletePolicyFiles") // builds an array of currently processable files at this looping time - currentlyProcessableFiles := []string{bsStatus.attributes.filename} // saved attributes file name - for _, policyStatus := range bsStatus.policiesStatus { - currentlyProcessableFiles = append(currentlyProcessableFiles, policyStatus.filename) // Downloaded tgz file names - currentlyProcessableFiles = append(currentlyProcessableFiles, policyStatus.name) // Uncompressed folder names + currentlyProcessableFiles := []string{bsProcess.attributes.fileName} // saved attributes file name + for _, bsPolicyFile := range bsProcess.policyFiles { + currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.fileName) // Downloaded tgz file names + currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.name) // Uncompressed folder names } // evaluates working folder @@ -289,7 +283,7 @@ func cleanObsoletePolicyFiles(directoryPath string, bsStatus *bootstrappingStatu for _, f := range files { if !utils.Contains(currentlyProcessableFiles, f.Name()) { log.Debug("Removing: ", f.Name()) - if err := utils.RemoveFileInfo(f, strings.Join([]string{directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { + if err := os.RemoveAll(strings.Join([]string{directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { // TODO should it be an error? log.Warn("Cannot remove: ", f.Name(), err) } @@ -299,20 +293,19 @@ func cleanObsoletePolicyFiles(directoryPath string, bsStatus *bootstrappingStatu } // saveAttributes stores the attributes as JSON in a file with name `attrs-.json` -func saveAttributes(bsStatus *bootstrappingStatus) error { +func saveAttributes(bsProcess *bootstrappingProcess) error { log.Debug("saveAttributes") - attrs, err := json.Marshal(bsStatus.attributes.rawData) + attrs, err := json.Marshal(bsProcess.attributes.rawData) if err != nil { return err } - if err := ioutil.WriteFile(bsStatus.attributes.filePath, attrs, 0600); err != nil { + if err := ioutil.WriteFile(bsProcess.attributes.filePath, attrs, 0600); err != nil { return err } return nil } - //For every policy file, apply them doing the following: // * Extract the tarball to a temporal work directory DIR // * Run `cd DIR; chef-client -z -j path/to/attrs-.json` while sending the stderr and stdout in bunches of 10 lines to the @@ -321,16 +314,16 @@ func saveAttributes(bsStatus *bootstrappingStatus) error { // TODO On the first iteration that applies successfully all policy files (runs all `chef-client -z` commands obtaining 0 return codes) only, run the boot scripts for the server by executing the `scripts boot` sub-command (as an external process). // TODO Just a POC, an starging point. To be completed... -func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsStatus *bootstrappingStatus) error { +func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { log.Debug("processPolicyFiles") // Run `cd DIR; chef-client -z -j path/to/attrs-.json` while sending the stderr and stdout in bunches of // 10 lines to the platform via `POST /blueprint/bootstrap_logs` (this resource is a copy of POST /command_polling/bootstrap_logs used in // the command_polling command). If the command returns with a non-zero value, stop applying policyfiles and continue with the next step. - for _, policyStatus := range bsStatus.policiesStatus { - log.Warn(policyStatus.folderPath) + for _, bsPolicyFile := range bsProcess.policyFiles { + log.Warn(bsPolicyFile.folderPath) - // TODO cd ; chef-client -z -j ` + // TODO cd ; chef-client -z -j ` command := "ping -c 100 8.8.8.8" // cli command threshold flag @@ -395,29 +388,29 @@ func retry(attempts int, sleep time.Duration, fn func() error) error { // reportAppliedConfiguration Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request //The `policy file_revision_ids` field should have revision ids set only for those policy files successfully applied on the iteration, that is, // it should not have any values set for those failing and those skipped because of a previous one failing. -func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsStatus *bootstrappingStatus) error { +func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { log.Debug("reportAppliedConfiguration") - bsStatus.finishedAt = time.Now().UTC().String() // TODO UTC? + bsProcess.finishedAt = time.Now().UTC().String() - var policyfileRevisionIDs string - for _, policyStatus := range bsStatus.policiesStatus { - if policyStatus.executed { // only for policies successfully applied - appliedPolicyMap := map[string]string{policyStatus.id: policyStatus.revisionID} + var policyFileRevisionIDs string + for _, bsPolicyFile := range bsProcess.policyFiles { + if bsPolicyFile.executed { // only for policies successfully applied + appliedPolicyMap := map[string]string{bsPolicyFile.id: bsPolicyFile.revisionID} appliedPolicyBytes, err := json.Marshal(appliedPolicyMap) if err != nil { // TODO should it be an error? return err } - policyfileRevisionIDs = strings.Join([]string{policyfileRevisionIDs, string(appliedPolicyBytes)}, "") + policyFileRevisionIDs = strings.Join([]string{policyFileRevisionIDs, string(appliedPolicyBytes)}, "") } } payload := map[string]interface{}{ - "started_at": bsStatus.startedAt, - "finished_at": bsStatus.finishedAt, - "policyfile_revision_ids": policyfileRevisionIDs, - "attribute_revision_id": bsStatus.attributes.revisionID, + "started_at": bsProcess.startedAt, + "finished_at": bsProcess.finishedAt, + "policyfile_revision_ids": policyFileRevisionIDs, + "attribute_revision_id": bsProcess.attributes.revisionID, } err := bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload) if err != nil { From d87d56ee41c453f9232d729c110aec973c9d933d Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Mon, 4 Mar 2019 17:06:28 +0100 Subject: [PATCH 03/20] Refactorized mechanism to download file (issue #90) --- api/blueprint/bootstrapping_api.go | 2 +- utils/webservice.go | 19 +++---------------- utils/webservice_mock.go | 4 ++-- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/api/blueprint/bootstrapping_api.go b/api/blueprint/bootstrapping_api.go index fe45013..244776a 100644 --- a/api/blueprint/bootstrapping_api.go +++ b/api/blueprint/bootstrapping_api.go @@ -84,7 +84,7 @@ func (bs *BootstrappingService) ReportBootstrappingLog(BootstrappingContinuousRe func (bs *BootstrappingService) DownloadPolicyFile(url string, filePath string) (realFileName string, status int, err error) { log.Debug("DownloadPolicyFile") - realFileName, status, err = bs.concertoService.GetFile(url, "", filePath) + realFileName, status, err = bs.concertoService.GetFile(url, filePath) if err != nil { return realFileName, status, err } diff --git a/utils/webservice.go b/utils/webservice.go index a30cab8..6e2952d 100644 --- a/utils/webservice.go +++ b/utils/webservice.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "net/http" "os" - "regexp" "strings" log "github.com/Sirupsen/logrus" @@ -20,7 +19,7 @@ type ConcertoService interface { Put(path string, payload *map[string]interface{}) ([]byte, int, error) Delete(path string) ([]byte, int, error) Get(path string) ([]byte, int, error) - GetFile(path string, directoryPath string, fileName string) (string, int, error) + GetFile(path string, filePath string) (string, int, error) } // HTTPConcertoservice web service manager. @@ -198,7 +197,7 @@ func (hcs *HTTPConcertoservice) Get(path string) ([]byte, int, error) { } // GetFile sends GET request to Concerto API and receives a file -func (hcs *HTTPConcertoservice) GetFile(path string, directoryPath string, fileName string) (string, int, error) { +func (hcs *HTTPConcertoservice) GetFile(path string, filePath string) (string, int, error) { url, _, err := hcs.prepareCall(path, nil) if err != nil { @@ -214,19 +213,7 @@ func (hcs *HTTPConcertoservice) GetFile(path string, directoryPath string, fileN defer response.Body.Close() log.Debugf("Status code:%d message:%s", response.StatusCode, response.Status) - realFileName := "" - if directoryPath != "" && fileName == "" { - r, err := regexp.Compile("filename=\\\"([^\\\"]*){1}\\\"") - if err != nil { - return "", response.StatusCode, err - } - - // TODO check errors - fileName = r.FindStringSubmatch(response.Header.Get("Content-Disposition"))[1] - realFileName = fmt.Sprintf("%s/%s", directoryPath, fileName) - } else { - realFileName = fileName - } + realFileName := filePath output, err := os.Create(realFileName) if err != nil { diff --git a/utils/webservice_mock.go b/utils/webservice_mock.go index aa9adeb..75dd0bc 100644 --- a/utils/webservice_mock.go +++ b/utils/webservice_mock.go @@ -34,7 +34,7 @@ func (m *MockConcertoService) Get(path string) ([]byte, int, error) { } // GetFile sends GET request to Concerto API and receives a file -func (m *MockConcertoService) GetFile(path string, directoryPath string, fileName string) (string, int, error) { - args := m.Called(path, directoryPath) +func (m *MockConcertoService) GetFile(path string, filePath string) (string, int, error) { + args := m.Called(path, filePath) return args.String(0), args.Int(1), args.Error(2) } From 87af58e46b6d309dca15687c0bac9505a4446fc1 Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Mon, 4 Mar 2019 17:37:28 +0100 Subject: [PATCH 04/20] Changed strategy for cleaning (issue #90) --- bootstrapping/bootstrapping.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 441b787..04c8008 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -265,6 +265,13 @@ func downloadPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsPro func cleanObsoletePolicyFiles(directoryPath string, bsProcess *bootstrappingProcess) error { log.Debug("cleanObsoletePolicyFiles") + // evaluates working folder + deletableFiles, err := ioutil.ReadDir(directoryPath) + if err != nil { + // TODO should it be an error? + log.Warn("Cannot read directory: ", directoryPath, err) + } + // builds an array of currently processable files at this looping time currentlyProcessableFiles := []string{bsProcess.attributes.fileName} // saved attributes file name for _, bsPolicyFile := range bsProcess.policyFiles { @@ -272,15 +279,7 @@ func cleanObsoletePolicyFiles(directoryPath string, bsProcess *bootstrappingProc currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.name) // Uncompressed folder names } - // evaluates working folder - files, err := ioutil.ReadDir(directoryPath) - if err != nil { - // TODO should it be an error? - log.Warn("Cannot read directory: ", directoryPath, err) - } - - // removes files not regarding to any of current policy files - for _, f := range files { + for _, f := range deletableFiles { if !utils.Contains(currentlyProcessableFiles, f.Name()) { log.Debug("Removing: ", f.Name()) if err := os.RemoveAll(strings.Join([]string{directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { @@ -289,6 +288,7 @@ func cleanObsoletePolicyFiles(directoryPath string, bsProcess *bootstrappingProc } } } + return nil // TODO should it be managed as error? } From d25d8e698852b049e517413c669982522e144c53 Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Tue, 5 Mar 2019 14:49:22 +0100 Subject: [PATCH 05/20] General review and refactoring task (issue #90) - Refactored - Implemented lines threshold for reporting while running continuos command - Most errors in looping are only traced in order to avoid break the task - Added chef commands - Implemented concurrency control --- api/blueprint/bootstrapping_api.go | 2 +- bootstrapping/bootstrapping.go | 204 +++++++++++++---------------- bootstrapping/subcommands.go | 7 +- cmdpolling/continuousreport.go | 19 +-- utils/exec.go | 30 ++++- 5 files changed, 125 insertions(+), 137 deletions(-) diff --git a/api/blueprint/bootstrapping_api.go b/api/blueprint/bootstrapping_api.go index 244776a..2b9f859 100644 --- a/api/blueprint/bootstrapping_api.go +++ b/api/blueprint/bootstrapping_api.go @@ -80,7 +80,7 @@ func (bs *BootstrappingService) ReportBootstrappingLog(BootstrappingContinuousRe } -// +// DownloadPolicyFile gets a file from given url saving file into given file path func (bs *BootstrappingService) DownloadPolicyFile(url string, filePath string) (realFileName string, status int, err error) { log.Debug("DownloadPolicyFile") diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 04c8008..6975c35 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -11,6 +11,8 @@ import ( "syscall" "time" "math/rand" + "fmt" + "runtime" log "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" @@ -19,27 +21,24 @@ import ( "github.com/ingrammicro/concerto/cmd" "github.com/ingrammicro/concerto/utils" "github.com/ingrammicro/concerto/utils/format" - "fmt" ) const ( - // DefaultTimingInterval Default period for looping + //DefaultTimingInterval Default period for looping DefaultTimingInterval = 600 // 600 seconds = 10 minutes DefaultRandomMaxThreshold = 6 // minutes - - // ProcessIDFile - ProcessIDFile = "imco-bootstrapping.pid" - - RetriesNumber = 5 - RetriesFactor = 3 - DefaultThresholdTime = 10 + DefaultThresholdLines = 10 + ProcessIDFile = "imco-bootstrapping.pid" + RetriesNumber = 5 ) type bootstrappingProcess struct { - startedAt string - finishedAt string - policyFiles []policyFile - attributes attributes + startedAt string + finishedAt string + policyFiles []policyFile + attributes attributes + thresholdLines int + directoryPath string } type attributes struct { revisionID string @@ -102,14 +101,20 @@ func start(c *cli.Context) error { // Adds a random value to the given timing interval! // Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance) timingInterval = timingInterval + int64(rand.New(rand.NewSource(time.Now().UnixNano())).Intn(DefaultRandomMaxThreshold)*60) - log.Debug("time interval:", timingInterval) + log.Debug("time interval: ", timingInterval) + + thresholdLines := c.Int("lines") + if !(thresholdLines > 0) { + thresholdLines = DefaultThresholdLines + } + log.Debug("routine lines threshold: ", thresholdLines) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go handleSysSignals(cancel) - bootstrappingRoutine(ctx, c, timingInterval) + bootstrappingRoutine(ctx, c, timingInterval, thresholdLines) return nil } @@ -128,20 +133,27 @@ func stop(c *cli.Context) error { } // Main bootstrapping background routine -func bootstrappingRoutine(ctx context.Context, c *cli.Context, timingInterval int64) { +func bootstrappingRoutine(ctx context.Context, c *cli.Context, timingInterval int64, thresholdLines int) { log.Debug("bootstrappingRoutine") - //formatter := format.GetFormatter() bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c) - - // initialization + commandProcessed := make(chan bool, 1) + isRunningCommandRoutine := false currentTicker := time.NewTicker(time.Duration(timingInterval) * time.Second) for { - go processingCommandRoutine(bootstrappingSvc, formatter) + if !isRunningCommandRoutine { + isRunningCommandRoutine = true + go processingCommandRoutine(bootstrappingSvc, formatter, thresholdLines, commandProcessed) + } - log.Debug("Waiting...", currentTicker) + log.Debug("waiting...", currentTicker) select { + case <-commandProcessed: + isRunningCommandRoutine = false + currentTicker.Stop() + currentTicker = time.NewTicker(time.Duration(timingInterval) * time.Second) + log.Debug("command processed") case <-currentTicker.C: log.Debug("ticker") case <-ctx.Done(): @@ -153,61 +165,54 @@ func bootstrappingRoutine(ctx context.Context, c *cli.Context, timingInterval in } // Subsidiary routine for commands processing -func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter) { +func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int, commandProcessed chan bool) { log.Debug("processingCommandRoutine") // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration() if err != nil { - formatter.PrintError("Couldn't receive bootstrapping data", err) + formatter.PrintError("couldn't receive bootstrapping data", err) } else { if status == 200 { bsProcess := new(bootstrappingProcess) - directoryPath := getProcessingFolderFilePath() + // Starting time + bsProcess.startedAt = time.Now().UTC().String() + bsProcess.thresholdLines = thresholdLines + bsProcess.directoryPath = getProcessingFolderFilePath() // proto structures - if err := initializePrototype(directoryPath, bsConfiguration, bsProcess); err != nil { - formatter.PrintError("Cannot initialize the policy files prototypes", err) - } + initializePrototype(bsConfiguration, bsProcess) - // TODO Currently as a step previous to process tarballs policies but this can be done as a part or processing, and using defer for removing files (tgz & folder!?) // For every policyFile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... - if err := downloadPolicyFiles(bootstrappingSvc, bsProcess); err != nil { - formatter.PrintError("Cannot download the policy files", err) - } + downloadPolicyFiles(bootstrappingSvc, bsProcess) //... and clean off any tarball that is no longer needed. - if err := cleanObsoletePolicyFiles(directoryPath, bsProcess); err != nil { - formatter.PrintError("Cannot clean obsolete policy files", err) - } + cleanObsoletePolicyFiles(bsProcess) // Store the attributes as JSON in a file with name `attrs-.json` - if err := saveAttributes(bsProcess); err != nil { - formatter.PrintError("Cannot save policy files attributes ", err) - } + saveAttributes(bsProcess) // Process tarballs policies - if err := processPolicyFiles(bootstrappingSvc, bsProcess); err != nil { - formatter.PrintError("Cannot process policy files ", err) - } + processPolicyFiles(bootstrappingSvc, bsProcess) + + // Finishing time + bsProcess.finishedAt = time.Now().UTC().String() // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to + log.Debug("reporting applied policy files") reportAppliedConfiguration(bootstrappingSvc, bsProcess) } } + commandProcessed <- true } -func initializePrototype(directoryPath string, bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) error { +func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) { log.Debug("initializePrototype") - log.Debug("Initializing bootstrapping structures") - - bsProcess.startedAt = time.Now().UTC().String() - // Attributes bsProcess.attributes.revisionID = bsConfiguration.AttributeRevisionID bsProcess.attributes.fileName = strings.Join([]string{"attrs-", bsProcess.attributes.revisionID, ".json"}, "") - bsProcess.attributes.filePath = strings.Join([]string{directoryPath, bsProcess.attributes.fileName}, "") + bsProcess.attributes.filePath = strings.Join([]string{bsProcess.directoryPath, bsProcess.attributes.fileName}, "") bsProcess.attributes.rawData = bsConfiguration.Attributes // Policies @@ -220,56 +225,54 @@ func initializePrototype(directoryPath string, bsConfiguration *types.Bootstrapp policyFile.fileName = strings.Join([]string{policyFile.name, ".tgz"}, "") policyFile.tarballURL = bsConfPolicyFile.DownloadURL - url, err := url.Parse(policyFile.tarballURL) - if err != nil { - // TODO should it be an error? - return err + if policyFile.tarballURL != "" { + url, err := url.Parse(policyFile.tarballURL) + if err != nil { + log.Errorf("cannot parse the tarball policy file url: %s [%s]", policyFile.tarballURL, err) + } else { + policyFile.queryURL = strings.Join([]string{url.Path[1:], url.RawQuery}, "?") + } } - policyFile.queryURL = strings.Join([]string{url.Path[1:], url.RawQuery}, "?") - policyFile.tarballPath = strings.Join([]string{directoryPath, policyFile.fileName}, "") - policyFile.folderPath = strings.Join([]string{directoryPath, policyFile.name}, "") + policyFile.tarballPath = strings.Join([]string{bsProcess.directoryPath, policyFile.fileName}, "") + policyFile.folderPath = strings.Join([]string{bsProcess.directoryPath, policyFile.name}, "") bsProcess.policyFiles = append(bsProcess.policyFiles, *policyFile) } log.Debug(bsProcess) - return nil } // downloadPolicyFiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... -func downloadPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { +func downloadPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { log.Debug("downloadPolicyFiles") for _, bsPolicyFile := range bsProcess.policyFiles { - log.Debug("Downloading: ", bsPolicyFile.tarballURL) + log.Debug("downloading: ", bsPolicyFile.tarballURL) _, status, err := bootstrappingSvc.DownloadPolicyFile(bsPolicyFile.queryURL, bsPolicyFile.tarballPath) if err != nil { - return err + log.Errorf("cannot download the tarball policy file: %s [%s]", bsPolicyFile.tarballURL, err) } if status == 200 { bsPolicyFile.downloaded = true - log.Debug("Uncompressing: ", bsPolicyFile.tarballPath) + log.Debug("decompressing: ", bsPolicyFile.tarballPath) if err = utils.Untar(bsPolicyFile.tarballPath, bsPolicyFile.folderPath); err != nil { - return err + log.Errorf("cannot decompress the tarball policy file: %s [%s]", bsPolicyFile.tarballPath, err) } bsPolicyFile.uncompressed = true } else { - // TODO should it be an error? - log.Error("Cannot download the policy file: ", bsPolicyFile.fileName) + log.Errorf("cannot download the policy file: %v", bsPolicyFile.fileName) } } - return nil } // cleanObsoletePolicyFiles cleans off any tarball that is no longer needed. -func cleanObsoletePolicyFiles(directoryPath string, bsProcess *bootstrappingProcess) error { +func cleanObsoletePolicyFiles(bsProcess *bootstrappingProcess) { log.Debug("cleanObsoletePolicyFiles") // evaluates working folder - deletableFiles, err := ioutil.ReadDir(directoryPath) + deletableFiles, err := ioutil.ReadDir(bsProcess.directoryPath) if err != nil { - // TODO should it be an error? - log.Warn("Cannot read directory: ", directoryPath, err) + log.Errorf("cannot read directory: %s [%s]", bsProcess.directoryPath, err) } // builds an array of currently processable files at this looping time @@ -279,31 +282,28 @@ func cleanObsoletePolicyFiles(directoryPath string, bsProcess *bootstrappingProc currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.name) // Uncompressed folder names } + // removes from deletableFiles array the policy files currently applied for _, f := range deletableFiles { if !utils.Contains(currentlyProcessableFiles, f.Name()) { - log.Debug("Removing: ", f.Name()) - if err := os.RemoveAll(strings.Join([]string{directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { - // TODO should it be an error? - log.Warn("Cannot remove: ", f.Name(), err) + log.Debug("removing: ", f.Name()) + if err := os.RemoveAll(strings.Join([]string{bsProcess.directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { + log.Errorf("cannot remove: %s [%s]", f.Name(), err) } } } - - return nil // TODO should it be managed as error? } // saveAttributes stores the attributes as JSON in a file with name `attrs-.json` -func saveAttributes(bsProcess *bootstrappingProcess) error { +func saveAttributes(bsProcess *bootstrappingProcess) { log.Debug("saveAttributes") attrs, err := json.Marshal(bsProcess.attributes.rawData) if err != nil { - return err + log.Errorf("cannot process policies attributes: %s [%s]", bsProcess.attributes.revisionID, err) } if err := ioutil.WriteFile(bsProcess.attributes.filePath, attrs, 0600); err != nil { - return err + log.Errorf("cannot save policies attributes: %s [%s]", bsProcess.attributes.revisionID, err) } - return nil } //For every policy file, apply them doing the following: @@ -314,26 +314,29 @@ func saveAttributes(bsProcess *bootstrappingProcess) error { // TODO On the first iteration that applies successfully all policy files (runs all `chef-client -z` commands obtaining 0 return codes) only, run the boot scripts for the server by executing the `scripts boot` sub-command (as an external process). // TODO Just a POC, an starging point. To be completed... -func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { +func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { log.Debug("processPolicyFiles") // Run `cd DIR; chef-client -z -j path/to/attrs-.json` while sending the stderr and stdout in bunches of // 10 lines to the platform via `POST /blueprint/bootstrap_logs` (this resource is a copy of POST /command_polling/bootstrap_logs used in // the command_polling command). If the command returns with a non-zero value, stop applying policyfiles and continue with the next step. for _, bsPolicyFile := range bsProcess.policyFiles { - log.Warn(bsPolicyFile.folderPath) - - // TODO cd ; chef-client -z -j ` - command := "ping -c 100 8.8.8.8" + command := strings.Join([]string{"cd", bsPolicyFile.folderPath}, " ") + if runtime.GOOS == "windows" { + command = strings.Join([]string{command, "SET \"PATH=%PATH%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\""}, ";") + } + command = strings.Join([]string{command, strings.Join([]string{"chef-client -z -j", bsProcess.attributes.filePath}, " ")}, ";") + log.Debug(command) - // cli command threshold flag - thresholdTime := DefaultThresholdTime - log.Debug("Time threshold: ", thresholdTime) + // ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** + // TODO ** TO BE REMOVED** !!! for debugging purposes, overriding real command + command = "ping -c 100 8.8.8.8" + // ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** // Custom method for chunks processing fn := func(chunk string) error { log.Debug("sendChunks") - err := retry(RetriesNumber, time.Second, func() error { + err := utils.Retry(RetriesNumber, time.Second, func() error { log.Debug("Sending: ", chunk) commandIn := map[string]interface{}{ @@ -360,47 +363,28 @@ func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc return nil } - // TODO This method was implemented in some moment based on nLines, nTime, bBytes? Currently only working with thresholdTime - exitCode, err := utils.RunContinuousCmd(fn, command, thresholdTime) + exitCode, err := utils.RunContinuousCmd(fn, command, -1, bsProcess.thresholdLines) if err != nil { - log.Error("cannot process continuous report command", err) + log.Errorf("cannot process continuous report command [%s]", err) } log.Info("completed: ", exitCode) } - return nil -} - -func retry(attempts int, sleep time.Duration, fn func() error) error { - log.Debug("retry") - - if err := fn(); err != nil { - if attempts--; attempts > 0 { - log.Debug("Waiting to retry: ", sleep) - time.Sleep(sleep) - return retry(attempts, RetriesFactor*sleep, fn) - } - return err - } - return nil } // reportAppliedConfiguration Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request //The `policy file_revision_ids` field should have revision ids set only for those policy files successfully applied on the iteration, that is, // it should not have any values set for those failing and those skipped because of a previous one failing. -func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { +func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { log.Debug("reportAppliedConfiguration") - bsProcess.finishedAt = time.Now().UTC().String() - var policyFileRevisionIDs string for _, bsPolicyFile := range bsProcess.policyFiles { if bsPolicyFile.executed { // only for policies successfully applied appliedPolicyMap := map[string]string{bsPolicyFile.id: bsPolicyFile.revisionID} appliedPolicyBytes, err := json.Marshal(appliedPolicyMap) if err != nil { - // TODO should it be an error? - return err + log.Errorf("corrupted candidates policies map [%s]", err) } policyFileRevisionIDs = strings.Join([]string{policyFileRevisionIDs, string(appliedPolicyBytes)}, "") } @@ -414,8 +398,6 @@ func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService } err := bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload) if err != nil { - // TODO should it be an error? - return err + log.Errorf("cannot report applied configuration [%s]", err) } - return nil } diff --git a/bootstrapping/subcommands.go b/bootstrapping/subcommands.go index 69a45d0..88d5db7 100644 --- a/bootstrapping/subcommands.go +++ b/bootstrapping/subcommands.go @@ -13,9 +13,14 @@ func SubCommands() []cli.Command { Flags: []cli.Flag{ cli.Int64Flag{ Name: "time, t", - Usage: "bootstrapping time interval (seconds)", + Usage: "Bootstrapping time interval (seconds)", Value: DefaultTimingInterval, }, + cli.IntFlag{ + Name: "lines, l", + Usage: "Maximum lines threshold per response chunk", + Value: DefaultThresholdLines, + }, }, }, { diff --git a/cmdpolling/continuousreport.go b/cmdpolling/continuousreport.go index 6183388..568314f 100644 --- a/cmdpolling/continuousreport.go +++ b/cmdpolling/continuousreport.go @@ -15,7 +15,6 @@ import ( const ( RetriesNumber = 5 - RetriesFactor = 3 DefaultThresholdTime = 10 ) @@ -43,7 +42,7 @@ func cmdContinuousReportRun(c *cli.Context) error { // Custom method for chunks processing fn := func(chunk string) error { log.Debug("sendChunks") - err := retry(RetriesNumber, time.Second, func() error { + err := utils.Retry(RetriesNumber, time.Second, func() error { log.Debug("Sending: ", chunk) commandIn := map[string]interface{}{ @@ -70,7 +69,7 @@ func cmdContinuousReportRun(c *cli.Context) error { return nil } - exitCode, err := utils.RunContinuousCmd(fn, cmdArg, thresholdTime) + exitCode, err := utils.RunContinuousCmd(fn, cmdArg, thresholdTime, -1) if err != nil { formatter.PrintFatal("cannot process continuous report command", err) } @@ -79,17 +78,3 @@ func cmdContinuousReportRun(c *cli.Context) error { os.Exit(exitCode) return nil } - -func retry(attempts int, sleep time.Duration, fn func() error) error { - log.Debug("retry") - - if err := fn(); err != nil { - if attempts--; attempts > 0 { - log.Debug("Waiting to retry: ", sleep) - time.Sleep(sleep) - return retry(attempts, RetriesFactor*sleep, fn) - } - return err - } - return nil -} diff --git a/utils/exec.go b/utils/exec.go index 7268bee..f16fedc 100644 --- a/utils/exec.go +++ b/utils/exec.go @@ -19,6 +19,7 @@ import ( const ( TimeStampLayout = "2006-01-02T15:04:05.000000-07:00" TimeLayoutYYYYMMDDHHMMSS = "20060102150405" + RetriesFactor = 3 ) func extractExitCode(err error) int { @@ -234,7 +235,9 @@ func RunTracedCmd(command string) (exitCode int, stdOut string, stdErr string, s return } -func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime int) (int, error) { +// thresholdTime > 0 continuous report +// thresholdLines > 0 bootstrapping +func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime int, thresholdLines int) (int, error) { log.Debug("RunContinuousCmd") // Saves script/command in a temp file @@ -256,20 +259,19 @@ func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime } chunk := "" - nTime := 0 + nLines, nTime := 0, 0 timeStart := time.Now() scanner := bufio.NewScanner(bufio.NewReader(stdout)) for scanner.Scan() { chunk = strings.Join([]string{chunk, scanner.Text(), "\n"}, "") + nLines++ nTime = int(time.Now().Sub(timeStart).Seconds()) - if nTime >= thresholdTime { - if err := fn(chunk); err != nil { - nTime = 0 - } else { + if (thresholdTime > 0 && nTime >= thresholdTime) || (thresholdLines > 0 && nLines >= thresholdLines) { + if err := fn(chunk); err == nil { chunk = "" - nTime = 0 } + nLines, nTime = 0, 0 timeStart = time.Now() } } @@ -291,3 +293,17 @@ func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime return exitCode, nil } + +func Retry(attempts int, sleep time.Duration, fn func() error) error { + log.Debug("Retry") + + if err := fn(); err != nil { + if attempts--; attempts > 0 { + log.Debug("Waiting to retry: ", sleep) + time.Sleep(sleep) + return Retry(attempts, RetriesFactor*sleep, fn) + } + return err + } + return nil +} From 9201451be7aef5b5e55d6c016a585ac765b9ccc9 Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Tue, 5 Mar 2019 20:27:18 +0100 Subject: [PATCH 06/20] Updated routine approach (issue #90) In addition: - Implemented success case management for booting - Redefined interval time parameters --- bootstrapping/bootstrapping.go | 135 ++++++++++++++++++--------------- bootstrapping/subcommands.go | 9 ++- 2 files changed, 79 insertions(+), 65 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 6975c35..6ec0199 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -25,17 +25,17 @@ import ( const ( //DefaultTimingInterval Default period for looping - DefaultTimingInterval = 600 // 600 seconds = 10 minutes - DefaultRandomMaxThreshold = 6 // minutes - DefaultThresholdLines = 10 - ProcessIDFile = "imco-bootstrapping.pid" - RetriesNumber = 5 + DefaultTimingInterval = 600 // 600 seconds = 10 minutes + DefaultTimingSplay = 360 // seconds + DefaultThresholdLines = 10 + ProcessIDFile = "imco-bootstrapping.pid" + RetriesNumber = 5 ) type bootstrappingProcess struct { startedAt string finishedAt string - policyFiles []policyFile + policyFiles []*policyFile attributes attributes thresholdLines int directoryPath string @@ -60,9 +60,10 @@ type policyFile struct { downloaded bool uncompressed bool executed bool - logged bool } +var allPolicyFilesSuccessfullyApplied bool + // Handle signals func handleSysSignals(cancelFunc context.CancelFunc) { log.Debug("handleSysSignals") @@ -94,27 +95,12 @@ func start(c *cli.Context) error { formatter.PrintFatal("cannot create the pid file", err) } - timingInterval := c.Int64("time") - if !(timingInterval > 0) { - timingInterval = DefaultTimingInterval - } - // Adds a random value to the given timing interval! - // Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance) - timingInterval = timingInterval + int64(rand.New(rand.NewSource(time.Now().UnixNano())).Intn(DefaultRandomMaxThreshold)*60) - log.Debug("time interval: ", timingInterval) - - thresholdLines := c.Int("lines") - if !(thresholdLines > 0) { - thresholdLines = DefaultThresholdLines - } - log.Debug("routine lines threshold: ", thresholdLines) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() go handleSysSignals(cancel) - bootstrappingRoutine(ctx, c, timingInterval, thresholdLines) + bootstrappingRoutine(ctx, c) return nil } @@ -133,40 +119,49 @@ func stop(c *cli.Context) error { } // Main bootstrapping background routine -func bootstrappingRoutine(ctx context.Context, c *cli.Context, timingInterval int64, thresholdLines int) { +func bootstrappingRoutine(ctx context.Context, c *cli.Context) { log.Debug("bootstrappingRoutine") + timingInterval := c.Int64("interval") + if !(timingInterval > 0) { + timingInterval = DefaultTimingInterval + } + + timingSplay := c.Int64("splay") + if !(timingSplay > 0) { + timingSplay = DefaultTimingSplay + } + + thresholdLines := c.Int("lines") + if !(thresholdLines > 0) { + thresholdLines = DefaultThresholdLines + } + log.Debug("routine lines threshold: ", thresholdLines) + bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c) - commandProcessed := make(chan bool, 1) - isRunningCommandRoutine := false - currentTicker := time.NewTicker(time.Duration(timingInterval) * time.Second) for { - if !isRunningCommandRoutine { - isRunningCommandRoutine = true - go processingCommandRoutine(bootstrappingSvc, formatter, thresholdLines, commandProcessed) - } + applyPolicyfiles(bootstrappingSvc, formatter, thresholdLines) - log.Debug("waiting...", currentTicker) + // Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance) + ticker := time.NewTicker(time.Duration(timingInterval + int64(rand.New(rand.NewSource(time.Now().UnixNano())).Intn(int(timingSplay)))) * time.Second) select { - case <-commandProcessed: - isRunningCommandRoutine = false - currentTicker.Stop() - currentTicker = time.NewTicker(time.Duration(timingInterval) * time.Second) - log.Debug("command processed") - case <-currentTicker.C: + case <- ticker.C: log.Debug("ticker") - case <-ctx.Done(): + case <- ctx.Done(): log.Debug(ctx.Err()) log.Debug("closing bootstrapping") - return + } + ticker.Stop() + if ctx.Err() != nil { + break } } } // Subsidiary routine for commands processing -func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int, commandProcessed chan bool) { - log.Debug("processingCommandRoutine") +func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) { + log.Debug("applyPolicyfiles") // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration() @@ -201,9 +196,10 @@ func processingCommandRoutine(bootstrappingSvc *blueprint.BootstrappingService, // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to log.Debug("reporting applied policy files") reportAppliedConfiguration(bootstrappingSvc, bsProcess) + + completeBootstrappingSequence(bsProcess) } } - commandProcessed <- true } func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) { @@ -237,7 +233,7 @@ func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsPr policyFile.tarballPath = strings.Join([]string{bsProcess.directoryPath, policyFile.fileName}, "") policyFile.folderPath = strings.Join([]string{bsProcess.directoryPath, policyFile.name}, "") - bsProcess.policyFiles = append(bsProcess.policyFiles, *policyFile) + bsProcess.policyFiles = append(bsProcess.policyFiles, policyFile) } log.Debug(bsProcess) } @@ -306,20 +302,10 @@ func saveAttributes(bsProcess *bootstrappingProcess) { } } -//For every policy file, apply them doing the following: -// * Extract the tarball to a temporal work directory DIR -// * Run `cd DIR; chef-client -z -j path/to/attrs-.json` while sending the stderr and stdout in bunches of 10 lines to the -// platform via `POST /blueprint/bootstrap_logs` (this resource is a copy of POST /command_polling/bootstrap_logs used in the command_polling command). -// If the command returns with a non-zero value, stop applying policy files and continue with the next step. - -// TODO On the first iteration that applies successfully all policy files (runs all `chef-client -z` commands obtaining 0 return codes) only, run the boot scripts for the server by executing the `scripts boot` sub-command (as an external process). -// TODO Just a POC, an starging point. To be completed... +// processPolicyFiles applies for each policy the required chef commands, reporting in bunches of N lines func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { log.Debug("processPolicyFiles") - // Run `cd DIR; chef-client -z -j path/to/attrs-.json` while sending the stderr and stdout in bunches of - // 10 lines to the platform via `POST /blueprint/bootstrap_logs` (this resource is a copy of POST /command_polling/bootstrap_logs used in - // the command_polling command). If the command returns with a non-zero value, stop applying policyfiles and continue with the next step. for _, bsPolicyFile := range bsProcess.policyFiles { command := strings.Join([]string{"cd", bsPolicyFile.folderPath}, " ") if runtime.GOOS == "windows" { @@ -328,11 +314,6 @@ func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc command = strings.Join([]string{command, strings.Join([]string{"chef-client -z -j", bsProcess.attributes.filePath}, " ")}, ";") log.Debug(command) - // ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** - // TODO ** TO BE REMOVED** !!! for debugging purposes, overriding real command - command = "ping -c 100 8.8.8.8" - // ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** - // Custom method for chunks processing fn := func(chunk string) error { log.Debug("sendChunks") @@ -369,12 +350,16 @@ func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc } log.Info("completed: ", exitCode) + + bsPolicyFile.executed = exitCode == 0 // policy successfully applied + //If the command returns with a non-zero value, stop applying policyfiles and continue with the next step. + if !bsPolicyFile.executed { + break + } } } -// reportAppliedConfiguration Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request -//The `policy file_revision_ids` field should have revision ids set only for those policy files successfully applied on the iteration, that is, -// it should not have any values set for those failing and those skipped because of a previous one failing. +// reportAppliedConfiguration Inform the platform of applied changes func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { log.Debug("reportAppliedConfiguration") @@ -401,3 +386,27 @@ func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService log.Errorf("cannot report applied configuration [%s]", err) } } + +// completeBootstrappingSequence evaluates if the first iteration of policies was completed; If case, execute the "scripts boot" command. +func completeBootstrappingSequence(bsProcess *bootstrappingProcess) { + log.Debug("completeBootstrappingSequence") + + if !allPolicyFilesSuccessfullyApplied { + checked := true + for _, bsPolicyFile := range bsProcess.policyFiles { + if !bsPolicyFile.executed { + checked = false + break + } + } + allPolicyFilesSuccessfullyApplied = checked + + if allPolicyFilesSuccessfullyApplied { + log.Debug("run the boot scripts") + //run the boot scripts for the server by executing the scripts boot sub-command (as an external process). + if output, exit, _, _ := utils.RunCmd( strings.Join([]string{os.Args[0], "scripts", "boot"}, " ")); exit != 0 { + log.Errorf("Error executing scripts boot: (%d) %s", exit, output) + } + } + } +} diff --git a/bootstrapping/subcommands.go b/bootstrapping/subcommands.go index 88d5db7..119ab29 100644 --- a/bootstrapping/subcommands.go +++ b/bootstrapping/subcommands.go @@ -12,10 +12,15 @@ func SubCommands() []cli.Command { Action: start, Flags: []cli.Flag{ cli.Int64Flag{ - Name: "time, t", - Usage: "Bootstrapping time interval (seconds)", + Name: "interval, i", + Usage: "The frequency (in seconds) at which the bootstrapping runs", Value: DefaultTimingInterval, }, + cli.Int64Flag{ + Name: "splay, s", + Usage: "A random number between zero and splay that is added to interval (seconds)", + Value: DefaultTimingSplay, + }, cli.IntFlag{ Name: "lines, l", Usage: "Maximum lines threshold per response chunk", From 54a1d11fee4f7dcb4904caca7eb7aaf09e75acfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ban=CC=83os=20Lo=CC=81pez?= Date: Wed, 6 Mar 2019 12:18:26 +0100 Subject: [PATCH 07/20] Some refactor of agent bootstrapping command including error processing (issue #90) --- api/blueprint/bootstrapping_api.go | 6 +- api/types/bootstrapping.go | 6 +- bootstrapping/bootstrapping.go | 377 ++++++++++++++--------------- 3 files changed, 192 insertions(+), 197 deletions(-) diff --git a/api/blueprint/bootstrapping_api.go b/api/blueprint/bootstrapping_api.go index 2b9f859..14ab9f2 100644 --- a/api/blueprint/bootstrapping_api.go +++ b/api/blueprint/bootstrapping_api.go @@ -80,9 +80,9 @@ func (bs *BootstrappingService) ReportBootstrappingLog(BootstrappingContinuousRe } -// DownloadPolicyFile gets a file from given url saving file into given file path -func (bs *BootstrappingService) DownloadPolicyFile(url string, filePath string) (realFileName string, status int, err error) { - log.Debug("DownloadPolicyFile") +// DownloadPolicyfile gets a file from given url saving file into given file path +func (bs *BootstrappingService) DownloadPolicyfile(url string, filePath string) (realFileName string, status int, err error) { + log.Debug("DownloadPolicyfile") realFileName, status, err = bs.concertoService.GetFile(url, filePath) if err != nil { diff --git a/api/types/bootstrapping.go b/api/types/bootstrapping.go index 76e9918..1fee663 100644 --- a/api/types/bootstrapping.go +++ b/api/types/bootstrapping.go @@ -5,12 +5,12 @@ import ( ) type BootstrappingConfiguration struct { - PolicyFiles []BootstrappingPolicyFile `json:"policyfiles,omitempty" header:"POLICY FILES" show:"nolist"` + Policyfiles []BootstrappingPolicyfile `json:"policyfiles,omitempty" header:"POLICY FILES" show:"nolist"` Attributes *json.RawMessage `json:"attributes,omitempty" header:"ATTRIBUTES" show:"nolist"` AttributeRevisionID string `json:"attribute_revision_id,omitempty" header:"ATTRIBUTE REVISION ID"` } -type BootstrappingPolicyFile struct { +type BootstrappingPolicyfile struct { ID string `json:"id,omitempty" header:"ID"` RevisionID string `json:"revision_id,omitempty" header:"REVISION ID"` DownloadURL string `json:"download_url,omitempty" header:"DOWNLOAD URL"` @@ -23,6 +23,6 @@ type BootstrappingContinuousReport struct { type BootstrappingAppliedConfiguration struct { StartedAt string `json:"started_at,omitempty" header:"STARTED AT"` FinishedAt string `json:"finished_at,omitempty" header:"FINISHED AT"` - PolicyFileRevisionIDs string `json:"policyfile_revision_ids,omitempty" header:"POLICY FILE REVISION IDS" show:"nolist"` + PolicyfileRevisionIDs string `json:"policyfile_revision_ids,omitempty" header:"POLICY FILE REVISION IDS" show:"nolist"` AttributeRevisionID string `json:"attribute_revision_id,omitempty" header:"ATTRIBUTE REVISION ID"` } diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 6ec0199..7adaeb9 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -3,16 +3,17 @@ package bootstrapping import ( "context" "encoding/json" + "fmt" "io/ioutil" + "math/rand" "net/url" "os" "os/signal" + "path/filepath" + "runtime" "strings" "syscall" "time" - "math/rand" - "fmt" - "runtime" log "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" @@ -33,36 +34,22 @@ const ( ) type bootstrappingProcess struct { - startedAt string - finishedAt string - policyFiles []*policyFile - attributes attributes - thresholdLines int - directoryPath string + startedAt time.Time + finishedAt time.Time + policyfiles []policyfile + attributes attributes + thresholdLines int + directoryPath string + appliedPolicyfileRevisionIDs map[string]string } type attributes struct { revisionID string - fileName string - filePath string rawData *json.RawMessage } -type policyFile struct { - id string - revisionID string - name string - fileName string - tarballURL string - queryURL string - tarballPath string - folderPath string - - downloaded bool - uncompressed bool - executed bool -} +type policyfile types.BootstrappingPolicyfile -var allPolicyFilesSuccessfullyApplied bool +var allPolicyfilesSuccessfullyApplied bool // Handle signals func handleSysSignals(cancelFunc context.CancelFunc) { @@ -74,12 +61,12 @@ func handleSysSignals(cancelFunc context.CancelFunc) { cancelFunc() } -// Returns the full path to the tmp folder joined with pid management file name +// Returns the full path to the tmp directory joined with pid management file name func getProcessIDFilePath() string { return strings.Join([]string{os.TempDir(), string(os.PathSeparator), ProcessIDFile}, "") } -// Returns the full path to the tmp folder +// Returns the full path to the tmp directory func getProcessingFolderFilePath() string { dir := strings.Join([]string{os.TempDir(), string(os.PathSeparator), "imco", string(os.PathSeparator)}, "") os.Mkdir(dir, 0777) @@ -100,28 +87,6 @@ func start(c *cli.Context) error { go handleSysSignals(cancel) - bootstrappingRoutine(ctx, c) - - return nil -} - -// Stop the bootstrapping process -func stop(c *cli.Context) error { - log.Debug("cmdStop") - - formatter := format.GetFormatter() - if err := utils.StopProcess(getProcessIDFilePath()); err != nil { - formatter.PrintFatal("cannot stop the bootstrapping process", err) - } - - log.Info("Bootstrapping routine successfully stopped") - return nil -} - -// Main bootstrapping background routine -func bootstrappingRoutine(ctx context.Context, c *cli.Context) { - log.Debug("bootstrappingRoutine") - timingInterval := c.Int64("interval") if !(timingInterval > 0) { timingInterval = DefaultTimingInterval @@ -138,17 +103,18 @@ func bootstrappingRoutine(ctx context.Context, c *cli.Context) { } log.Debug("routine lines threshold: ", thresholdLines) + r := rand.New(rand.NewSource(time.Now().UnixNano())) bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c) for { applyPolicyfiles(bootstrappingSvc, formatter, thresholdLines) // Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance) - ticker := time.NewTicker(time.Duration(timingInterval + int64(rand.New(rand.NewSource(time.Now().UnixNano())).Intn(int(timingSplay)))) * time.Second) + ticker := time.NewTicker(time.Duration(timingInterval+int64(r.Intn(int(timingSplay)))) * time.Second) select { - case <- ticker.C: + case <-ticker.C: log.Debug("ticker") - case <- ctx.Done(): + case <-ctx.Done(): log.Debug(ctx.Err()) log.Debug("closing bootstrapping") } @@ -157,161 +123,180 @@ func bootstrappingRoutine(ctx context.Context, c *cli.Context) { break } } + + return nil +} + +// Stop the bootstrapping process +func stop(c *cli.Context) error { + log.Debug("cmdStop") + + formatter := format.GetFormatter() + if err := utils.StopProcess(getProcessIDFilePath()); err != nil { + formatter.PrintFatal("cannot stop the bootstrapping process", err) + } + + log.Info("Bootstrapping routine successfully stopped") + return nil } // Subsidiary routine for commands processing -func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) { +func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) error { log.Debug("applyPolicyfiles") // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration() + if err == nil && status != 200 { + err = fmt.Errorf("received non-ok %d response") + } if err != nil { formatter.PrintError("couldn't receive bootstrapping data", err) - } else { - if status == 200 { - bsProcess := new(bootstrappingProcess) - // Starting time - bsProcess.startedAt = time.Now().UTC().String() - bsProcess.thresholdLines = thresholdLines - bsProcess.directoryPath = getProcessingFolderFilePath() - - // proto structures - initializePrototype(bsConfiguration, bsProcess) - - // For every policyFile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... - downloadPolicyFiles(bootstrappingSvc, bsProcess) - - //... and clean off any tarball that is no longer needed. - cleanObsoletePolicyFiles(bsProcess) - - // Store the attributes as JSON in a file with name `attrs-.json` - saveAttributes(bsProcess) - - // Process tarballs policies - processPolicyFiles(bootstrappingSvc, bsProcess) - - // Finishing time - bsProcess.finishedAt = time.Now().UTC().String() + return err + } + bsProcess := &bootstrappingProcess{ + startedAt: time.Now().UTC(), + thresholdLines: thresholdLines, + directoryPath: getProcessingFolderFilePath(), + appliedPolicyfileRevisionIDs: make(map[string]string), + } - // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to - log.Debug("reporting applied policy files") - reportAppliedConfiguration(bootstrappingSvc, bsProcess) + // proto structures + err = initializePrototype(bsConfiguration, bsProcess) + if err != nil { + return err + } + // For every policyfile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... + err = downloadPolicyfiles(bootstrappingSvc, bsProcess) + if err != nil { + return err + } + //... and clean off any tarball that is no longer needed. + err = cleanObsoletePolicyfiles(bsProcess) + if err != nil { + return err + } + // Store the attributes as JSON in a file with name `attrs-.json` + err = saveAttributes(bsProcess) + if err != nil { + return err + } + // Process tarballs policies + err = processPolicyfiles(bootstrappingSvc, bsProcess) + if err != nil { + return err + } + // Finishing time + bsProcess.finishedAt = time.Now().UTC() - completeBootstrappingSequence(bsProcess) - } + // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to + log.Debug("reporting applied policy files") + err = reportAppliedConfiguration(bootstrappingSvc, bsProcess) + if err != nil { + return err } + return completeBootstrappingSequence(bsProcess) } -func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) { +func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) error { log.Debug("initializePrototype") // Attributes bsProcess.attributes.revisionID = bsConfiguration.AttributeRevisionID - bsProcess.attributes.fileName = strings.Join([]string{"attrs-", bsProcess.attributes.revisionID, ".json"}, "") - bsProcess.attributes.filePath = strings.Join([]string{bsProcess.directoryPath, bsProcess.attributes.fileName}, "") bsProcess.attributes.rawData = bsConfiguration.Attributes // Policies - for _, bsConfPolicyFile := range bsConfiguration.PolicyFiles { - policyFile := new(policyFile) - policyFile.id = bsConfPolicyFile.ID - policyFile.revisionID = bsConfPolicyFile.RevisionID - - policyFile.name = strings.Join([]string{policyFile.id, "-", policyFile.revisionID}, "") - policyFile.fileName = strings.Join([]string{policyFile.name, ".tgz"}, "") - policyFile.tarballURL = bsConfPolicyFile.DownloadURL - - if policyFile.tarballURL != "" { - url, err := url.Parse(policyFile.tarballURL) - if err != nil { - log.Errorf("cannot parse the tarball policy file url: %s [%s]", policyFile.tarballURL, err) - } else { - policyFile.queryURL = strings.Join([]string{url.Path[1:], url.RawQuery}, "?") - } - } - - policyFile.tarballPath = strings.Join([]string{bsProcess.directoryPath, policyFile.fileName}, "") - policyFile.folderPath = strings.Join([]string{bsProcess.directoryPath, policyFile.name}, "") - - bsProcess.policyFiles = append(bsProcess.policyFiles, policyFile) + for _, bsConfPolicyfile := range bsConfiguration.Policyfiles { + bsProcess.policyfiles = append(bsProcess.policyfiles, policyfile(bsConfPolicyfile)) } log.Debug(bsProcess) + return nil } -// downloadPolicyFiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... -func downloadPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { - log.Debug("downloadPolicyFiles") +// downloadPolicyfiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... +func downloadPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { + log.Debug("downloadPolicyfiles") - for _, bsPolicyFile := range bsProcess.policyFiles { - log.Debug("downloading: ", bsPolicyFile.tarballURL) - _, status, err := bootstrappingSvc.DownloadPolicyFile(bsPolicyFile.queryURL, bsPolicyFile.tarballPath) + for _, bsPolicyfile := range bsProcess.policyfiles { + tarballPath := bsPolicyfile.TarballPath(bsProcess.directoryPath) + log.Debug("downloading: ", tarballPath) + queryURL, err := bsPolicyfile.QueryURL() if err != nil { - log.Errorf("cannot download the tarball policy file: %s [%s]", bsPolicyFile.tarballURL, err) + return err } - if status == 200 { - bsPolicyFile.downloaded = true - log.Debug("decompressing: ", bsPolicyFile.tarballPath) - if err = utils.Untar(bsPolicyFile.tarballPath, bsPolicyFile.folderPath); err != nil { - log.Errorf("cannot decompress the tarball policy file: %s [%s]", bsPolicyFile.tarballPath, err) - } - bsPolicyFile.uncompressed = true - } else { - log.Errorf("cannot download the policy file: %v", bsPolicyFile.fileName) + _, status, err := bootstrappingSvc.DownloadPolicyfile(queryURL, tarballPath) + if err == nil && status != 200 { + err = fmt.Errorf("obtained non-ok response when downloading policyfile %s", queryURL) + } + if err != nil { + return err + } + if err = utils.Untar(tarballPath, bsPolicyfile.Path(bsProcess.directoryPath)); err != nil { + return err } } + return nil } -// cleanObsoletePolicyFiles cleans off any tarball that is no longer needed. -func cleanObsoletePolicyFiles(bsProcess *bootstrappingProcess) { - log.Debug("cleanObsoletePolicyFiles") +// cleanObsoletePolicyfiles cleans off any tarball that is no longer needed. +func cleanObsoletePolicyfiles(bsProcess *bootstrappingProcess) error { + log.Debug("cleanObsoletePolicyfiles") - // evaluates working folder + // evaluates working directory deletableFiles, err := ioutil.ReadDir(bsProcess.directoryPath) if err != nil { - log.Errorf("cannot read directory: %s [%s]", bsProcess.directoryPath, err) + return err } - // builds an array of currently processable files at this looping time - currentlyProcessableFiles := []string{bsProcess.attributes.fileName} // saved attributes file name - for _, bsPolicyFile := range bsProcess.policyFiles { - currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.fileName) // Downloaded tgz file names - currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.name) // Uncompressed folder names + // removes from deletableFiles those files we are going to use + for _, bsPolicyfile := range bsProcess.policyfiles { + for i, file := range deletableFiles { + if file.Name() == bsPolicyfile.FileName() { + deletableFiles[i] = deletableFiles[len(deletableFiles)-1] + deletableFiles = deletableFiles[:len(deletableFiles)-1] + break + } + if file.Name() == bsPolicyfile.Name() { + deletableFiles[i] = deletableFiles[len(deletableFiles)-1] + deletableFiles = deletableFiles[:len(deletableFiles)-1] + break + } + } } // removes from deletableFiles array the policy files currently applied for _, f := range deletableFiles { - if !utils.Contains(currentlyProcessableFiles, f.Name()) { - log.Debug("removing: ", f.Name()) - if err := os.RemoveAll(strings.Join([]string{bsProcess.directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { - log.Errorf("cannot remove: %s [%s]", f.Name(), err) - } + log.Debug("removing: ", f.Name()) + if err := os.RemoveAll(strings.Join([]string{bsProcess.directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { + return err } } + return nil } // saveAttributes stores the attributes as JSON in a file with name `attrs-.json` -func saveAttributes(bsProcess *bootstrappingProcess) { +func saveAttributes(bsProcess *bootstrappingProcess) error { log.Debug("saveAttributes") attrs, err := json.Marshal(bsProcess.attributes.rawData) if err != nil { - log.Errorf("cannot process policies attributes: %s [%s]", bsProcess.attributes.revisionID, err) + return err } - if err := ioutil.WriteFile(bsProcess.attributes.filePath, attrs, 0600); err != nil { - log.Errorf("cannot save policies attributes: %s [%s]", bsProcess.attributes.revisionID, err) + if err := ioutil.WriteFile(bsProcess.attributes.FilePath(bsProcess.directoryPath), attrs, 0600); err != nil { + return err } + return nil } -// processPolicyFiles applies for each policy the required chef commands, reporting in bunches of N lines -func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { - log.Debug("processPolicyFiles") +// processPolicyfiles applies for each policy the required chef commands, reporting in bunches of N lines +func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { + log.Debug("processPolicyfiles") - for _, bsPolicyFile := range bsProcess.policyFiles { - command := strings.Join([]string{"cd", bsPolicyFile.folderPath}, " ") + for _, bsPolicyfile := range bsProcess.policyfiles { + command := strings.Join([]string{"cd", bsPolicyfile.Path(bsProcess.directoryPath)}, " ") if runtime.GOOS == "windows" { command = strings.Join([]string{command, "SET \"PATH=%PATH%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\""}, ";") } - command = strings.Join([]string{command, strings.Join([]string{"chef-client -z -j", bsProcess.attributes.filePath}, " ")}, ";") + command = strings.Join([]string{command, strings.Join([]string{"chef-client -z -j", bsProcess.attributes.FilePath(bsProcess.directoryPath)}, " ")}, ";") log.Debug(command) // Custom method for chunks processing @@ -345,68 +330,78 @@ func processPolicyFiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc } exitCode, err := utils.RunContinuousCmd(fn, command, -1, bsProcess.thresholdLines) + if err == nil && exitCode != 0 { + err = fmt.Errorf("policyfile application exited with %d code", exitCode) + } if err != nil { - log.Errorf("cannot process continuous report command [%s]", err) + return err } log.Info("completed: ", exitCode) - - bsPolicyFile.executed = exitCode == 0 // policy successfully applied - //If the command returns with a non-zero value, stop applying policyfiles and continue with the next step. - if !bsPolicyFile.executed { - break - } + bsProcess.appliedPolicyfileRevisionIDs[bsPolicyfile.ID] = bsPolicyfile.RevisionID } + return nil } // reportAppliedConfiguration Inform the platform of applied changes -func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) { +func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { log.Debug("reportAppliedConfiguration") - var policyFileRevisionIDs string - for _, bsPolicyFile := range bsProcess.policyFiles { - if bsPolicyFile.executed { // only for policies successfully applied - appliedPolicyMap := map[string]string{bsPolicyFile.id: bsPolicyFile.revisionID} - appliedPolicyBytes, err := json.Marshal(appliedPolicyMap) - if err != nil { - log.Errorf("corrupted candidates policies map [%s]", err) - } - policyFileRevisionIDs = strings.Join([]string{policyFileRevisionIDs, string(appliedPolicyBytes)}, "") - } - } - payload := map[string]interface{}{ "started_at": bsProcess.startedAt, "finished_at": bsProcess.finishedAt, - "policyfile_revision_ids": policyFileRevisionIDs, + "policyfile_revision_ids": bsProcess.appliedPolicyfileRevisionIDs, "attribute_revision_id": bsProcess.attributes.revisionID, } - err := bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload) - if err != nil { - log.Errorf("cannot report applied configuration [%s]", err) - } + return bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload) } // completeBootstrappingSequence evaluates if the first iteration of policies was completed; If case, execute the "scripts boot" command. -func completeBootstrappingSequence(bsProcess *bootstrappingProcess) { +func completeBootstrappingSequence(bsProcess *bootstrappingProcess) error { log.Debug("completeBootstrappingSequence") - if !allPolicyFilesSuccessfullyApplied { - checked := true - for _, bsPolicyFile := range bsProcess.policyFiles { - if !bsPolicyFile.executed { - checked = false - break - } + if !allPolicyfilesSuccessfullyApplied { + log.Debug("run the boot scripts") + //run the boot scripts for the server by executing the scripts boot sub-command (as an external process). + if output, exit, _, _ := utils.RunCmd(strings.Join([]string{os.Args[0], "scripts", "boot"}, " ")); exit != 0 { + return fmt.Errorf("boot scripts run failed with exit code %d and following output: %s", exit, output) } - allPolicyFilesSuccessfullyApplied = checked + allPolicyfilesSuccessfullyApplied = true + } + return nil +} - if allPolicyFilesSuccessfullyApplied { - log.Debug("run the boot scripts") - //run the boot scripts for the server by executing the scripts boot sub-command (as an external process). - if output, exit, _, _ := utils.RunCmd( strings.Join([]string{os.Args[0], "scripts", "boot"}, " ")); exit != 0 { - log.Errorf("Error executing scripts boot: (%d) %s", exit, output) - } - } +func (pf policyfile) Name() string { + return strings.Join([]string{pf.ID, "-", pf.RevisionID}, "") +} + +func (pf *policyfile) FileName() string { + return strings.Join([]string{pf.Name(), "tgz"}, ".") +} + +func (pf *policyfile) QueryURL() (string, error) { + if pf.DownloadURL == "" { + return "", fmt.Errorf("obtaining URL query: empty download URL") } + url, err := url.Parse(pf.DownloadURL) + if err != nil { + return "", fmt.Errorf("parsing URL to extract query: %v", err) + } + return strings.Join([]string{url.Path[1:], url.RawQuery}, "?"), nil +} + +func (pf *policyfile) TarballPath(dir string) string { + return filepath.Join(dir, pf.FileName()) +} + +func (pf *policyfile) Path(dir string) string { + return filepath.Join(dir, pf.Name()) +} + +func (a *attributes) FileName() string { + return fmt.Sprintf("attrs-%s.json", a.revisionID) +} + +func (a *attributes) FilePath(dir string) string { + return filepath.Join(dir, a.FileName()) } From 1bf8859b6e19dff3a5343845161141ae299bb100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ban=CC=83os=20Lo=CC=81pez?= Date: Wed, 6 Mar 2019 13:19:15 +0100 Subject: [PATCH 08/20] Ensure a single instance of the bootstrapping command works at a time (issue #90) --- Gopkg.lock | 8 +++++++- Gopkg.toml | 4 ++++ bootstrapping/bootstrapping.go | 34 ++++++++++++++++++++++++++++------ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index e7ea940..5503c24 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -17,6 +17,12 @@ revision = "4b6ea7319e214d98c938f12692336f7ca9348d6b" version = "v0.10.0" +[[projects]] + branch = "master" + name = "github.com/allan-simon/go-singleinstance" + packages = ["."] + revision = "79edcfdc2dfc93da913f46ae8d9f8a9602250431" + [[projects]] name = "github.com/asaskevich/govalidator" packages = ["."] @@ -91,6 +97,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "92b7325f6a94cda806d124a617bcc6703c943d68c51e2be3182ef4ed42dc35a7" + inputs-digest = "110d97437b0209f38ce5b41683f4bdb8842f608d32b27d184ddc13edb6bfd8c9" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 62d3944..99977a0 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -55,3 +55,7 @@ [[constraint]] name = "github.com/pmezard/go-difflib" revision = "792786c7400a136282c1664665ae0a8db921c6c2" + +[[constraint]] + branch = "master" + name = "github.com/allan-simon/go-singleinstance" diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 7adaeb9..bb85bff 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -16,6 +16,7 @@ import ( "time" log "github.com/Sirupsen/logrus" + singleinstance "github.com/allan-simon/go-singleinstance" "github.com/codegangsta/cli" "github.com/ingrammicro/concerto/api/blueprint" "github.com/ingrammicro/concerto/api/types" @@ -67,16 +68,33 @@ func getProcessIDFilePath() string { } // Returns the full path to the tmp directory -func getProcessingFolderFilePath() string { - dir := strings.Join([]string{os.TempDir(), string(os.PathSeparator), "imco", string(os.PathSeparator)}, "") - os.Mkdir(dir, 0777) - return dir +func generateWorkspaceDir() (string, error) { + dir := filepath.Join(os.TempDir(), "imco") + dirInfo, err := os.Stat(dir) + if err != nil { + err := os.Mkdir(dir, 0777) + if err != nil { + return "", err + } + } else { + if !dirInfo.Mode().IsDir() { + return "", fmt.Errorf("%s exists but is not a directory", dir) + } + } + return dir, nil } // Start the bootstrapping process func start(c *cli.Context) error { log.Debug("start") + // TODO: replace /etc/imco with a directory taken from configuration/that depends on OS + lockFile, err := singleinstance.CreateLockFile(filepath.Join("/etc/imco", "imco-bootstrapping.lock")) + if err != nil { + return err + } + defer lockFile.Close() + formatter := format.GetFormatter() if err := utils.SetProcessIdToFile(getProcessIDFilePath()); err != nil { formatter.PrintFatal("cannot create the pid file", err) @@ -147,16 +165,20 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration() if err == nil && status != 200 { - err = fmt.Errorf("received non-ok %d response") + err = fmt.Errorf("received non-ok %d response", status) } if err != nil { formatter.PrintError("couldn't receive bootstrapping data", err) return err } + dir, err := generateWorkspaceDir() + if err != nil { + return err + } bsProcess := &bootstrappingProcess{ startedAt: time.Now().UTC(), thresholdLines: thresholdLines, - directoryPath: getProcessingFolderFilePath(), + directoryPath: dir, appliedPolicyfileRevisionIDs: make(map[string]string), } From 5365e95b8f11d3b26be623ea5e184c4f01d97ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ban=CC=83os=20Lo=CC=81pez?= Date: Wed, 6 Mar 2019 16:50:02 +0100 Subject: [PATCH 09/20] Fix bootstrapping issues (issue #90) --- bootstrapping/bootstrapping.go | 45 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index bb85bff..43d7869 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -185,26 +185,31 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte // proto structures err = initializePrototype(bsConfiguration, bsProcess) if err != nil { + log.Debug(err) return err } // For every policyfile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... err = downloadPolicyfiles(bootstrappingSvc, bsProcess) if err != nil { + log.Debug(err) return err } //... and clean off any tarball that is no longer needed. err = cleanObsoletePolicyfiles(bsProcess) if err != nil { + log.Debug(err) return err } // Store the attributes as JSON in a file with name `attrs-.json` err = saveAttributes(bsProcess) if err != nil { + log.Debug(err) return err } // Process tarballs policies err = processPolicyfiles(bootstrappingSvc, bsProcess) if err != nil { + log.Debug(err) return err } // Finishing time @@ -214,6 +219,7 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte log.Debug("reporting applied policy files") err = reportAppliedConfiguration(bootstrappingSvc, bsProcess) if err != nil { + log.Debug(err) return err } return completeBootstrappingSequence(bsProcess) @@ -261,35 +267,28 @@ func downloadPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsPro // cleanObsoletePolicyfiles cleans off any tarball that is no longer needed. func cleanObsoletePolicyfiles(bsProcess *bootstrappingProcess) error { - log.Debug("cleanObsoletePolicyfiles") + log.Debug("cleanObsoletePolicyFiles") - // evaluates working directory + // evaluates working folder deletableFiles, err := ioutil.ReadDir(bsProcess.directoryPath) if err != nil { return err } - // removes from deletableFiles those files we are going to use - for _, bsPolicyfile := range bsProcess.policyfiles { - for i, file := range deletableFiles { - if file.Name() == bsPolicyfile.FileName() { - deletableFiles[i] = deletableFiles[len(deletableFiles)-1] - deletableFiles = deletableFiles[:len(deletableFiles)-1] - break - } - if file.Name() == bsPolicyfile.Name() { - deletableFiles[i] = deletableFiles[len(deletableFiles)-1] - deletableFiles = deletableFiles[:len(deletableFiles)-1] - break - } - } + // builds an array of currently processable files at this looping time + currentlyProcessableFiles := []string{bsProcess.attributes.FileName()} // saved attributes file name + for _, bsPolicyFile := range bsProcess.policyfiles { + currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.FileName()) // Downloaded tgz file names + currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.Name()) // Uncompressed folder names } // removes from deletableFiles array the policy files currently applied for _, f := range deletableFiles { - log.Debug("removing: ", f.Name()) - if err := os.RemoveAll(strings.Join([]string{bsProcess.directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { - return err + if !utils.Contains(currentlyProcessableFiles, f.Name()) { + log.Debug("removing: ", f.Name()) + if err := os.RemoveAll(strings.Join([]string{bsProcess.directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { + return err + } } } return nil @@ -314,11 +313,11 @@ func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc log.Debug("processPolicyfiles") for _, bsPolicyfile := range bsProcess.policyfiles { - command := strings.Join([]string{"cd", bsPolicyfile.Path(bsProcess.directoryPath)}, " ") + command := fmt.Sprintf("cd %s", bsPolicyfile.Path(bsProcess.directoryPath)) if runtime.GOOS == "windows" { - command = strings.Join([]string{command, "SET \"PATH=%PATH%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\""}, ";") + command = fmt.Sprintf("%s\nSET \"PATH=%PATH%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"", command) } - command = strings.Join([]string{command, strings.Join([]string{"chef-client -z -j", bsProcess.attributes.FilePath(bsProcess.directoryPath)}, " ")}, ";") + command = fmt.Sprintf("%s\nchef-client -z -j %s", command, bsProcess.attributes.FilePath(bsProcess.directoryPath)) log.Debug(command) // Custom method for chunks processing @@ -409,7 +408,7 @@ func (pf *policyfile) QueryURL() (string, error) { if err != nil { return "", fmt.Errorf("parsing URL to extract query: %v", err) } - return strings.Join([]string{url.Path[1:], url.RawQuery}, "?"), nil + return fmt.Sprintf("%s?%s", url.Path, url.RawQuery), nil } func (pf *policyfile) TarballPath(dir string) string { From b5ced19ce5ab9ab48831c02c573ab09a108c25ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ban=CC=83os=20Lo=CC=81pez?= Date: Wed, 6 Mar 2019 17:01:37 +0100 Subject: [PATCH 10/20] Make bootstrapping report applied policyfiles when some fail (issue #90) --- bootstrapping/bootstrapping.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 43d7869..309aefb 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -208,20 +208,19 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte } // Process tarballs policies err = processPolicyfiles(bootstrappingSvc, bsProcess) - if err != nil { - log.Debug(err) - return err - } // Finishing time bsProcess.finishedAt = time.Now().UTC() // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to log.Debug("reporting applied policy files") - err = reportAppliedConfiguration(bootstrappingSvc, bsProcess) - if err != nil { + reportErr := reportAppliedConfiguration(bootstrappingSvc, bsProcess) + if reportErr != nil { log.Debug(err) return err } + if err != nil { + return err + } return completeBootstrappingSequence(bsProcess) } @@ -286,7 +285,7 @@ func cleanObsoletePolicyfiles(bsProcess *bootstrappingProcess) error { for _, f := range deletableFiles { if !utils.Contains(currentlyProcessableFiles, f.Name()) { log.Debug("removing: ", f.Name()) - if err := os.RemoveAll(strings.Join([]string{bsProcess.directoryPath, string(os.PathSeparator), f.Name()}, "")); err != nil { + if err := os.RemoveAll(filepath.Join(bsProcess.directoryPath, f.Name())); err != nil { return err } } From b8f8aaaa4633c3b7b4e72844e205022e4ad1e9c3 Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Wed, 6 Mar 2019 18:42:57 +0100 Subject: [PATCH 11/20] Refactored single instance management (issue #90) --- bootstrapping/bootstrapping.go | 92 ++++++++++++++++------------------ 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 309aefb..3c571a0 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -30,7 +30,7 @@ const ( DefaultTimingInterval = 600 // 600 seconds = 10 minutes DefaultTimingSplay = 360 // seconds DefaultThresholdLines = 10 - ProcessIDFile = "imco-bootstrapping.pid" + ProcessLockFile = "imco-bootstrapping.lock" RetriesNumber = 5 ) @@ -48,9 +48,44 @@ type attributes struct { rawData *json.RawMessage } +var allPolicyfilesSuccessfullyApplied bool + type policyfile types.BootstrappingPolicyfile -var allPolicyfilesSuccessfullyApplied bool +func (pf policyfile) Name() string { + return strings.Join([]string{pf.ID, "-", pf.RevisionID}, "") +} + +func (pf *policyfile) FileName() string { + return strings.Join([]string{pf.Name(), "tgz"}, ".") +} + +func (pf *policyfile) QueryURL() (string, error) { + if pf.DownloadURL == "" { + return "", fmt.Errorf("obtaining URL query: empty download URL") + } + url, err := url.Parse(pf.DownloadURL) + if err != nil { + return "", fmt.Errorf("parsing URL to extract query: %v", err) + } + return fmt.Sprintf("%s?%s", url.Path, url.RawQuery), nil +} + +func (pf *policyfile) TarballPath(dir string) string { + return filepath.Join(dir, pf.FileName()) +} + +func (pf *policyfile) Path(dir string) string { + return filepath.Join(dir, pf.Name()) +} + +func (a *attributes) FileName() string { + return fmt.Sprintf("attrs-%s.json", a.revisionID) +} + +func (a *attributes) FilePath(dir string) string { + return filepath.Join(dir, a.FileName()) +} // Handle signals func handleSysSignals(cancelFunc context.CancelFunc) { @@ -62,9 +97,9 @@ func handleSysSignals(cancelFunc context.CancelFunc) { cancelFunc() } -// Returns the full path to the tmp directory joined with pid management file name -func getProcessIDFilePath() string { - return strings.Join([]string{os.TempDir(), string(os.PathSeparator), ProcessIDFile}, "") +// Returns the full path to the tmp directory joined with lock management file name +func getProcessLockFilePath() string { + return filepath.Join(os.TempDir(), string(os.PathSeparator), ProcessLockFile) } // Returns the full path to the tmp directory @@ -88,17 +123,13 @@ func generateWorkspaceDir() (string, error) { func start(c *cli.Context) error { log.Debug("start") - // TODO: replace /etc/imco with a directory taken from configuration/that depends on OS - lockFile, err := singleinstance.CreateLockFile(filepath.Join("/etc/imco", "imco-bootstrapping.lock")) + lockFile, err := singleinstance.CreateLockFile(getProcessLockFilePath()) if err != nil { return err } defer lockFile.Close() formatter := format.GetFormatter() - if err := utils.SetProcessIdToFile(getProcessIDFilePath()); err != nil { - formatter.PrintFatal("cannot create the pid file", err) - } ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -150,7 +181,7 @@ func stop(c *cli.Context) error { log.Debug("cmdStop") formatter := format.GetFormatter() - if err := utils.StopProcess(getProcessIDFilePath()); err != nil { + if err := utils.StopProcess(getProcessLockFilePath()); err != nil { formatter.PrintFatal("cannot stop the bootstrapping process", err) } @@ -266,7 +297,7 @@ func downloadPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsPro // cleanObsoletePolicyfiles cleans off any tarball that is no longer needed. func cleanObsoletePolicyfiles(bsProcess *bootstrappingProcess) error { - log.Debug("cleanObsoletePolicyFiles") + log.Debug("cleanObsoletePolicyfiles") // evaluates working folder deletableFiles, err := ioutil.ReadDir(bsProcess.directoryPath) @@ -389,39 +420,4 @@ func completeBootstrappingSequence(bsProcess *bootstrappingProcess) error { allPolicyfilesSuccessfullyApplied = true } return nil -} - -func (pf policyfile) Name() string { - return strings.Join([]string{pf.ID, "-", pf.RevisionID}, "") -} - -func (pf *policyfile) FileName() string { - return strings.Join([]string{pf.Name(), "tgz"}, ".") -} - -func (pf *policyfile) QueryURL() (string, error) { - if pf.DownloadURL == "" { - return "", fmt.Errorf("obtaining URL query: empty download URL") - } - url, err := url.Parse(pf.DownloadURL) - if err != nil { - return "", fmt.Errorf("parsing URL to extract query: %v", err) - } - return fmt.Sprintf("%s?%s", url.Path, url.RawQuery), nil -} - -func (pf *policyfile) TarballPath(dir string) string { - return filepath.Join(dir, pf.FileName()) -} - -func (pf *policyfile) Path(dir string) string { - return filepath.Join(dir, pf.Name()) -} - -func (a *attributes) FileName() string { - return fmt.Sprintf("attrs-%s.json", a.revisionID) -} - -func (a *attributes) FilePath(dir string) string { - return filepath.Join(dir, a.FileName()) -} +} \ No newline at end of file From b51bed764038dc156a4b4019ae3a703b74ccad1a Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Wed, 6 Mar 2019 19:24:08 +0100 Subject: [PATCH 12/20] Make more readable the processing error messages (issue #90) --- bootstrapping/bootstrapping.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 3c571a0..c61ebf6 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -204,6 +204,7 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte } dir, err := generateWorkspaceDir() if err != nil { + formatter.PrintError("couldn't generated workspace directory", err) return err } bsProcess := &bootstrappingProcess{ @@ -216,25 +217,25 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte // proto structures err = initializePrototype(bsConfiguration, bsProcess) if err != nil { - log.Debug(err) + formatter.PrintError("couldn't initialize prototype", err) return err } // For every policyfile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... err = downloadPolicyfiles(bootstrappingSvc, bsProcess) if err != nil { - log.Debug(err) + formatter.PrintError("couldn't download policy files", err) return err } //... and clean off any tarball that is no longer needed. err = cleanObsoletePolicyfiles(bsProcess) if err != nil { - log.Debug(err) + formatter.PrintError("couldn't clean obsolete policy files", err) return err } // Store the attributes as JSON in a file with name `attrs-.json` err = saveAttributes(bsProcess) if err != nil { - log.Debug(err) + formatter.PrintError("couldn't save attributes for policy files", err) return err } // Process tarballs policies @@ -246,10 +247,11 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte log.Debug("reporting applied policy files") reportErr := reportAppliedConfiguration(bootstrappingSvc, bsProcess) if reportErr != nil { - log.Debug(err) + formatter.PrintError("couldn't report applied status for policy files", err) return err } if err != nil { + formatter.PrintError("couldn't process policy files", err) return err } return completeBootstrappingSequence(bsProcess) From 8a22cbe339c33e098b05d134ec1e8b50bc6a470d Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Thu, 7 Mar 2019 10:11:40 +0100 Subject: [PATCH 13/20] Added context management (issue #90) Used with CommandContext at uncompressing "tgz" time --- bootstrapping/bootstrapping.go | 10 +++++----- utils/utils.go | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index c61ebf6..50ab34d 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -155,7 +155,7 @@ func start(c *cli.Context) error { r := rand.New(rand.NewSource(time.Now().UnixNano())) bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c) for { - applyPolicyfiles(bootstrappingSvc, formatter, thresholdLines) + applyPolicyfiles(ctx, bootstrappingSvc, formatter, thresholdLines) // Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance) ticker := time.NewTicker(time.Duration(timingInterval+int64(r.Intn(int(timingSplay)))) * time.Second) @@ -190,7 +190,7 @@ func stop(c *cli.Context) error { } // Subsidiary routine for commands processing -func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) error { +func applyPolicyfiles(ctx context.Context, bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) error { log.Debug("applyPolicyfiles") // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes @@ -221,7 +221,7 @@ func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatte return err } // For every policyfile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... - err = downloadPolicyfiles(bootstrappingSvc, bsProcess) + err = downloadPolicyfiles(ctx, bootstrappingSvc, bsProcess) if err != nil { formatter.PrintError("couldn't download policy files", err) return err @@ -273,7 +273,7 @@ func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsPr } // downloadPolicyfiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... -func downloadPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { +func downloadPolicyfiles(ctx context.Context, bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { log.Debug("downloadPolicyfiles") for _, bsPolicyfile := range bsProcess.policyfiles { @@ -290,7 +290,7 @@ func downloadPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsPro if err != nil { return err } - if err = utils.Untar(tarballPath, bsPolicyfile.Path(bsProcess.directoryPath)); err != nil { + if err = utils.Untar(ctx, tarballPath, bsPolicyfile.Path(bsProcess.directoryPath)); err != nil { return err } } diff --git a/utils/utils.go b/utils/utils.go index ece700e..53cab57 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -2,10 +2,12 @@ package utils import ( "archive/zip" + "context" "fmt" "io" "math/rand" "os" + "os/exec" "path/filepath" "regexp" "strings" @@ -13,8 +15,6 @@ import ( "github.com/codegangsta/cli" - "os/exec" - log "github.com/Sirupsen/logrus" ) @@ -62,14 +62,13 @@ func Unzip(archive, target string) error { return nil } -// TODO using cmd := exec.CommandContext(ctx,... -func Untar(source, target string) error { +func Untar(ctx context.Context, source, target string) error { if err := os.MkdirAll(target, 0600); err != nil { return err } - cmd := exec.Command("tar", "-xzf", source, "-C", target) + cmd := exec.CommandContext(ctx, "tar", "-xzf", source, "-C", target) if err := cmd.Run(); err != nil { return err } From 5607bf006e89ad14054a419f397821b5b1ee7e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ban=CC=83os=20Lo=CC=81pez?= Date: Thu, 7 Mar 2019 16:30:09 +0100 Subject: [PATCH 14/20] Refactor bootstrapping workspace dir management (issue #90) --- bootstrapping/bootstrapping.go | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 50ab34d..cd330a9 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -97,33 +97,41 @@ func handleSysSignals(cancelFunc context.CancelFunc) { cancelFunc() } -// Returns the full path to the tmp directory joined with lock management file name -func getProcessLockFilePath() string { - return filepath.Join(os.TempDir(), string(os.PathSeparator), ProcessLockFile) +// Returns the full path to the tmp directory joined with pid management file name +func lockFilePath() string { + return filepath.Join(workspaceDir(), ProcessLockFile) +} + +func workspaceDir() string { + return filepath.Join(os.TempDir(), "imco") } // Returns the full path to the tmp directory -func generateWorkspaceDir() (string, error) { - dir := filepath.Join(os.TempDir(), "imco") +func generateWorkspaceDir() error { + dir := workspaceDir() dirInfo, err := os.Stat(dir) if err != nil { err := os.Mkdir(dir, 0777) if err != nil { - return "", err + return err } } else { if !dirInfo.Mode().IsDir() { - return "", fmt.Errorf("%s exists but is not a directory", dir) + return fmt.Errorf("%s exists but is not a directory", dir) } } - return dir, nil + return nil } // Start the bootstrapping process func start(c *cli.Context) error { log.Debug("start") - lockFile, err := singleinstance.CreateLockFile(getProcessLockFilePath()) + err := generateWorkspaceDir() + if err != nil { + return err + } + lockFile, err := singleinstance.CreateLockFile(lockFilePath()) if err != nil { return err } @@ -181,7 +189,7 @@ func stop(c *cli.Context) error { log.Debug("cmdStop") formatter := format.GetFormatter() - if err := utils.StopProcess(getProcessLockFilePath()); err != nil { + if err := utils.StopProcess(lockFilePath()); err != nil { formatter.PrintFatal("cannot stop the bootstrapping process", err) } @@ -202,7 +210,7 @@ func applyPolicyfiles(ctx context.Context, bootstrappingSvc *blueprint.Bootstrap formatter.PrintError("couldn't receive bootstrapping data", err) return err } - dir, err := generateWorkspaceDir() + err = generateWorkspaceDir() if err != nil { formatter.PrintError("couldn't generated workspace directory", err) return err @@ -210,7 +218,7 @@ func applyPolicyfiles(ctx context.Context, bootstrappingSvc *blueprint.Bootstrap bsProcess := &bootstrappingProcess{ startedAt: time.Now().UTC(), thresholdLines: thresholdLines, - directoryPath: dir, + directoryPath: workspaceDir(), appliedPolicyfileRevisionIDs: make(map[string]string), } @@ -422,4 +430,4 @@ func completeBootstrappingSequence(bsProcess *bootstrappingProcess) error { allPolicyfilesSuccessfullyApplied = true } return nil -} \ No newline at end of file +} From 4556ce199b2b40922c36d5cc15c0a878eae60643 Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Fri, 8 Mar 2019 00:05:30 +0100 Subject: [PATCH 15/20] Added test cases (issue #90) --- api/blueprint/bootstrapping_api_mocked.go | 315 +++++++++++++++++++++- api/blueprint/bootstrapping_api_test.go | 28 +- testdata/boostrapping_data.go | 40 +++ 3 files changed, 379 insertions(+), 4 deletions(-) create mode 100644 testdata/boostrapping_data.go diff --git a/api/blueprint/bootstrapping_api_mocked.go b/api/blueprint/bootstrapping_api_mocked.go index 28bf3c3..29ef6b8 100644 --- a/api/blueprint/bootstrapping_api_mocked.go +++ b/api/blueprint/bootstrapping_api_mocked.go @@ -1,3 +1,316 @@ package blueprint -// TODO +import ( + "encoding/json" + "github.com/ingrammicro/concerto/api/types" + "github.com/ingrammicro/concerto/utils" + "github.com/stretchr/testify/assert" + "testing" + "fmt" +) + +// GetBootstrappingConfigurationMocked test mocked function +func GetBootstrappingConfigurationMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(bcConfIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 200, nil) + bcConfOut, status, err := ds.GetBootstrappingConfiguration() + assert.Nil(err, "Error getting bootstrapping configuration") + assert.Equal(status, 200, "GetBootstrappingConfiguration returned invalid response") + assert.Equal(bcConfIn, bcConfOut, "GetBootstrappingConfiguration returned different services") + return bcConfOut +} + +// GetBootstrappingConfigurationFailErrMocked test mocked function +func GetBootstrappingConfigurationFailErrMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(bcConfIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 404, fmt.Errorf("Mocked error")) + bcConfOut, _, err := ds.GetBootstrappingConfiguration() + + assert.NotNil(err, "We are expecting an error") + assert.Nil(bcConfOut, "Expecting nil output") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") + + return bcConfOut +} + +// GetBootstrappingConfigurationFailStatusMocked test mocked function +func GetBootstrappingConfigurationFailStatusMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(bcConfIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 499, nil) + bcConfOut, status, err := ds.GetBootstrappingConfiguration() + + assert.NotNil(err, "We are expecting an status code error") + assert.Nil(bcConfOut, "Expecting nil output") + assert.Equal(499, status, "Expecting http code 499") + assert.Contains(err.Error(), "499", "Error should contain http code 499") + + return bcConfOut +} + +// GetBootstrappingConfigurationFailJSONMocked test mocked function +func GetBootstrappingConfigurationFailJSONMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // wrong json + dIn := []byte{10, 20, 30} + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 200, nil) + bcConfOut, _, err := ds.GetBootstrappingConfiguration() + + assert.NotNil(err, "We are expecting a marshalling error") + assert.Nil(bcConfOut, "Expecting nil output") + + return bcConfOut +} + +// ReportBootstrappingAppliedConfigurationMocked test mocked function +func ReportBootstrappingAppliedConfigurationMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dOut, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dOut, 200, nil) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.Nil(err, "Error getting bootstrapping command") +} + +// ReportBootstrappingAppliedConfigurationFailErrMocked test mocked function +func ReportBootstrappingAppliedConfigurationFailErrMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 400, fmt.Errorf("Mocked error")) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.NotNil(err, "We are expecting an error") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") +} + +// ReportBootstrappingAppliedConfigurationFailStatusMocked test mocked function +func ReportBootstrappingAppliedConfigurationFailStatusMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 499, fmt.Errorf("Error 499 Mocked error")) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.NotNil(err, "We are expecting a status code error") + assert.Contains(err.Error(), "499", "Error should contain http code 499") +} + +// ReportBootstrappingAppliedConfigurationFailJSONMocked test mocked function +func ReportBootstrappingAppliedConfigurationFailJSONMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // wrong json + dIn := []byte{0} + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 499, nil) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.Contains(err.Error(), "499", "Error should contain http code 499") +} + +// ReportBootstrappingLogMocked test mocked function +func ReportBootstrappingLogMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dOut, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dOut, 200, nil) + commandOut, status, err := ds.ReportBootstrappingLog(&payload) + + assert.Nil(err, "Error posting report command") + assert.Equal(status, 200, "ReportBootstrappingLog returned invalid response") + assert.Equal(commandOut.Stdout, "Bootstrap log created", "ReportBootstrapLog returned unexpected message") + + return commandOut +} + +// ReportBootstrappingLogFailErrMocked test mocked function +func ReportBootstrappingLogFailErrMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 400, fmt.Errorf("Mocked error")) + commandOut, _, err := ds.ReportBootstrappingLog(&payload) + + assert.NotNil(err, "We are expecting an error") + assert.Nil(commandOut, "Expecting nil output") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") + + return commandOut +} + +// ReportBootstrappingLogFailStatusMocked test mocked function +func ReportBootstrappingLogFailStatusMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 499, fmt.Errorf("Error 499 Mocked error")) + commandOut, status, err := ds.ReportBootstrappingLog(&payload) + + assert.Equal(status, 499, "ReportBootstrappingLog returned an unexpected status code") + assert.NotNil(err, "We are expecting a status code error") + assert.Nil(commandOut, "Expecting nil output") + assert.Contains(err.Error(), "499", "Error should contain http code 499") + + return commandOut +} + +// ReportBootstrappingLogFailJSONMocked test mocked function +func ReportBootstrappingLogFailJSONMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // wrong json + dIn := []byte{10, 20, 30} + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 200, nil) + commandOut, _, err := ds.ReportBootstrappingLog(&payload) + + assert.NotNil(err, "We are expecting a marshalling error") + assert.Nil(commandOut, "Expecting nil output") + assert.Contains(err.Error(), "invalid character", "Error message should include the string 'invalid character'") + + return commandOut +} \ No newline at end of file diff --git a/api/blueprint/bootstrapping_api_test.go b/api/blueprint/bootstrapping_api_test.go index 74ba96c..fa4e0b4 100644 --- a/api/blueprint/bootstrapping_api_test.go +++ b/api/blueprint/bootstrapping_api_test.go @@ -1,9 +1,9 @@ package blueprint import ( - "testing" - + "github.com/ingrammicro/concerto/testdata" "github.com/stretchr/testify/assert" + "testing" ) func TestNewBootstrappingServiceNil(t *testing.T) { @@ -13,4 +13,26 @@ func TestNewBootstrappingServiceNil(t *testing.T) { assert.NotNil(err, "Uninitialized service should return error") } -// TODO +func TestGetBootstrappingConfiguration(t *testing.T) { + bcIn := testdata.GetBootstrappingConfigurationData() + GetBootstrappingConfigurationMocked(t, bcIn) + GetBootstrappingConfigurationFailErrMocked(t, bcIn) + GetBootstrappingConfigurationFailStatusMocked(t, bcIn) + GetBootstrappingConfigurationFailJSONMocked(t, bcIn) +} + +func TestReportBootstrappingAppliedConfiguration(t *testing.T) { + bcIn := testdata.GetBootstrappingConfigurationData() + ReportBootstrappingAppliedConfigurationMocked(t, bcIn) + ReportBootstrappingAppliedConfigurationFailErrMocked(t, bcIn) + ReportBootstrappingAppliedConfigurationFailStatusMocked(t, bcIn) + ReportBootstrappingAppliedConfigurationFailJSONMocked(t, bcIn) +} + +func TestReportBootstrappingLog(t *testing.T) { + commandIn := testdata.GetBootstrappingContinuousReportData() + ReportBootstrappingLogMocked(t, commandIn) + ReportBootstrappingLogFailErrMocked(t, commandIn) + ReportBootstrappingLogFailStatusMocked(t, commandIn) + ReportBootstrappingLogFailJSONMocked(t, commandIn) +} \ No newline at end of file diff --git a/testdata/boostrapping_data.go b/testdata/boostrapping_data.go new file mode 100644 index 0000000..4e542ff --- /dev/null +++ b/testdata/boostrapping_data.go @@ -0,0 +1,40 @@ +package testdata + +import ( + "github.com/ingrammicro/concerto/api/types" + "encoding/json" +) + +// GetBootstrappingConfigurationData loads test data +func GetBootstrappingConfigurationData() *types.BootstrappingConfiguration { + + attrs := json.RawMessage(`{"fakeAttribute0":"val0","fakeAttribute1":"val1"}`) + test := types.BootstrappingConfiguration{ + Policyfiles: []types.BootstrappingPolicyfile{ + { + ID: "fakeProfileID0", + RevisionID: "fakeProfileRevisionID0", + DownloadURL: "fakeProfileDownloadURL0", + }, + { + ID: "fakeProfileID1", + RevisionID: "fakeProfileRevisionID1", + DownloadURL: "fakeProfileDownloadURL1", + }, + }, + Attributes: &attrs, + AttributeRevisionID: "fakeAttributeRevisionID", + } + + return &test +} + +// GetBootstrappingContinuousReportData loads test data +func GetBootstrappingContinuousReportData() *types.BootstrappingContinuousReport{ + + testBootstrappingContinuousReport := types.BootstrappingContinuousReport{ + Stdout: "Bootstrap log created", + } + + return &testBootstrappingContinuousReport +} From 1955329925955cb73c01cb56f444beecb11acfa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ban=CC=83os=20Lo=CC=81pez?= Date: Fri, 8 Mar 2019 11:48:58 +0100 Subject: [PATCH 16/20] Remove scripts boot execution in bootstrapping and move config files to cio directory --- README.md | 4 ++-- bootstrapping/bootstrapping.go | 27 ++++----------------------- cmdpolling/polling.go | 2 +- utils/config.go | 18 +++++++++--------- 4 files changed, 16 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index d9fdd43..a10ee2b 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Extract the contents with your zip compressor of choice and continue using the s ### Configuration -IMCO CLI configuration will usually be located in your personal folder under `.concerto`. If you are using root, CLI will look for contiguration files under `/etc/imco`. +IMCO CLI configuration will usually be located in your personal folder under `.concerto`. If you are using root, CLI will look for contiguration files under `/etc/cio`. We will assume that you are not root, so create the folder and drop the certificates to this location: ```bash @@ -184,7 +184,7 @@ If you got an error executing IMCO CLI: - check that your internet connection can reach `clients.{IMCO_DOMAIN}` - make sure that your firewall lets you access to - check that `client.xml` is pointing to the correct certificates location -- if `concerto` executes but only shows server commands, you are probably trying to use `concerto` from a commissioned server, and the configuration is being read from `/etc/imco`. If that's the case, you should leave `concerto` configuration untouched so that server commands are available for our remote management. +- if `concerto` executes but only shows server commands, you are probably trying to use `concerto` from a commissioned server, and the configuration is being read from `/etc/cio`. If that's the case, you should leave `concerto` configuration untouched so that server commands are available for our remote management. ## Usage diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index cd330a9..d272113 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -30,7 +30,7 @@ const ( DefaultTimingInterval = 600 // 600 seconds = 10 minutes DefaultTimingSplay = 360 // seconds DefaultThresholdLines = 10 - ProcessLockFile = "imco-bootstrapping.lock" + ProcessLockFile = "cio-bootstrapping.lock" RetriesNumber = 5 ) @@ -103,7 +103,7 @@ func lockFilePath() string { } func workspaceDir() string { - return filepath.Join(os.TempDir(), "imco") + return filepath.Join(os.TempDir(), "cio") } // Returns the full path to the tmp directory @@ -258,11 +258,7 @@ func applyPolicyfiles(ctx context.Context, bootstrappingSvc *blueprint.Bootstrap formatter.PrintError("couldn't report applied status for policy files", err) return err } - if err != nil { - formatter.PrintError("couldn't process policy files", err) - return err - } - return completeBootstrappingSequence(bsProcess) + return err } func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) error { @@ -355,7 +351,7 @@ func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc for _, bsPolicyfile := range bsProcess.policyfiles { command := fmt.Sprintf("cd %s", bsPolicyfile.Path(bsProcess.directoryPath)) if runtime.GOOS == "windows" { - command = fmt.Sprintf("%s\nSET \"PATH=%PATH%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"", command) + command = fmt.Sprintf("%s\nSET \"PATH=%%PATH%%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"", command) } command = fmt.Sprintf("%s\nchef-client -z -j %s", command, bsProcess.attributes.FilePath(bsProcess.directoryPath)) log.Debug(command) @@ -416,18 +412,3 @@ func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService } return bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload) } - -// completeBootstrappingSequence evaluates if the first iteration of policies was completed; If case, execute the "scripts boot" command. -func completeBootstrappingSequence(bsProcess *bootstrappingProcess) error { - log.Debug("completeBootstrappingSequence") - - if !allPolicyfilesSuccessfullyApplied { - log.Debug("run the boot scripts") - //run the boot scripts for the server by executing the scripts boot sub-command (as an external process). - if output, exit, _, _ := utils.RunCmd(strings.Join([]string{os.Args[0], "scripts", "boot"}, " ")); exit != 0 { - return fmt.Errorf("boot scripts run failed with exit code %d and following output: %s", exit, output) - } - allPolicyfilesSuccessfullyApplied = true - } - return nil -} diff --git a/cmdpolling/polling.go b/cmdpolling/polling.go index 07f6274..c97990b 100644 --- a/cmdpolling/polling.go +++ b/cmdpolling/polling.go @@ -19,7 +19,7 @@ import ( const ( DefaultPollingPingTimingIntervalLong = 30 DefaultPollingPingTimingIntervalShort = 5 - ProcessIdFile = "imco-polling.pid" + ProcessIdFile = "cio-polling.pid" ) // Handle signals diff --git a/utils/config.go b/utils/config.go index ae8fd8f..9523b52 100644 --- a/utils/config.go +++ b/utils/config.go @@ -20,18 +20,18 @@ import ( "github.com/mitchellh/go-homedir" ) -const windowsServerConfigFile = "c:\\imco\\client.xml" -const nixServerConfigFile = "/etc/imco/client.xml" +const windowsServerConfigFile = "c:\\cio\\client.xml" +const nixServerConfigFile = "/etc/cio/client.xml" const defaultConcertoEndpoint = "https://clients.concerto.io:886/" -const windowsServerLogFilePath = "c:\\imco\\log\\concerto-client.log" -const windowsServerCaCertPath = "c:\\imco\\client_ssl\\ca_cert.pem" -const windowsServerCertPath = "c:\\imco\\client_ssl\\cert.pem" -const windowsServerKeyPath = "c:\\imco\\client_ssl\\private\\key.pem" +const windowsServerLogFilePath = "c:\\cio\\log\\concerto-client.log" +const windowsServerCaCertPath = "c:\\cio\\client_ssl\\ca_cert.pem" +const windowsServerCertPath = "c:\\cio\\client_ssl\\cert.pem" +const windowsServerKeyPath = "c:\\cio\\client_ssl\\private\\key.pem" const nixServerLogFilePath = "/var/log/concerto-client.log" -const nixServerCaCertPath = "/etc/imco/client_ssl/ca_cert.pem" -const nixServerCertPath = "/etc/imco/client_ssl/cert.pem" -const nixServerKeyPath = "/etc/imco/client_ssl/private/key.pem" +const nixServerCaCertPath = "/etc/cio/client_ssl/ca_cert.pem" +const nixServerCertPath = "/etc/cio/client_ssl/cert.pem" +const nixServerKeyPath = "/etc/cio/client_ssl/private/key.pem" // Config stores configuration file contents type Config struct { From 0ec978e7226da6cec326965625f2fee74320cf85 Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Mon, 11 Mar 2019 13:49:44 +0100 Subject: [PATCH 17/20] Completed test cases (issue #90) - Downloading file --- api/blueprint/bootstrapping_api_mocked.go | 50 +++++++++++++++++++++-- api/blueprint/bootstrapping_api_test.go | 8 +++- testdata/boostrapping_data.go | 14 +++++-- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/api/blueprint/bootstrapping_api_mocked.go b/api/blueprint/bootstrapping_api_mocked.go index 29ef6b8..916ddca 100644 --- a/api/blueprint/bootstrapping_api_mocked.go +++ b/api/blueprint/bootstrapping_api_mocked.go @@ -2,11 +2,11 @@ package blueprint import ( "encoding/json" + "fmt" "github.com/ingrammicro/concerto/api/types" "github.com/ingrammicro/concerto/utils" "github.com/stretchr/testify/assert" "testing" - "fmt" ) // GetBootstrappingConfigurationMocked test mocked function @@ -151,7 +151,7 @@ func ReportBootstrappingAppliedConfigurationFailErrMocked(t *testing.T, commandI // call service payload := make(map[string]interface{}) - cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 400, fmt.Errorf("Mocked error")) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 499, fmt.Errorf("Mocked error")) err = ds.ReportBootstrappingAppliedConfiguration(&payload) assert.NotNil(err, "We are expecting an error") assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") @@ -249,7 +249,7 @@ func ReportBootstrappingLogFailErrMocked(t *testing.T, commandIn *types.Bootstra // call service payload := make(map[string]interface{}) - cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 400, fmt.Errorf("Mocked error")) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 499, fmt.Errorf("Mocked error")) commandOut, _, err := ds.ReportBootstrappingLog(&payload) assert.NotNil(err, "We are expecting an error") @@ -313,4 +313,46 @@ func ReportBootstrappingLogFailJSONMocked(t *testing.T, commandIn *types.Bootstr assert.Contains(err.Error(), "invalid character", "Error message should include the string 'invalid character'") return commandOut -} \ No newline at end of file +} + +// DownloadPolicyfileMocked +func DownloadPolicyfileMocked(t *testing.T, dataIn map[string]string) { + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + urlSource := dataIn["fakeURLToFile"] + pathFile := dataIn["fakeFileDownloadFile"] + + // call service + cs.On("GetFile", urlSource, pathFile).Return(pathFile, 200, nil) + realFileName, status, err := ds.DownloadPolicyfile(urlSource, pathFile) + assert.Nil(err, "Error downloading bootstrapping policy file") + assert.Equal(status, 200, "DownloadPolicyfile returned invalid response") + assert.Equal(realFileName, pathFile, "Invalid downloaded file path") +} + +// DownloadPolicyfileFailErrMocked +func DownloadPolicyfileFailErrMocked(t *testing.T, dataIn map[string]string) { + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + urlSource := dataIn["fakeURLToFile"] + pathFile := dataIn["fakeFileDownloadFile"] + + // call service + cs.On("GetFile", urlSource, pathFile).Return("", 499, fmt.Errorf("Mocked error")) + _, status, err := ds.DownloadPolicyfile(urlSource, pathFile) + assert.NotNil(err, "We are expecting an error") + assert.Equal(status, 499, "DownloadPolicyfile returned an unexpected status code") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") +} diff --git a/api/blueprint/bootstrapping_api_test.go b/api/blueprint/bootstrapping_api_test.go index fa4e0b4..8815905 100644 --- a/api/blueprint/bootstrapping_api_test.go +++ b/api/blueprint/bootstrapping_api_test.go @@ -35,4 +35,10 @@ func TestReportBootstrappingLog(t *testing.T) { ReportBootstrappingLogFailErrMocked(t, commandIn) ReportBootstrappingLogFailStatusMocked(t, commandIn) ReportBootstrappingLogFailJSONMocked(t, commandIn) -} \ No newline at end of file +} + +func TestDownloadPolicyfile(t *testing.T) { + dataIn := testdata.GetBootstrappingDownloadFileData() + DownloadPolicyfileMocked(t, dataIn) + DownloadPolicyfileFailErrMocked(t, dataIn) +} diff --git a/testdata/boostrapping_data.go b/testdata/boostrapping_data.go index 4e542ff..5467995 100644 --- a/testdata/boostrapping_data.go +++ b/testdata/boostrapping_data.go @@ -1,8 +1,8 @@ package testdata import ( - "github.com/ingrammicro/concerto/api/types" "encoding/json" + "github.com/ingrammicro/concerto/api/types" ) // GetBootstrappingConfigurationData loads test data @@ -10,7 +10,7 @@ func GetBootstrappingConfigurationData() *types.BootstrappingConfiguration { attrs := json.RawMessage(`{"fakeAttribute0":"val0","fakeAttribute1":"val1"}`) test := types.BootstrappingConfiguration{ - Policyfiles: []types.BootstrappingPolicyfile{ + Policyfiles: []types.BootstrappingPolicyfile{ { ID: "fakeProfileID0", RevisionID: "fakeProfileRevisionID0", @@ -30,7 +30,7 @@ func GetBootstrappingConfigurationData() *types.BootstrappingConfiguration { } // GetBootstrappingContinuousReportData loads test data -func GetBootstrappingContinuousReportData() *types.BootstrappingContinuousReport{ +func GetBootstrappingContinuousReportData() *types.BootstrappingContinuousReport { testBootstrappingContinuousReport := types.BootstrappingContinuousReport{ Stdout: "Bootstrap log created", @@ -38,3 +38,11 @@ func GetBootstrappingContinuousReportData() *types.BootstrappingContinuousReport return &testBootstrappingContinuousReport } + +//GetBootstrappingDownloadFileData +func GetBootstrappingDownloadFileData() map[string]string { + return map[string]string{ + "fakeURLToFile": "http://fakeURLToFile.xxx/filename.tgz", + "fakeFileDownloadFile": "filename.tgz", + } +} From 4be9ada7074311603d6dfdcd4e57de156e01191d Mon Sep 17 00:00:00 2001 From: Samuel Patino Date: Thu, 14 Mar 2019 12:48:31 +0100 Subject: [PATCH 18/20] Updated bootstrapping process to works in windows platform (Issue #90) --- bootstrapping/bootstrapping.go | 2 +- utils/utils.go | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index d272113..ffaac0b 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -99,7 +99,7 @@ func handleSysSignals(cancelFunc context.CancelFunc) { // Returns the full path to the tmp directory joined with pid management file name func lockFilePath() string { - return filepath.Join(workspaceDir(), ProcessLockFile) + return filepath.Join(os.TempDir(), ProcessLockFile) } func workspaceDir() string { diff --git a/utils/utils.go b/utils/utils.go index 53cab57..f67b5c6 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -10,6 +10,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strings" "time" @@ -68,11 +69,15 @@ func Untar(ctx context.Context, source, target string) error { return err } - cmd := exec.CommandContext(ctx, "tar", "-xzf", source, "-C", target) + tarExecutable := "tar" + if runtime.GOOS == "windows" { + tarExecutable = "C:\\opscode\\chef\\bin\\tar.exe" + } + cmd := exec.CommandContext(ctx, tarExecutable, "-xzf", source, "-C", target) if err := cmd.Run(); err != nil { return err } - + return nil } From 041df51f1992308d70c426df44eebf816c0b81ff Mon Sep 17 00:00:00 2001 From: Pablo Cantera Date: Mon, 18 Mar 2019 10:46:44 +0100 Subject: [PATCH 19/20] Removed unused variable (issue #90) --- bootstrapping/bootstrapping.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index ffaac0b..52c19ca 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -48,8 +48,6 @@ type attributes struct { rawData *json.RawMessage } -var allPolicyfilesSuccessfullyApplied bool - type policyfile types.BootstrappingPolicyfile func (pf policyfile) Name() string { From 2ad5989de9936f352d5a521600fe9dc0b18ea1e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ban=CC=83os=20Lo=CC=81pez?= Date: Tue, 19 Mar 2019 10:21:24 +0100 Subject: [PATCH 20/20] Have bootstrapping server command rename policyfile dir for chef runs on windows (issue #90) --- bootstrapping/bootstrapping.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go index 52c19ca..b730555 100644 --- a/bootstrapping/bootstrapping.go +++ b/bootstrapping/bootstrapping.go @@ -347,11 +347,20 @@ func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc log.Debug("processPolicyfiles") for _, bsPolicyfile := range bsProcess.policyfiles { - command := fmt.Sprintf("cd %s", bsPolicyfile.Path(bsProcess.directoryPath)) + command := fmt.Sprintf("chef-client -z -j %s", bsProcess.attributes.FilePath(bsProcess.directoryPath)) + policyfileDir := bsPolicyfile.Path(bsProcess.directoryPath) + var renamedPolicyfileDir string if runtime.GOOS == "windows" { - command = fmt.Sprintf("%s\nSET \"PATH=%%PATH%%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"", command) + renamedPolicyfileDir = policyfileDir + policyfileDir = filepath.Join(bsProcess.directoryPath, "active") + err := os.Rename(renamedPolicyfileDir, policyfileDir) + if err != nil { + return fmt.Errorf("could not rename %s as %s: %v", renamedPolicyfileDir, policyfileDir, err) + } + command = fmt.Sprintf("SET \"PATH=%%PATH%%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"\n%s", command) } - command = fmt.Sprintf("%s\nchef-client -z -j %s", command, bsProcess.attributes.FilePath(bsProcess.directoryPath)) + command = fmt.Sprintf("cd %s\n%s", policyfileDir, command) + log.Debug(command) // Custom method for chunks processing @@ -394,6 +403,12 @@ func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProc log.Info("completed: ", exitCode) bsProcess.appliedPolicyfileRevisionIDs[bsPolicyfile.ID] = bsPolicyfile.RevisionID + if renamedPolicyfileDir != "" { + err = os.Rename(policyfileDir, renamedPolicyfileDir) + if err != nil { + return fmt.Errorf("could not rename %s as %s back: %v", policyfileDir, renamedPolicyfileDir, err) + } + } } return nil }