diff --git a/Gopkg.lock b/Gopkg.lock index e7ea940..5503c24 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -17,6 +17,12 @@ revision = "4b6ea7319e214d98c938f12692336f7ca9348d6b" version = "v0.10.0" +[[projects]] + branch = "master" + name = "github.com/allan-simon/go-singleinstance" + packages = ["."] + revision = "79edcfdc2dfc93da913f46ae8d9f8a9602250431" + [[projects]] name = "github.com/asaskevich/govalidator" packages = ["."] @@ -91,6 +97,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "92b7325f6a94cda806d124a617bcc6703c943d68c51e2be3182ef4ed42dc35a7" + inputs-digest = "110d97437b0209f38ce5b41683f4bdb8842f608d32b27d184ddc13edb6bfd8c9" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 62d3944..99977a0 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -55,3 +55,7 @@ [[constraint]] name = "github.com/pmezard/go-difflib" revision = "792786c7400a136282c1664665ae0a8db921c6c2" + +[[constraint]] + branch = "master" + name = "github.com/allan-simon/go-singleinstance" diff --git a/README.md b/README.md index d9fdd43..a10ee2b 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Extract the contents with your zip compressor of choice and continue using the s ### Configuration -IMCO CLI configuration will usually be located in your personal folder under `.concerto`. If you are using root, CLI will look for contiguration files under `/etc/imco`. +IMCO CLI configuration will usually be located in your personal folder under `.concerto`. If you are using root, CLI will look for contiguration files under `/etc/cio`. We will assume that you are not root, so create the folder and drop the certificates to this location: ```bash @@ -184,7 +184,7 @@ If you got an error executing IMCO CLI: - check that your internet connection can reach `clients.{IMCO_DOMAIN}` - make sure that your firewall lets you access to - check that `client.xml` is pointing to the correct certificates location -- if `concerto` executes but only shows server commands, you are probably trying to use `concerto` from a commissioned server, and the configuration is being read from `/etc/imco`. If that's the case, you should leave `concerto` configuration untouched so that server commands are available for our remote management. +- if `concerto` executes but only shows server commands, you are probably trying to use `concerto` from a commissioned server, and the configuration is being read from `/etc/cio`. If that's the case, you should leave `concerto` configuration untouched so that server commands are available for our remote management. ## Usage diff --git a/api/blueprint/bootstrapping_api.go b/api/blueprint/bootstrapping_api.go new file mode 100644 index 0000000..14ab9f2 --- /dev/null +++ b/api/blueprint/bootstrapping_api.go @@ -0,0 +1,93 @@ +package blueprint + +import ( + "encoding/json" + "fmt" + + log "github.com/Sirupsen/logrus" + "github.com/ingrammicro/concerto/api/types" + "github.com/ingrammicro/concerto/utils" +) + + +// BootstrappingService manages bootstrapping operations +type BootstrappingService struct { + concertoService utils.ConcertoService +} + +// NewBootstrappingService returns a bootstrapping service +func NewBootstrappingService(concertoService utils.ConcertoService) (*BootstrappingService, error) { + if concertoService == nil { + return nil, fmt.Errorf("must initialize ConcertoService before using it") + } + + return &BootstrappingService{ + concertoService: concertoService, + }, nil + +} + +// GetBootstrappingConfiguration returns the list of policy files as a JSON response with the desired configuration changes +func (bs *BootstrappingService) GetBootstrappingConfiguration() (bootstrappingConfigurations *types.BootstrappingConfiguration, status int, err error) { + log.Debug("GetBootstrappingConfiguration") + + data, status, err := bs.concertoService.Get("/blueprint/configuration") + if err != nil { + return nil, status, err + } + + if err = utils.CheckStandardStatus(status, data); err != nil { + return nil, status, err + } + + if err = json.Unmarshal(data, &bootstrappingConfigurations); err != nil { + return nil, status, err + } + + return bootstrappingConfigurations, status, nil +} + +// ReportBootstrappingAppliedConfiguration +func (bs *BootstrappingService) ReportBootstrappingAppliedConfiguration(BootstrappingAppliedConfigurationVector *map[string]interface{}) (err error) { + log.Debug("ReportBootstrappingAppliedConfiguration") + + data, status, err := bs.concertoService.Put("/blueprint/applied_configuration", BootstrappingAppliedConfigurationVector) + if err != nil { + return err + } + + if err = utils.CheckStandardStatus(status, data); err != nil { + return err + } + + return nil +} + +// ReportBootstrappingLog reports a policy files application result +func (bs *BootstrappingService) ReportBootstrappingLog(BootstrappingContinuousReportVector *map[string]interface{}) (command *types.BootstrappingContinuousReport, status int, err error) { + log.Debug("ReportBootstrappingLog") + + data, status, err := bs.concertoService.Post("/blueprint/bootstrap_logs", BootstrappingContinuousReportVector) + if err != nil { + return nil, status, err + } + + if err = json.Unmarshal(data, &command); err != nil { + return nil, status, err + } + + return command, status, nil +} + + +// DownloadPolicyfile gets a file from given url saving file into given file path +func (bs *BootstrappingService) DownloadPolicyfile(url string, filePath string) (realFileName string, status int, err error) { + log.Debug("DownloadPolicyfile") + + realFileName, status, err = bs.concertoService.GetFile(url, filePath) + if err != nil { + return realFileName, status, err + } + + return realFileName, status, nil +} \ No newline at end of file diff --git a/api/blueprint/bootstrapping_api_mocked.go b/api/blueprint/bootstrapping_api_mocked.go new file mode 100644 index 0000000..916ddca --- /dev/null +++ b/api/blueprint/bootstrapping_api_mocked.go @@ -0,0 +1,358 @@ +package blueprint + +import ( + "encoding/json" + "fmt" + "github.com/ingrammicro/concerto/api/types" + "github.com/ingrammicro/concerto/utils" + "github.com/stretchr/testify/assert" + "testing" +) + +// GetBootstrappingConfigurationMocked test mocked function +func GetBootstrappingConfigurationMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(bcConfIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 200, nil) + bcConfOut, status, err := ds.GetBootstrappingConfiguration() + assert.Nil(err, "Error getting bootstrapping configuration") + assert.Equal(status, 200, "GetBootstrappingConfiguration returned invalid response") + assert.Equal(bcConfIn, bcConfOut, "GetBootstrappingConfiguration returned different services") + return bcConfOut +} + +// GetBootstrappingConfigurationFailErrMocked test mocked function +func GetBootstrappingConfigurationFailErrMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(bcConfIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 404, fmt.Errorf("Mocked error")) + bcConfOut, _, err := ds.GetBootstrappingConfiguration() + + assert.NotNil(err, "We are expecting an error") + assert.Nil(bcConfOut, "Expecting nil output") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") + + return bcConfOut +} + +// GetBootstrappingConfigurationFailStatusMocked test mocked function +func GetBootstrappingConfigurationFailStatusMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(bcConfIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 499, nil) + bcConfOut, status, err := ds.GetBootstrappingConfiguration() + + assert.NotNil(err, "We are expecting an status code error") + assert.Nil(bcConfOut, "Expecting nil output") + assert.Equal(499, status, "Expecting http code 499") + assert.Contains(err.Error(), "499", "Error should contain http code 499") + + return bcConfOut +} + +// GetBootstrappingConfigurationFailJSONMocked test mocked function +func GetBootstrappingConfigurationFailJSONMocked(t *testing.T, bcConfIn *types.BootstrappingConfiguration) *types.BootstrappingConfiguration { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // wrong json + dIn := []byte{10, 20, 30} + + // call service + cs.On("Get", "/blueprint/configuration").Return(dIn, 200, nil) + bcConfOut, _, err := ds.GetBootstrappingConfiguration() + + assert.NotNil(err, "We are expecting a marshalling error") + assert.Nil(bcConfOut, "Expecting nil output") + + return bcConfOut +} + +// ReportBootstrappingAppliedConfigurationMocked test mocked function +func ReportBootstrappingAppliedConfigurationMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dOut, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dOut, 200, nil) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.Nil(err, "Error getting bootstrapping command") +} + +// ReportBootstrappingAppliedConfigurationFailErrMocked test mocked function +func ReportBootstrappingAppliedConfigurationFailErrMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 499, fmt.Errorf("Mocked error")) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.NotNil(err, "We are expecting an error") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") +} + +// ReportBootstrappingAppliedConfigurationFailStatusMocked test mocked function +func ReportBootstrappingAppliedConfigurationFailStatusMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 499, fmt.Errorf("Error 499 Mocked error")) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.NotNil(err, "We are expecting a status code error") + assert.Contains(err.Error(), "499", "Error should contain http code 499") +} + +// ReportBootstrappingAppliedConfigurationFailJSONMocked test mocked function +func ReportBootstrappingAppliedConfigurationFailJSONMocked(t *testing.T, commandIn *types.BootstrappingConfiguration) { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // wrong json + dIn := []byte{0} + + // call service + payload := make(map[string]interface{}) + cs.On("Put", fmt.Sprintf("/blueprint/applied_configuration"), &payload).Return(dIn, 499, nil) + err = ds.ReportBootstrappingAppliedConfiguration(&payload) + assert.Contains(err.Error(), "499", "Error should contain http code 499") +} + +// ReportBootstrappingLogMocked test mocked function +func ReportBootstrappingLogMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dOut, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dOut, 200, nil) + commandOut, status, err := ds.ReportBootstrappingLog(&payload) + + assert.Nil(err, "Error posting report command") + assert.Equal(status, 200, "ReportBootstrappingLog returned invalid response") + assert.Equal(commandOut.Stdout, "Bootstrap log created", "ReportBootstrapLog returned unexpected message") + + return commandOut +} + +// ReportBootstrappingLogFailErrMocked test mocked function +func ReportBootstrappingLogFailErrMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 499, fmt.Errorf("Mocked error")) + commandOut, _, err := ds.ReportBootstrappingLog(&payload) + + assert.NotNil(err, "We are expecting an error") + assert.Nil(commandOut, "Expecting nil output") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") + + return commandOut +} + +// ReportBootstrappingLogFailStatusMocked test mocked function +func ReportBootstrappingLogFailStatusMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // to json + dIn, err := json.Marshal(commandIn) + assert.Nil(err, "Bootstrapping test data corrupted") + + dIn = nil + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 499, fmt.Errorf("Error 499 Mocked error")) + commandOut, status, err := ds.ReportBootstrappingLog(&payload) + + assert.Equal(status, 499, "ReportBootstrappingLog returned an unexpected status code") + assert.NotNil(err, "We are expecting a status code error") + assert.Nil(commandOut, "Expecting nil output") + assert.Contains(err.Error(), "499", "Error should contain http code 499") + + return commandOut +} + +// ReportBootstrappingLogFailJSONMocked test mocked function +func ReportBootstrappingLogFailJSONMocked(t *testing.T, commandIn *types.BootstrappingContinuousReport) *types.BootstrappingContinuousReport { + + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + // wrong json + dIn := []byte{10, 20, 30} + + // call service + payload := make(map[string]interface{}) + cs.On("Post", fmt.Sprintf("/blueprint/bootstrap_logs"), &payload).Return(dIn, 200, nil) + commandOut, _, err := ds.ReportBootstrappingLog(&payload) + + assert.NotNil(err, "We are expecting a marshalling error") + assert.Nil(commandOut, "Expecting nil output") + assert.Contains(err.Error(), "invalid character", "Error message should include the string 'invalid character'") + + return commandOut +} + +// DownloadPolicyfileMocked +func DownloadPolicyfileMocked(t *testing.T, dataIn map[string]string) { + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + urlSource := dataIn["fakeURLToFile"] + pathFile := dataIn["fakeFileDownloadFile"] + + // call service + cs.On("GetFile", urlSource, pathFile).Return(pathFile, 200, nil) + realFileName, status, err := ds.DownloadPolicyfile(urlSource, pathFile) + assert.Nil(err, "Error downloading bootstrapping policy file") + assert.Equal(status, 200, "DownloadPolicyfile returned invalid response") + assert.Equal(realFileName, pathFile, "Invalid downloaded file path") +} + +// DownloadPolicyfileFailErrMocked +func DownloadPolicyfileFailErrMocked(t *testing.T, dataIn map[string]string) { + assert := assert.New(t) + + // wire up + cs := &utils.MockConcertoService{} + ds, err := NewBootstrappingService(cs) + assert.Nil(err, "Couldn't load bootstrapping service") + assert.NotNil(ds, "Bootstrapping service not instanced") + + urlSource := dataIn["fakeURLToFile"] + pathFile := dataIn["fakeFileDownloadFile"] + + // call service + cs.On("GetFile", urlSource, pathFile).Return("", 499, fmt.Errorf("Mocked error")) + _, status, err := ds.DownloadPolicyfile(urlSource, pathFile) + assert.NotNil(err, "We are expecting an error") + assert.Equal(status, 499, "DownloadPolicyfile returned an unexpected status code") + assert.Equal(err.Error(), "Mocked error", "Error should be 'Mocked error'") +} diff --git a/api/blueprint/bootstrapping_api_test.go b/api/blueprint/bootstrapping_api_test.go new file mode 100644 index 0000000..8815905 --- /dev/null +++ b/api/blueprint/bootstrapping_api_test.go @@ -0,0 +1,44 @@ +package blueprint + +import ( + "github.com/ingrammicro/concerto/testdata" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewBootstrappingServiceNil(t *testing.T) { + assert := assert.New(t) + rs, err := NewBootstrappingService(nil) + assert.Nil(rs, "Uninitialized service should return nil") + assert.NotNil(err, "Uninitialized service should return error") +} + +func TestGetBootstrappingConfiguration(t *testing.T) { + bcIn := testdata.GetBootstrappingConfigurationData() + GetBootstrappingConfigurationMocked(t, bcIn) + GetBootstrappingConfigurationFailErrMocked(t, bcIn) + GetBootstrappingConfigurationFailStatusMocked(t, bcIn) + GetBootstrappingConfigurationFailJSONMocked(t, bcIn) +} + +func TestReportBootstrappingAppliedConfiguration(t *testing.T) { + bcIn := testdata.GetBootstrappingConfigurationData() + ReportBootstrappingAppliedConfigurationMocked(t, bcIn) + ReportBootstrappingAppliedConfigurationFailErrMocked(t, bcIn) + ReportBootstrappingAppliedConfigurationFailStatusMocked(t, bcIn) + ReportBootstrappingAppliedConfigurationFailJSONMocked(t, bcIn) +} + +func TestReportBootstrappingLog(t *testing.T) { + commandIn := testdata.GetBootstrappingContinuousReportData() + ReportBootstrappingLogMocked(t, commandIn) + ReportBootstrappingLogFailErrMocked(t, commandIn) + ReportBootstrappingLogFailStatusMocked(t, commandIn) + ReportBootstrappingLogFailJSONMocked(t, commandIn) +} + +func TestDownloadPolicyfile(t *testing.T) { + dataIn := testdata.GetBootstrappingDownloadFileData() + DownloadPolicyfileMocked(t, dataIn) + DownloadPolicyfileFailErrMocked(t, dataIn) +} diff --git a/api/types/bootstrapping.go b/api/types/bootstrapping.go new file mode 100644 index 0000000..1fee663 --- /dev/null +++ b/api/types/bootstrapping.go @@ -0,0 +1,28 @@ +package types + +import ( + "encoding/json" +) + +type BootstrappingConfiguration struct { + Policyfiles []BootstrappingPolicyfile `json:"policyfiles,omitempty" header:"POLICY FILES" show:"nolist"` + Attributes *json.RawMessage `json:"attributes,omitempty" header:"ATTRIBUTES" show:"nolist"` + AttributeRevisionID string `json:"attribute_revision_id,omitempty" header:"ATTRIBUTE REVISION ID"` +} + +type BootstrappingPolicyfile struct { + ID string `json:"id,omitempty" header:"ID"` + RevisionID string `json:"revision_id,omitempty" header:"REVISION ID"` + DownloadURL string `json:"download_url,omitempty" header:"DOWNLOAD URL"` +} + +type BootstrappingContinuousReport struct { + Stdout string `json:"stdout" header:"STDOUT"` +} + +type BootstrappingAppliedConfiguration struct { + StartedAt string `json:"started_at,omitempty" header:"STARTED AT"` + FinishedAt string `json:"finished_at,omitempty" header:"FINISHED AT"` + PolicyfileRevisionIDs string `json:"policyfile_revision_ids,omitempty" header:"POLICY FILE REVISION IDS" show:"nolist"` + AttributeRevisionID string `json:"attribute_revision_id,omitempty" header:"ATTRIBUTE REVISION ID"` +} diff --git a/bootstrapping/bootstrapping.go b/bootstrapping/bootstrapping.go new file mode 100644 index 0000000..b730555 --- /dev/null +++ b/bootstrapping/bootstrapping.go @@ -0,0 +1,427 @@ +package bootstrapping + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/url" + "os" + "os/signal" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + log "github.com/Sirupsen/logrus" + singleinstance "github.com/allan-simon/go-singleinstance" + "github.com/codegangsta/cli" + "github.com/ingrammicro/concerto/api/blueprint" + "github.com/ingrammicro/concerto/api/types" + "github.com/ingrammicro/concerto/cmd" + "github.com/ingrammicro/concerto/utils" + "github.com/ingrammicro/concerto/utils/format" +) + +const ( + //DefaultTimingInterval Default period for looping + DefaultTimingInterval = 600 // 600 seconds = 10 minutes + DefaultTimingSplay = 360 // seconds + DefaultThresholdLines = 10 + ProcessLockFile = "cio-bootstrapping.lock" + RetriesNumber = 5 +) + +type bootstrappingProcess struct { + startedAt time.Time + finishedAt time.Time + policyfiles []policyfile + attributes attributes + thresholdLines int + directoryPath string + appliedPolicyfileRevisionIDs map[string]string +} +type attributes struct { + revisionID string + rawData *json.RawMessage +} + +type policyfile types.BootstrappingPolicyfile + +func (pf policyfile) Name() string { + return strings.Join([]string{pf.ID, "-", pf.RevisionID}, "") +} + +func (pf *policyfile) FileName() string { + return strings.Join([]string{pf.Name(), "tgz"}, ".") +} + +func (pf *policyfile) QueryURL() (string, error) { + if pf.DownloadURL == "" { + return "", fmt.Errorf("obtaining URL query: empty download URL") + } + url, err := url.Parse(pf.DownloadURL) + if err != nil { + return "", fmt.Errorf("parsing URL to extract query: %v", err) + } + return fmt.Sprintf("%s?%s", url.Path, url.RawQuery), nil +} + +func (pf *policyfile) TarballPath(dir string) string { + return filepath.Join(dir, pf.FileName()) +} + +func (pf *policyfile) Path(dir string) string { + return filepath.Join(dir, pf.Name()) +} + +func (a *attributes) FileName() string { + return fmt.Sprintf("attrs-%s.json", a.revisionID) +} + +func (a *attributes) FilePath(dir string) string { + return filepath.Join(dir, a.FileName()) +} + +// Handle signals +func handleSysSignals(cancelFunc context.CancelFunc) { + log.Debug("handleSysSignals") + + gracefulStop := make(chan os.Signal, 1) + signal.Notify(gracefulStop, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL) + log.Debug("Ending, signal detected:", <-gracefulStop) + cancelFunc() +} + +// Returns the full path to the tmp directory joined with pid management file name +func lockFilePath() string { + return filepath.Join(os.TempDir(), ProcessLockFile) +} + +func workspaceDir() string { + return filepath.Join(os.TempDir(), "cio") +} + +// Returns the full path to the tmp directory +func generateWorkspaceDir() error { + dir := workspaceDir() + dirInfo, err := os.Stat(dir) + if err != nil { + err := os.Mkdir(dir, 0777) + if err != nil { + return err + } + } else { + if !dirInfo.Mode().IsDir() { + return fmt.Errorf("%s exists but is not a directory", dir) + } + } + return nil +} + +// Start the bootstrapping process +func start(c *cli.Context) error { + log.Debug("start") + + err := generateWorkspaceDir() + if err != nil { + return err + } + lockFile, err := singleinstance.CreateLockFile(lockFilePath()) + if err != nil { + return err + } + defer lockFile.Close() + + formatter := format.GetFormatter() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go handleSysSignals(cancel) + + timingInterval := c.Int64("interval") + if !(timingInterval > 0) { + timingInterval = DefaultTimingInterval + } + + timingSplay := c.Int64("splay") + if !(timingSplay > 0) { + timingSplay = DefaultTimingSplay + } + + thresholdLines := c.Int("lines") + if !(thresholdLines > 0) { + thresholdLines = DefaultThresholdLines + } + log.Debug("routine lines threshold: ", thresholdLines) + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c) + for { + applyPolicyfiles(ctx, bootstrappingSvc, formatter, thresholdLines) + + // Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance) + ticker := time.NewTicker(time.Duration(timingInterval+int64(r.Intn(int(timingSplay)))) * time.Second) + + select { + case <-ticker.C: + log.Debug("ticker") + case <-ctx.Done(): + log.Debug(ctx.Err()) + log.Debug("closing bootstrapping") + } + ticker.Stop() + if ctx.Err() != nil { + break + } + } + + return nil +} + +// Stop the bootstrapping process +func stop(c *cli.Context) error { + log.Debug("cmdStop") + + formatter := format.GetFormatter() + if err := utils.StopProcess(lockFilePath()); err != nil { + formatter.PrintFatal("cannot stop the bootstrapping process", err) + } + + log.Info("Bootstrapping routine successfully stopped") + return nil +} + +// Subsidiary routine for commands processing +func applyPolicyfiles(ctx context.Context, bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) error { + log.Debug("applyPolicyfiles") + + // Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes + bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration() + if err == nil && status != 200 { + err = fmt.Errorf("received non-ok %d response", status) + } + if err != nil { + formatter.PrintError("couldn't receive bootstrapping data", err) + return err + } + err = generateWorkspaceDir() + if err != nil { + formatter.PrintError("couldn't generated workspace directory", err) + return err + } + bsProcess := &bootstrappingProcess{ + startedAt: time.Now().UTC(), + thresholdLines: thresholdLines, + directoryPath: workspaceDir(), + appliedPolicyfileRevisionIDs: make(map[string]string), + } + + // proto structures + err = initializePrototype(bsConfiguration, bsProcess) + if err != nil { + formatter.PrintError("couldn't initialize prototype", err) + return err + } + // For every policyfile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... + err = downloadPolicyfiles(ctx, bootstrappingSvc, bsProcess) + if err != nil { + formatter.PrintError("couldn't download policy files", err) + return err + } + //... and clean off any tarball that is no longer needed. + err = cleanObsoletePolicyfiles(bsProcess) + if err != nil { + formatter.PrintError("couldn't clean obsolete policy files", err) + return err + } + // Store the attributes as JSON in a file with name `attrs-.json` + err = saveAttributes(bsProcess) + if err != nil { + formatter.PrintError("couldn't save attributes for policy files", err) + return err + } + // Process tarballs policies + err = processPolicyfiles(bootstrappingSvc, bsProcess) + // Finishing time + bsProcess.finishedAt = time.Now().UTC() + + // Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to + log.Debug("reporting applied policy files") + reportErr := reportAppliedConfiguration(bootstrappingSvc, bsProcess) + if reportErr != nil { + formatter.PrintError("couldn't report applied status for policy files", err) + return err + } + return err +} + +func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) error { + log.Debug("initializePrototype") + + // Attributes + bsProcess.attributes.revisionID = bsConfiguration.AttributeRevisionID + bsProcess.attributes.rawData = bsConfiguration.Attributes + + // Policies + for _, bsConfPolicyfile := range bsConfiguration.Policyfiles { + bsProcess.policyfiles = append(bsProcess.policyfiles, policyfile(bsConfPolicyfile)) + } + log.Debug(bsProcess) + return nil +} + +// downloadPolicyfiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ... +func downloadPolicyfiles(ctx context.Context, bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { + log.Debug("downloadPolicyfiles") + + for _, bsPolicyfile := range bsProcess.policyfiles { + tarballPath := bsPolicyfile.TarballPath(bsProcess.directoryPath) + log.Debug("downloading: ", tarballPath) + queryURL, err := bsPolicyfile.QueryURL() + if err != nil { + return err + } + _, status, err := bootstrappingSvc.DownloadPolicyfile(queryURL, tarballPath) + if err == nil && status != 200 { + err = fmt.Errorf("obtained non-ok response when downloading policyfile %s", queryURL) + } + if err != nil { + return err + } + if err = utils.Untar(ctx, tarballPath, bsPolicyfile.Path(bsProcess.directoryPath)); err != nil { + return err + } + } + return nil +} + +// cleanObsoletePolicyfiles cleans off any tarball that is no longer needed. +func cleanObsoletePolicyfiles(bsProcess *bootstrappingProcess) error { + log.Debug("cleanObsoletePolicyfiles") + + // evaluates working folder + deletableFiles, err := ioutil.ReadDir(bsProcess.directoryPath) + if err != nil { + return err + } + + // builds an array of currently processable files at this looping time + currentlyProcessableFiles := []string{bsProcess.attributes.FileName()} // saved attributes file name + for _, bsPolicyFile := range bsProcess.policyfiles { + currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.FileName()) // Downloaded tgz file names + currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.Name()) // Uncompressed folder names + } + + // removes from deletableFiles array the policy files currently applied + for _, f := range deletableFiles { + if !utils.Contains(currentlyProcessableFiles, f.Name()) { + log.Debug("removing: ", f.Name()) + if err := os.RemoveAll(filepath.Join(bsProcess.directoryPath, f.Name())); err != nil { + return err + } + } + } + return nil +} + +// saveAttributes stores the attributes as JSON in a file with name `attrs-.json` +func saveAttributes(bsProcess *bootstrappingProcess) error { + log.Debug("saveAttributes") + + attrs, err := json.Marshal(bsProcess.attributes.rawData) + if err != nil { + return err + } + if err := ioutil.WriteFile(bsProcess.attributes.FilePath(bsProcess.directoryPath), attrs, 0600); err != nil { + return err + } + return nil +} + +// processPolicyfiles applies for each policy the required chef commands, reporting in bunches of N lines +func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { + log.Debug("processPolicyfiles") + + for _, bsPolicyfile := range bsProcess.policyfiles { + command := fmt.Sprintf("chef-client -z -j %s", bsProcess.attributes.FilePath(bsProcess.directoryPath)) + policyfileDir := bsPolicyfile.Path(bsProcess.directoryPath) + var renamedPolicyfileDir string + if runtime.GOOS == "windows" { + renamedPolicyfileDir = policyfileDir + policyfileDir = filepath.Join(bsProcess.directoryPath, "active") + err := os.Rename(renamedPolicyfileDir, policyfileDir) + if err != nil { + return fmt.Errorf("could not rename %s as %s: %v", renamedPolicyfileDir, policyfileDir, err) + } + command = fmt.Sprintf("SET \"PATH=%%PATH%%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"\n%s", command) + } + command = fmt.Sprintf("cd %s\n%s", policyfileDir, command) + + log.Debug(command) + + // Custom method for chunks processing + fn := func(chunk string) error { + log.Debug("sendChunks") + err := utils.Retry(RetriesNumber, time.Second, func() error { + log.Debug("Sending: ", chunk) + + commandIn := map[string]interface{}{ + "stdout": chunk, + } + + _, statusCode, err := bootstrappingSvc.ReportBootstrappingLog(&commandIn) + switch { + // 0<100 error cases?? + case statusCode == 0: + return fmt.Errorf("communication error %v %v", statusCode, err) + case statusCode >= 500: + return fmt.Errorf("server error %v %v", statusCode, err) + case statusCode >= 400: + return fmt.Errorf("client error %v %v", statusCode, err) + default: + return nil + } + }) + + if err != nil { + return fmt.Errorf("cannot send the chunk data, %v", err) + } + return nil + } + + exitCode, err := utils.RunContinuousCmd(fn, command, -1, bsProcess.thresholdLines) + if err == nil && exitCode != 0 { + err = fmt.Errorf("policyfile application exited with %d code", exitCode) + } + if err != nil { + return err + } + + log.Info("completed: ", exitCode) + bsProcess.appliedPolicyfileRevisionIDs[bsPolicyfile.ID] = bsPolicyfile.RevisionID + if renamedPolicyfileDir != "" { + err = os.Rename(policyfileDir, renamedPolicyfileDir) + if err != nil { + return fmt.Errorf("could not rename %s as %s back: %v", policyfileDir, renamedPolicyfileDir, err) + } + } + } + return nil +} + +// reportAppliedConfiguration Inform the platform of applied changes +func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error { + log.Debug("reportAppliedConfiguration") + + payload := map[string]interface{}{ + "started_at": bsProcess.startedAt, + "finished_at": bsProcess.finishedAt, + "policyfile_revision_ids": bsProcess.appliedPolicyfileRevisionIDs, + "attribute_revision_id": bsProcess.attributes.revisionID, + } + return bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload) +} diff --git a/bootstrapping/subcommands.go b/bootstrapping/subcommands.go new file mode 100644 index 0000000..119ab29 --- /dev/null +++ b/bootstrapping/subcommands.go @@ -0,0 +1,37 @@ +package bootstrapping + +import ( + "github.com/codegangsta/cli" +) + +func SubCommands() []cli.Command { + return []cli.Command{ + { + Name: "start", + Usage: "Starts a bootstrapping routine to check and execute required activities", + Action: start, + Flags: []cli.Flag{ + cli.Int64Flag{ + Name: "interval, i", + Usage: "The frequency (in seconds) at which the bootstrapping runs", + Value: DefaultTimingInterval, + }, + cli.Int64Flag{ + Name: "splay, s", + Usage: "A random number between zero and splay that is added to interval (seconds)", + Value: DefaultTimingSplay, + }, + cli.IntFlag{ + Name: "lines, l", + Usage: "Maximum lines threshold per response chunk", + Value: DefaultThresholdLines, + }, + }, + }, + { + Name: "stop", + Usage: "Stops the running bootstrapping process", + Action: stop, + }, + } +} diff --git a/cmd/bootstrapping_cmd.go b/cmd/bootstrapping_cmd.go new file mode 100644 index 0000000..9394b3f --- /dev/null +++ b/cmd/bootstrapping_cmd.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "github.com/codegangsta/cli" + "github.com/ingrammicro/concerto/api/blueprint" + "github.com/ingrammicro/concerto/utils" + "github.com/ingrammicro/concerto/utils/format" +) + +// WireUpBootstrapping prepares common resources to send request to API +func WireUpBootstrapping(c *cli.Context) (ds *blueprint.BootstrappingService, f format.Formatter) { + + f = format.GetFormatter() + + config, err := utils.GetConcertoConfig() + if err != nil { + f.PrintFatal("Couldn't wire up config", err) + } + hcs, err := utils.NewHTTPConcertoService(config) + if err != nil { + f.PrintFatal("Couldn't wire up concerto service", err) + } + ds, err = blueprint.NewBootstrappingService(hcs) + if err != nil { + f.PrintFatal("Couldn't wire up serverPlan service", err) + } + + return ds, f +} \ No newline at end of file diff --git a/cmdpolling/continuousreport.go b/cmdpolling/continuousreport.go index 6183388..568314f 100644 --- a/cmdpolling/continuousreport.go +++ b/cmdpolling/continuousreport.go @@ -15,7 +15,6 @@ import ( const ( RetriesNumber = 5 - RetriesFactor = 3 DefaultThresholdTime = 10 ) @@ -43,7 +42,7 @@ func cmdContinuousReportRun(c *cli.Context) error { // Custom method for chunks processing fn := func(chunk string) error { log.Debug("sendChunks") - err := retry(RetriesNumber, time.Second, func() error { + err := utils.Retry(RetriesNumber, time.Second, func() error { log.Debug("Sending: ", chunk) commandIn := map[string]interface{}{ @@ -70,7 +69,7 @@ func cmdContinuousReportRun(c *cli.Context) error { return nil } - exitCode, err := utils.RunContinuousCmd(fn, cmdArg, thresholdTime) + exitCode, err := utils.RunContinuousCmd(fn, cmdArg, thresholdTime, -1) if err != nil { formatter.PrintFatal("cannot process continuous report command", err) } @@ -79,17 +78,3 @@ func cmdContinuousReportRun(c *cli.Context) error { os.Exit(exitCode) return nil } - -func retry(attempts int, sleep time.Duration, fn func() error) error { - log.Debug("retry") - - if err := fn(); err != nil { - if attempts--; attempts > 0 { - log.Debug("Waiting to retry: ", sleep) - time.Sleep(sleep) - return retry(attempts, RetriesFactor*sleep, fn) - } - return err - } - return nil -} diff --git a/cmdpolling/polling.go b/cmdpolling/polling.go index 07f6274..c97990b 100644 --- a/cmdpolling/polling.go +++ b/cmdpolling/polling.go @@ -19,7 +19,7 @@ import ( const ( DefaultPollingPingTimingIntervalLong = 30 DefaultPollingPingTimingIntervalShort = 5 - ProcessIdFile = "imco-polling.pid" + ProcessIdFile = "cio-polling.pid" ) // Handle signals diff --git a/main.go b/main.go index cd170f5..fb8383e 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + //"context" "fmt" "os" "sort" @@ -11,6 +12,7 @@ import ( "github.com/ingrammicro/concerto/blueprint/scripts" "github.com/ingrammicro/concerto/blueprint/services" "github.com/ingrammicro/concerto/blueprint/templates" + "github.com/ingrammicro/concerto/bootstrapping" "github.com/ingrammicro/concerto/brownfield" cl_prov "github.com/ingrammicro/concerto/cloud/cloud_providers" "github.com/ingrammicro/concerto/cloud/generic_images" @@ -68,6 +70,13 @@ var ServerCommands = []cli.Command{ cmdpolling.SubCommands(), ), }, + { + Name: "bootstrap", + Usage: "Manages bootstrapping commands", + Subcommands: append( + bootstrapping.SubCommands(), + ), + }, } var BlueprintCommands = []cli.Command{ diff --git a/testdata/boostrapping_data.go b/testdata/boostrapping_data.go new file mode 100644 index 0000000..5467995 --- /dev/null +++ b/testdata/boostrapping_data.go @@ -0,0 +1,48 @@ +package testdata + +import ( + "encoding/json" + "github.com/ingrammicro/concerto/api/types" +) + +// GetBootstrappingConfigurationData loads test data +func GetBootstrappingConfigurationData() *types.BootstrappingConfiguration { + + attrs := json.RawMessage(`{"fakeAttribute0":"val0","fakeAttribute1":"val1"}`) + test := types.BootstrappingConfiguration{ + Policyfiles: []types.BootstrappingPolicyfile{ + { + ID: "fakeProfileID0", + RevisionID: "fakeProfileRevisionID0", + DownloadURL: "fakeProfileDownloadURL0", + }, + { + ID: "fakeProfileID1", + RevisionID: "fakeProfileRevisionID1", + DownloadURL: "fakeProfileDownloadURL1", + }, + }, + Attributes: &attrs, + AttributeRevisionID: "fakeAttributeRevisionID", + } + + return &test +} + +// GetBootstrappingContinuousReportData loads test data +func GetBootstrappingContinuousReportData() *types.BootstrappingContinuousReport { + + testBootstrappingContinuousReport := types.BootstrappingContinuousReport{ + Stdout: "Bootstrap log created", + } + + return &testBootstrappingContinuousReport +} + +//GetBootstrappingDownloadFileData +func GetBootstrappingDownloadFileData() map[string]string { + return map[string]string{ + "fakeURLToFile": "http://fakeURLToFile.xxx/filename.tgz", + "fakeFileDownloadFile": "filename.tgz", + } +} diff --git a/utils/config.go b/utils/config.go index ae8fd8f..9523b52 100644 --- a/utils/config.go +++ b/utils/config.go @@ -20,18 +20,18 @@ import ( "github.com/mitchellh/go-homedir" ) -const windowsServerConfigFile = "c:\\imco\\client.xml" -const nixServerConfigFile = "/etc/imco/client.xml" +const windowsServerConfigFile = "c:\\cio\\client.xml" +const nixServerConfigFile = "/etc/cio/client.xml" const defaultConcertoEndpoint = "https://clients.concerto.io:886/" -const windowsServerLogFilePath = "c:\\imco\\log\\concerto-client.log" -const windowsServerCaCertPath = "c:\\imco\\client_ssl\\ca_cert.pem" -const windowsServerCertPath = "c:\\imco\\client_ssl\\cert.pem" -const windowsServerKeyPath = "c:\\imco\\client_ssl\\private\\key.pem" +const windowsServerLogFilePath = "c:\\cio\\log\\concerto-client.log" +const windowsServerCaCertPath = "c:\\cio\\client_ssl\\ca_cert.pem" +const windowsServerCertPath = "c:\\cio\\client_ssl\\cert.pem" +const windowsServerKeyPath = "c:\\cio\\client_ssl\\private\\key.pem" const nixServerLogFilePath = "/var/log/concerto-client.log" -const nixServerCaCertPath = "/etc/imco/client_ssl/ca_cert.pem" -const nixServerCertPath = "/etc/imco/client_ssl/cert.pem" -const nixServerKeyPath = "/etc/imco/client_ssl/private/key.pem" +const nixServerCaCertPath = "/etc/cio/client_ssl/ca_cert.pem" +const nixServerCertPath = "/etc/cio/client_ssl/cert.pem" +const nixServerKeyPath = "/etc/cio/client_ssl/private/key.pem" // Config stores configuration file contents type Config struct { diff --git a/utils/exec.go b/utils/exec.go index 7268bee..f16fedc 100644 --- a/utils/exec.go +++ b/utils/exec.go @@ -19,6 +19,7 @@ import ( const ( TimeStampLayout = "2006-01-02T15:04:05.000000-07:00" TimeLayoutYYYYMMDDHHMMSS = "20060102150405" + RetriesFactor = 3 ) func extractExitCode(err error) int { @@ -234,7 +235,9 @@ func RunTracedCmd(command string) (exitCode int, stdOut string, stdErr string, s return } -func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime int) (int, error) { +// thresholdTime > 0 continuous report +// thresholdLines > 0 bootstrapping +func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime int, thresholdLines int) (int, error) { log.Debug("RunContinuousCmd") // Saves script/command in a temp file @@ -256,20 +259,19 @@ func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime } chunk := "" - nTime := 0 + nLines, nTime := 0, 0 timeStart := time.Now() scanner := bufio.NewScanner(bufio.NewReader(stdout)) for scanner.Scan() { chunk = strings.Join([]string{chunk, scanner.Text(), "\n"}, "") + nLines++ nTime = int(time.Now().Sub(timeStart).Seconds()) - if nTime >= thresholdTime { - if err := fn(chunk); err != nil { - nTime = 0 - } else { + if (thresholdTime > 0 && nTime >= thresholdTime) || (thresholdLines > 0 && nLines >= thresholdLines) { + if err := fn(chunk); err == nil { chunk = "" - nTime = 0 } + nLines, nTime = 0, 0 timeStart = time.Now() } } @@ -291,3 +293,17 @@ func RunContinuousCmd(fn func(chunk string) error, command string, thresholdTime return exitCode, nil } + +func Retry(attempts int, sleep time.Duration, fn func() error) error { + log.Debug("Retry") + + if err := fn(); err != nil { + if attempts--; attempts > 0 { + log.Debug("Waiting to retry: ", sleep) + time.Sleep(sleep) + return Retry(attempts, RetriesFactor*sleep, fn) + } + return err + } + return nil +} diff --git a/utils/utils.go b/utils/utils.go index 8b8b2a6..f67b5c6 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -2,12 +2,15 @@ package utils import ( "archive/zip" + "context" "fmt" "io" "math/rand" "os" + "os/exec" "path/filepath" "regexp" + "runtime" "strings" "time" @@ -60,6 +63,24 @@ func Unzip(archive, target string) error { return nil } +func Untar(ctx context.Context, source, target string) error { + + if err := os.MkdirAll(target, 0600); err != nil { + return err + } + + tarExecutable := "tar" + if runtime.GOOS == "windows" { + tarExecutable = "C:\\opscode\\chef\\bin\\tar.exe" + } + cmd := exec.CommandContext(ctx, tarExecutable, "-xzf", source, "-C", target) + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + func ScrapeErrorMessage(message string, regExpression string) string { re, err := regexp.Compile(regExpression) @@ -218,3 +239,28 @@ func Subset(s1, s2 []string) bool { } return true } + +func RemoveFileInfo(fileInfo os.FileInfo, fileInfoName string) error { + if fileInfo.IsDir() { + d, err := os.Open(fileInfoName) + if err != nil { + return err + } + defer d.Close() + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + for _, name := range names { + err = os.RemoveAll(filepath.Join(fileInfoName, name)) + if err != nil { + return err + } + } + } + + if err := os.Remove(fileInfoName); err != nil { + return err + } + return nil +} diff --git a/utils/webservice.go b/utils/webservice.go index c356184..6e2952d 100644 --- a/utils/webservice.go +++ b/utils/webservice.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "net/http" "os" - "regexp" "strings" log "github.com/Sirupsen/logrus" @@ -20,7 +19,7 @@ type ConcertoService interface { Put(path string, payload *map[string]interface{}) ([]byte, int, error) Delete(path string) ([]byte, int, error) Get(path string) ([]byte, int, error) - GetFile(path string, directoryPath string) (string, int, error) + GetFile(path string, filePath string) (string, int, error) } // HTTPConcertoservice web service manager. @@ -198,7 +197,7 @@ func (hcs *HTTPConcertoservice) Get(path string) ([]byte, int, error) { } // GetFile sends GET request to Concerto API and receives a file -func (hcs *HTTPConcertoservice) GetFile(path string, directoryPath string) (string, int, error) { +func (hcs *HTTPConcertoservice) GetFile(path string, filePath string) (string, int, error) { url, _, err := hcs.prepareCall(path, nil) if err != nil { @@ -214,14 +213,7 @@ func (hcs *HTTPConcertoservice) GetFile(path string, directoryPath string) (stri defer response.Body.Close() log.Debugf("Status code:%d message:%s", response.StatusCode, response.Status) - r, err := regexp.Compile("filename=\\\"([^\\\"]*){1}\\\"") - if err != nil { - return "", response.StatusCode, err - } - - // TODO check errors - fileName := r.FindStringSubmatch(response.Header.Get("Content-Disposition"))[1] - realFileName := fmt.Sprintf("%s/%s", directoryPath, fileName) + realFileName := filePath output, err := os.Create(realFileName) if err != nil { diff --git a/utils/webservice_mock.go b/utils/webservice_mock.go index a70f810..75dd0bc 100644 --- a/utils/webservice_mock.go +++ b/utils/webservice_mock.go @@ -34,7 +34,7 @@ func (m *MockConcertoService) Get(path string) ([]byte, int, error) { } // GetFile sends GET request to Concerto API and receives a file -func (m *MockConcertoService) GetFile(path string, directoryPath string) (string, int, error) { - args := m.Called(path, directoryPath) +func (m *MockConcertoService) GetFile(path string, filePath string) (string, int, error) { + args := m.Called(path, filePath) return args.String(0), args.Int(1), args.Error(2) }