Merge branch 'master' of github.com:replicatedcom/ship into kustomize…

…-usability

fixtures/app.yml

@@ -45,8 +45,11 @@ lifecycle:
       contents: |
         This tool will prepare assets so you can deploy Cooltool-Enterpise
         to your existing Kubernetes cluster
-  - render: {}
+  - render:
+      id: render
   - message:
+     requires:
+        - render
      contents: |
         SuperBigTool is ready to deploy to your kubernetes cluster.
 

pkg/lifecycle/daemon/daemontypes/progress.go

@@ -52,12 +52,12 @@ func MessageProgress(source string, msg Message) Progress {
 
 // the empty value is initialized and ready to use
 type ProgressMap struct {
-	syncmap sync.Map
+	Map sync.Map
 }
 
 func (p *ProgressMap) Load(stepID string) (Progress, bool) {
 	empty := Progress{}
-	value, ok := p.syncmap.Load(stepID)
+	value, ok := p.Map.Load(stepID)
 	if !ok {
 		return empty, false
 	}
@@ -71,5 +71,5 @@ func (p *ProgressMap) Load(stepID string) (Progress, bool) {
 }
 
 func (p *ProgressMap) Store(stepID string, progress Progress) {
-	p.syncmap.Store(stepID, progress)
+	p.Map.Store(stepID, progress)
 }

pkg/lifecycle/daemon/routes_v2_completestep.go

@@ -14,13 +14,14 @@ import (
 	"github.com/replicatedhq/ship/pkg/api"
 	"github.com/replicatedhq/ship/pkg/lifecycle/daemon/daemontypes"
 	"github.com/replicatedhq/ship/pkg/lifecycle/daemon/statusonly"
+	"github.com/replicatedhq/ship/pkg/state"
 )
 
 func (d *V2Routes) completeStep(c *gin.Context) {
 	requestedStep := c.Param("step")
 	logger := log.With(d.Logger, "handler", "completeStep", "step", requestedStep)
 	debug := level.Debug(logger)
-	debug.Log()
+	debug.Log("event", "call")
 
 	for _, step := range d.Release.Spec.Lifecycle.V1 {
 		stepShared := step.Shared()
@@ -54,6 +55,8 @@ func (d *V2Routes) completeStep(c *gin.Context) {
 		case err = <-errChan:
 			if err != nil {
 				debug.Log("event", "step.fail", "err", err)
+				// todo need some kind of errprogress
+				d.StepProgress.Store(stepID, daemontypes.StringProgress("v2router", fmt.Sprintf("failed - %v", err)))
 				c.AbortWithError(500, err)
 				return
 			}
@@ -71,25 +74,11 @@ func (d *V2Routes) completeStep(c *gin.Context) {
 				"phase":  step.ShortName(),
 				"poll":   fmt.Sprintf("/lifecycle/step/%s", stepID),
 			})
-			go func() {
-				err := d.awaitAsyncStep(errChan, debug, step)
-				if err != nil {
-					debug.Log("event", "execute.fail", "err", err)
-					d.StepProgress.Store(stepID, daemontypes.StringProgress("v2router", "failed"))
-					return
-				}
-				newState := state.Versioned().WithCompletedStep(step)
-
-				err = d.StateManager.Save(newState)
-				if err != nil {
-					debug.Log("event", "state.save.fail", "err", err)
-					return
-				}
-			}()
+			go d.handleAsync(errChan, debug, step, stepID, state)
 			return
 		}
 		level.Info(logger).Log("event", "task.complete", "progess", d.progress(step))
-		d.StepProgress.Store(stepID, daemontypes.StringProgress("v2router", "failed"))
+		d.StepProgress.Store(stepID, daemontypes.StringProgress("v2router", "success"))
 		newState := state.Versioned().WithCompletedStep(step)
 
 		err = d.StateManager.Save(newState)
@@ -109,6 +98,22 @@ func (d *V2Routes) completeStep(c *gin.Context) {
 	d.errNotFond(c)
 }
 
+func (d *V2Routes) handleAsync(errChan chan error, debug log.Logger, step api.Step, stepID string, state state.State) {
+	err := d.awaitAsyncStep(errChan, debug, step)
+	if err != nil {
+		debug.Log("event", "execute.fail", "err", err)
+		d.StepProgress.Store(stepID, daemontypes.StringProgress("v2router", fmt.Sprintf("failed - %v", err)))
+		return
+	}
+	d.StepProgress.Store(stepID, daemontypes.StringProgress("v2router", "success"))
+	newState := state.Versioned().WithCompletedStep(step)
+	err = d.StateManager.Save(newState)
+	if err != nil {
+		debug.Log("event", "state.save.fail", "err", err)
+		return
+	}
+}
+
 func (d *V2Routes) awaitAsyncStep(errChan chan error, debug log.Logger, step api.Step) error {
 	debug.Log("event", "async.await")
 	for {
@@ -140,9 +145,6 @@ func (d *V2Routes) execute(step api.Step) error {
 			d.StepProgress.Store(step.Shared().ID, progress)
 		},
 	}
-	planner := d.Planner.WithStatusReceiver(statusReceiver)
-	renderer := d.Renderer.WithPlanner(planner)
-	renderer = renderer.WithStatusReceiver(statusReceiver)
 
 	if step.Message != nil {
 		debug.Log("event", "step.resolve", "type", "message")
@@ -156,9 +158,12 @@ func (d *V2Routes) execute(step api.Step) error {
 		return errors.Wrap(err, "execute helmIntro step")
 	} else if step.Render != nil {
 		debug.Log("event", "step.resolve", "type", "helmIntro")
-		err := d.Renderer.Execute(context.Background(), d.Release, step.Render)
-		debug.Log("event", "step.complete", "type", "helmIntro", "err", err)
-		return errors.Wrap(err, "execute helmIntro step")
+		planner := d.Planner.WithStatusReceiver(statusReceiver)
+		renderer := d.Renderer.WithPlanner(planner)
+		renderer = renderer.WithStatusReceiver(statusReceiver)
+		err := renderer.Execute(context.Background(), d.Release, step.Render)
+		debug.Log("event", "step.complete", "type", "render", "err", err)
+		return errors.Wrap(err, "execute render step")
 	}
 
 	return errors.Errorf("unknown step %s:%s", step.ShortName(), step.Shared().ID)

pkg/lifecycle/daemon/routes_v2_lifecycle.go

@@ -2,25 +2,30 @@ package daemon
 
 import (
 	"github.com/gin-gonic/gin"
-	"github.com/go-kit/kit/log/level"
+	"github.com/replicatedhq/ship/pkg/lifecycle/daemon/daemontypes"
 )
 
 type lifeycleStep struct {
-	ID          string `json:"id"`
-	Description string `json:"description"`
-	Phase       string `json:"phase"`
+	ID          string                `json:"id"`
+	Description string                `json:"description"`
+	Phase       string                `json:"phase"`
+	Progress    *daemontypes.Progress `json:"progress,omitempty"`
 }
 
 func (d *V2Routes) getLifecycle(c *gin.Context) {
 	lifecycleIDs := make([]lifeycleStep, 0)
 	for _, step := range d.Release.Spec.Lifecycle.V1 {
-		level.Debug(d.Logger).Log("step", step)
-
-		lifecycleIDs = append(lifecycleIDs, lifeycleStep{
+		stepResponse := lifeycleStep{
 			ID:          step.Shared().ID,
 			Description: step.Shared().Description,
 			Phase:       step.ShortName(),
-		})
+		}
+
+		if progress, ok := d.StepProgress.Load(step.Shared().ID); ok {
+			stepResponse.Progress = &progress
+		}
+
+		lifecycleIDs = append(lifecycleIDs, stepResponse)
 	}
 	c.JSON(200, lifecycleIDs)
 }

pkg/lifecycle/daemon/routes_v2_lifecycle_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/go-test/deep"
 	"github.com/replicatedhq/ship/pkg/api"
+	"github.com/replicatedhq/ship/pkg/lifecycle/daemon/daemontypes"
 	"github.com/replicatedhq/ship/pkg/testing/logger"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
@@ -36,7 +37,8 @@ func TestV2Lifecycle(t *testing.T) {
 			}
 			testLogger := &logger.TestLogger{T: t}
 			v2 := &V2Routes{
-				Logger: testLogger,
+				Logger:       testLogger,
+				StepProgress: &daemontypes.ProgressMap{},
 			}
 
 			func() {

pkg/lifecycle/render/helm/template.go

@@ -138,6 +138,10 @@ func (f *ForkTemplater) Template(
 	tempRenderedChartTemplatesDir := path.Join(tempRenderedChartDir, "templates")
 	tempRenderedSubChartsDir := path.Join(tempRenderedChartDir, subChartsDirName)
 
+	if err := f.tryRemoveRenderedHelmPath(); err != nil {
+		return errors.Wrap(err, "removeAll failed while trying to remove rendered Helm values base dir")
+	}
+
 	debug.Log("event", "rename")
 	if templatesDirExists, err := f.FS.IsDir(tempRenderedChartTemplatesDir); err == nil && templatesDirExists {
 		if err := f.FS.Rename(tempRenderedChartTemplatesDir, asset.Dest); err != nil {
@@ -157,13 +161,24 @@ func (f *ForkTemplater) Template(
 
 	debug.Log("event", "temphelmvalues.remove", "path", constants.TempHelmValuesPath)
 	if err := f.FS.RemoveAll(constants.TempHelmValuesPath); err != nil {
-		return errors.Wrap(err, "failed to remove Helm values tmp dir")
+		return errors.Wrap(err, "removeAll failed while trying to remove Helm values tmp dir")
 	}
 
 	// todo link up stdout/stderr debug logs
 	return nil
 }
 
+func (f *ForkTemplater) tryRemoveRenderedHelmPath() error {
+	debug := level.Debug(log.With(f.Logger, "method", "tryRemoveRenderedHelmPath"))
+
+	if err := f.FS.RemoveAll(constants.RenderedHelmPath); err != nil {
+		return err
+	}
+	debug.Log("event", "renderedHelmPath.remove", "path", constants.RenderedHelmPath)
+
+	return nil
+}
+
 func (f *ForkTemplater) appendHelmValues(
 	configGroups []libyaml.ConfigGroup,
 	templateContext map[string]interface{},

pkg/lifecycle/render/helm/template_test.go

@@ -11,6 +11,8 @@ import (
 	"reflect"
 	"strings"
 
+	"path"
+
 	"github.com/replicatedhq/libyaml"
 	"github.com/replicatedhq/ship/pkg/api"
 	"github.com/replicatedhq/ship/pkg/constants"
@@ -232,3 +234,64 @@ func TestMockHelm(t *testing.T) {
 	}
 
 }
+
+func TestTryRemoveRenderedHelmPath(t *testing.T) {
+	tests := []struct {
+		name        string
+		describe    string
+		baseDir     string
+		expectError bool
+	}{
+		{
+			name:        "base exists",
+			describe:    "ensure base is removed and removeAll doesn't error",
+			baseDir:     constants.RenderedHelmPath,
+			expectError: false,
+		},
+		{
+			name:        "base does not exist",
+			describe:    "missing base, ensure removeAll doesn't error",
+			baseDir:     "",
+			expectError: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req := require.New(t)
+
+			testLogger := &logger.TestLogger{T: t}
+
+			fakeFS := afero.Afero{Fs: afero.NewMemMapFs()}
+
+			// create the base directory
+			err := fakeFS.MkdirAll(path.Join(test.baseDir, "myCoolManifest.yaml"), 0777)
+			req.NoError(err)
+
+			// verify path actually exists
+			successfulMkdirAll, err := fakeFS.DirExists(path.Join(test.baseDir, "myCoolManifest.yaml"))
+			req.True(successfulMkdirAll)
+			req.NoError(err)
+
+			ft := &ForkTemplater{
+				FS:     fakeFS,
+				Logger: testLogger,
+			}
+
+			removeErr := ft.tryRemoveRenderedHelmPath()
+
+			if test.expectError {
+				req.Error(removeErr)
+			} else {
+				if dirExists, existErr := ft.FS.DirExists(constants.RenderedHelmPath); dirExists {
+					req.NoError(existErr)
+					// if dir exists, we expect tryRemoveRenderedHelmPath to have err'd
+					req.Error(removeErr)
+				} else {
+					// if dir does not exist, we expect tryRemoveRenderedHelmPath to have succeeded without err'ing
+					req.NoError(removeErr)
+				}
+			}
+		})
+	}
+}

pkg/lifecycle/step.go

@@ -60,6 +60,6 @@ func (s *StepExecutor) Execute(ctx context.Context, release *api.Release, step *
 		debug.Log("event", "step.complete", "type", "kubectl", "err", err)
 	}
 
-	debug.Log("event", "step.unknown")
+	debug.Log("event", "step.unknown", "name", step.ShortName(), "id", step.Shared().ID)
 	return nil
 }

pkg/ship/dig.go

@@ -60,6 +60,7 @@ func buildInjector() (*dig.Container, error) {
 		filetree.NewLoader,
 		templates.NewBuilderBuilder,
 		patch.NewShipPatcher,
+		specs.NewIDPatcher,
 
 		daemon.NewV1Router,
 		resolve.NewRenderer,