fix: refactor out a sub-interface from Vault for VaultURL injection (j…

…enkins-x#4330)

* fix: refactor out a sub-interface from Vault for VaultURL injection

* so we can support local file system vault-like behaviour or real Vault from a small simple interface (which is a small subset of Vault client)
* same URL structure works for vault + local file system referencing

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: fix hound warning

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: refactor vaulturl -> secreturl

Signed-off-by: James Strachan <james.strachan@gmail.com>

* fix: lets move URL handling into the secreturl.Client

and add support for `vault:` for the vault client and `local:` for the local file system client

Signed-off-by: James Strachan <james.strachan@gmail.com>

jenkins-x#4328

* chore: fix hound warning

Signed-off-by: James Strachan <james.strachan@gmail.com>

* fix: allow `values.yaml` to include go template functions

* also support referencing logical Parameters in a `parameters.yaml` file which can include a logical structure + schema (for nice install tooling) which then contains inline values for simple values or URLs to vault/local secret files for better secret management

fixes jenkins-x#4328

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: fix hound warning

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: fix failing tests due to refactor

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: fix failing tests due to refactor

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: avoid failing when bootstrapping a cluster

and we don't yet have the install config ConfigMap setup

Signed-off-by: James Strachan <james.strachan@gmail.com>

* fix: lets populate the cluster information in the cluster/values.yaml

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: fix broken test

Signed-off-by: James Strachan <james.strachan@gmail.com>

* fix: lets allow templating in the root `values.yaml` too

added a test + fix for templating in the root dir as well as any nested `values.yaml` files

Signed-off-by: James Strachan <james.strachan@gmail.com>

* chore: polished the code

thanks for the great feedback @ccojocar

* renamed `vaultClient` -> `secretURLClient`
* fixed up mock generation
* zapped the `GetClusterName` and reused the existing helper

Signed-off-by: James Strachan <james.strachan@gmail.com>

go.sum

@@ -43,6 +43,7 @@ github.com/Masterminds/semver v1.3.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/sprig v0.0.0-20180403013413-6b2a58267f6a/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
+github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w=
 github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/Microsoft/go-winio v0.4.6 h1:Tu8dlnF1wvUKKqr011GFneCoyIn7D+Q2uq6AKmQnGrA=
 github.com/Microsoft/go-winio v0.4.6/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
@@ -85,6 +86,7 @@ github.com/antham/chyle v1.4.0 h1:RBCXpnmj3Wcl43bKbcuRU3LXkarhYwSigWAPUyWEjvY=
 github.com/antham/chyle v1.4.0/go.mod h1:D94Z4aE/ECudyNoTHwkhqu77mjGPZtfPG8dNoeIG9CU=
 github.com/antham/envh v1.2.0/go.mod h1:ocIRPHuwwjyBVBtuUJOJc2TYzGg+d23xSAZexl4y9hQ=
 github.com/antham/strumt v0.0.0-20171215230529-6776189777d3/go.mod h1:sE7EYIUE0nQzPiv5zQAmw2aVkei0j2xmb4gTIIqSFSI=
+github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
 github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
 github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/appscode/jsonpatch v0.0.0-20180911074601-5af499cf01c8/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M=
@@ -391,6 +393,7 @@ github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c h1:kp3AxgXgDOmIJFR7b
 github.com/hinshun/vt10x v0.0.0-20180809195222-d55458df857c/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13 h1:0XE8qtre7NNhsKWo+PuYJNmoR3szkSzpDtZLEW+5HE0=
 github.com/iancoleman/orderedmap v0.0.0-20181121102841-22c6ecc9fe13/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=

pkg/apps/values.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/jenkins-x/jx/pkg/secreturl"
 	"gopkg.in/AlecAivazis/survey.v1/terminal"
 
 	"github.com/ghodss/yaml"
@@ -187,7 +188,7 @@ func GenerateQuestions(schema []byte, batchMode bool, askExisting bool, basePath
 			secrets = append(secrets, secret)
 			if passthrough {
 				if useVault {
-					return vault.ToURI(secret.Path, secret.Key), nil
+					return secreturl.ToURI(secret.Path, secret.Key), nil
 				}
 				return value, nil
 			}

pkg/cmd/opts/helm.go

@@ -10,6 +10,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jenkins-x/jx/pkg/kube/cluster"
+	"github.com/jenkins-x/jx/pkg/secreturl"
+	"github.com/jenkins-x/jx/pkg/secreturl/localvault"
 	"github.com/pborman/uuid"
 
 	"github.com/jenkins-x/jx/pkg/environments"
@@ -24,8 +27,8 @@ import (
 	"github.com/jenkins-x/jx/pkg/util"
 	version2 "github.com/jenkins-x/jx/pkg/version"
 	"github.com/pkg/errors"
-	survey "gopkg.in/AlecAivazis/survey.v1"
-	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/AlecAivazis/survey.v1"
+	"gopkg.in/src-d/go-git.v4"
 	gitconfig "gopkg.in/src-d/go-git.v4/config"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -403,11 +406,32 @@ func (o *CommonOptions) InstallChartWithOptionsAndTimeout(options helm.InstallCh
 			return err
 		}
 	}
+	secretURLClient, err := o.GetSecretURLClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to create a Secret RL client")
+	}
+	return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, secretURLClient)
+}
+
+// GetSecretURLClient create a new secret URL client
+func (o *CommonOptions) GetSecretURLClient() (secreturl.Client, error) {
 	vaultClient, err := o.SystemVaultClient(o.devNamespace)
 	if err != nil {
 		vaultClient = nil
 	}
-	return helm.InstallFromChartOptions(options, o.Helm(), client, timeout, vaultClient)
+	if vaultClient != nil {
+		return vaultClient, nil
+	}
+	clusterName, err := cluster.Name(o.Kube())
+	if err != nil || clusterName == "" {
+		// we could be bootstrapping the cluster
+		clusterName = os.Getenv("JX_CLUSTER_NAME")
+		if clusterName == "" {
+			clusterName = "default-cluster"
+		}
+	}
+	dir, err := util.LocalFileSystemSecretsDir(clusterName)
+	return localvault.NewFileSystemClient(dir), nil
 }
 
 // CloneJXVersionsRepo clones the jenkins-x versions repo to a local working dir

pkg/cmd/step/create/step_create_install_values.go

@@ -23,11 +23,11 @@ import (
 
 var (
 	createInstallValuesLong = templates.LongDesc(`
-		Creates the installation values.yaml file from an init-values.yaml defaulting any missing values from the cluster itself
+		Creates any mising cluster values into the cluster/values.yaml file 
 `)
 
 	createInstallValuesExample = templates.Examples(`
-		# create the values.yaml file in the current directory
+		# populate the cluster/values.yaml file
 		jx step create install vales
 	
 			`)
@@ -62,7 +62,7 @@ func NewCmdStepCreateInstallValues(commonOpts *opts.CommonOptions) *cobra.Comman
 
 	cmd := &cobra.Command{
 		Use:     "install values",
-		Short:   "Creates the installation values.yaml file from an init-values.yaml defaulting any missing values from the cluster itself",
+		Short:   "Creates any mising cluster values into the cluster/values.yaml file ",
 		Long:    createInstallValuesLong,
 		Example: createInstallValuesExample,
 		Aliases: []string{"version pullrequest"},
@@ -93,30 +93,28 @@ func (o *StepCreateInstallValuesOptions) Run() error {
 			return err
 		}
 	}
-	valuesFile := filepath.Join(o.Dir, "values.yaml")
-	initValuesFile := filepath.Join(o.Dir, "init-values.yaml")
+	clusterDir := filepath.Join(o.Dir, "cluster")
+	err = os.MkdirAll(clusterDir, util.DefaultWritePermissions)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create cluster dir: %s", clusterDir)
+	}
 
-	exists, err := util.FileExists(initValuesFile)
+	valuesFile := filepath.Join(clusterDir, helm.ValuesFileName)
+	values, err := helm.LoadValuesFile(valuesFile)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to load helm values: %s", valuesFile)
 	}
-	if exists {
-		values, err := helm.LoadValuesFile(initValuesFile)
-		if err != nil {
-			return errors.Wrapf(err, "failed to load helm values: %s", initValuesFile)
-		}
 
-		values, err = o.defaultMissingValues(values)
-		if err != nil {
-			return errors.Wrapf(err, "failed to default helm values into: %s", initValuesFile)
-		}
+	values, err = o.defaultMissingValues(values)
+	if err != nil {
+		return errors.Wrapf(err, "failed to default helm values into: %s", valuesFile)
+	}
 
-		err = helm.SaveFile(valuesFile, values)
-		if err != nil {
-			return errors.Wrapf(err, "failed to save helm values: %s", valuesFile)
-		}
-		log.Logger().Infof("wrote %s\n", util.ColorInfo(valuesFile))
+	err = helm.SaveFile(valuesFile, values)
+	if err != nil {
+		return errors.Wrapf(err, "failed to save helm values: %s", valuesFile)
 	}
+	log.Logger().Infof("wrote %s\n", util.ColorInfo(valuesFile))
 	return nil
 }
 
@@ -126,23 +124,23 @@ func (o *StepCreateInstallValuesOptions) defaultMissingValues(values map[string]
 		ns = os.Getenv("DEPLOY_NAMESPACE")
 	}
 	if ns != "" {
-		current := util.GetMapValueAsStringViaPath(values, "cluster.namespaceSubDomain")
+		current := util.GetMapValueAsStringViaPath(values, "namespaceSubDomain")
 		if current == "" {
 			subDomain := "." + ns + "."
-			util.SetMapValueViaPath(values, "cluster.namespaceSubDomain", subDomain)
+			util.SetMapValueViaPath(values, "namespaceSubDomain", subDomain)
 		}
 	}
 
-	domain := util.GetMapValueAsStringViaPath(values, "cluster.domain")
+	domain := util.GetMapValueAsStringViaPath(values, "domain")
 	if domain == "" {
 		domain, err := o.discoverIngressDomain(values)
 		if err != nil {
 			return values, errors.Wrapf(err, "failed to discover the Ingress domain")
 		}
 		if domain == "" {
-			return values, fmt.Errorf("could not detect a domain. Pleae configure one at 'cluster.domain' in the init-values.yaml")
+			return values, fmt.Errorf("could not detect a domain. Pleae configure one at 'domain' in the init-values.yaml")
 		}
-		util.SetMapValueViaPath(values, "cluster.domain", domain)
+		util.SetMapValueViaPath(values, "domain", domain)
 	}
 	return values, nil
 }

pkg/cmd/step/create/step_create_install_values_test.go

@@ -81,7 +81,7 @@ func TestCreateInstallValues(t *testing.T) {
 	err = o.Run()
 	require.NoError(t, err, "failed to run step")
 
-	fileName := filepath.Join(outputDir, "values.yaml")
+	fileName := filepath.Join(outputDir, "cluster", "values.yaml")
 
 	t.Logf("Generated values file at %s\n", fileName)
 
@@ -90,8 +90,8 @@ func TestCreateInstallValues(t *testing.T) {
 	values, err := helm.LoadValuesFile(fileName)
 	require.NoError(t, err, "failed to load file %s", fileName)
 
-	AssertMapPathValueAsString(t, values, "cluster.namespaceSubDomain", ".jx.")
-	AssertMapPathValueAsString(t, values, "cluster.domain", expectedDomain)
+	AssertMapPathValueAsString(t, values, "namespaceSubDomain", ".jx.")
+	AssertMapPathValueAsString(t, values, "domain", expectedDomain)
 }
 
 func AssertMapPathValueAsString(t *testing.T, values map[string]interface{}, path string, expected string) {

pkg/cmd/step/helm/step_helm_apply.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/jenkins-x/jx/pkg/cmd/helper"
-
 	"github.com/jenkins-x/jx/pkg/cmd/opts"
 	"github.com/jenkins-x/jx/pkg/cmd/templates"
 	"github.com/jenkins-x/jx/pkg/helm"
@@ -194,7 +193,11 @@ func (o *StepHelmApplyOptions) Run() error {
 		}()
 	}
 
-	chartValues, err := helm.GenerateValues(dir, nil, true)
+	secretURLClient, err := o.GetSecretURLClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to create a Secret RL client")
+	}
+	chartValues, err := helm.GenerateValues(dir, nil, true, secretURLClient)
 	if err != nil {
 		return errors.Wrapf(err, "generating values.yaml for tree from %s", dir)
 	}