Reading variables from file

acceptance/bundle/variables/flag_overrides/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+  name: TestResolveVariablesFromFile
+
+variables:
+  cluster:
+    type: "complex"
+    default:
+      node_type_id: "unused"
+  cluster_key:
+    default: "unused"
+  cluster_workers:
+    default: 1
+
+resources:
+  jobs:
+    job1:
+      job_clusters:
+        - job_cluster_key: ${var.cluster_key}
+          new_cluster:
+            node_type_id: "${var.cluster.node_type_id}"
+            num_workers: ${var.cluster_workers}

acceptance/bundle/variables/flag_overrides/output.txt

@@ -0,0 +1,8 @@
+{
+  "job_cluster_key": "mlops_stacks-cluster",
+  "new_cluster": {
+    "node_type_id": "Standard_DS3_v2",
+    "num_workers": 2
+  }
+}
+"mlops_stacks-cluster"

acceptance/bundle/variables/flag_overrides/script

@@ -0,0 +1,5 @@
+# variable file
+$CLI bundle validate -o json --vars-file-path=vars.json | jq .resources.jobs.job1.job_clusters[0]
+
+# variable flag
+$CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key

acceptance/bundle/variables/flag_overrides/vars.json

@@ -0,0 +1,7 @@
+{
+  "cluster": {
+    "node_type_id": "Standard_DS3_v2"
+  },
+  "cluster_key": "mlops_stacks-cluster",
+  "cluster_workers": 2
+}

bundle/config/root.go

@@ -260,6 +260,22 @@ func (r *Root) InitializeVariables(vars []string) error {
 	return nil
 }
 
+// Initializes variables parsed from variable file
+// Variables can have any type of value, including complex types
+func (r *Root) InitializeAnyTypeVariables(vars map[string]any) error {
+	for name, val := range vars {
+		if _, ok := r.Variables[name]; !ok {
+			return fmt.Errorf("variable %s has not been defined", name)
+		}
+
+		err := r.Variables[name].Set(val)
+		if err != nil {
+			return fmt.Errorf("failed to assign %s to %s: %s", val, name, err)
+		}
+	}
+	return nil
+}
+
 func (r *Root) Merge(other *Root) error {
 	// Merge dynamic configuration values.
 	return r.Mutate(func(root dyn.Value) (dyn.Value, error) {

bundle/config/root_test.go

@@ -51,6 +51,56 @@ func TestInitializeVariables(t *testing.T) {
 	assert.Equal(t, "456", (root.Variables["bar"].Value))
 }
 
+func TestInitializeAnyTypeVariables(t *testing.T) {
+	root := &Root{
+		Variables: map[string]*variable.Variable{
+			"string": {
+				Default:     "default",
+				Description: "string variable",
+			},
+			"int": {
+				Default:     0,
+				Description: "int variable",
+			},
+			"complex": {
+				Default:     []map[string]int{{"a": 1}, {"b": 2}},
+				Description: "complex variable",
+				Type:        variable.VariableTypeComplex,
+			},
+			"unused": {
+				Default:     "should remain default",
+				Description: "unused variable",
+			},
+		},
+	}
+
+	err := root.InitializeAnyTypeVariables(map[string]any{
+		"string":  "value",
+		"int":     1,
+		"complex": []map[string]int{{"c": 3}},
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, "value", (root.Variables["string"].Value))
+	assert.Equal(t, 1, (root.Variables["int"].Value))
+	assert.Equal(t, []map[string]int{{"c": 3}}, (root.Variables["complex"].Value))
+}
+
+func TestInitializeAnyTypeVariablesUndeclared(t *testing.T) {
+	root := &Root{
+		Variables: map[string]*variable.Variable{
+			"string": {
+				Default:     "default",
+				Description: "string variable",
+			},
+		},
+	}
+
+	err := root.InitializeAnyTypeVariables(map[string]any{
+		"not_declared": "value",
+	})
+	assert.ErrorContains(t, err, "variable not_declared has not been defined")
+}
+
 func TestInitializeVariablesWithAnEqualSignInValue(t *testing.T) {
 	root := &Root{
 		Variables: map[string]*variable.Variable{

bundle/config/variable/variable.go

@@ -36,9 +36,11 @@ type Variable struct {
 	// This field stores the resolved value for the variable. The variable are
 	// resolved in the following priority order (from highest to lowest)
 	//
-	// 1. Command line flag. For example: `--var="foo=bar"`
-	// 2. Target variable. eg: BUNDLE_VAR_foo=bar
-	// 3. Default value as defined in the applicable environments block
+	// 1. Command line flag, one of these is used
+	//   a. Variable value obtained from arguments, example: `--var="foo=bar"`
+	//   b. Variable value obtained from the file, example: `--vars-file-path="/path/to/file"`
+	// 2. Environment variable. eg: BUNDLE_VAR_foo=bar
+	// 3. Default value as defined in the applicable targets block
 	// 4. Default value defined in variable definition
 	// 5. Throw error, since if no default value is defined, then the variable
 	//    is required

cmd/bundle/utils/utils.go

@@ -2,6 +2,9 @@ package utils
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"os"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/cmd/root"
@@ -16,6 +19,30 @@ func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string
 	})
 }
 
+func configureVariablesFromFile(cmd *cobra.Command, b *bundle.Bundle, filePath string) diag.Diagnostics {
+	return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+		f, err := os.ReadFile(filePath)
+		if err != nil {
+			return diag.FromErr(fmt.Errorf("failed to read variables file: %w", err))
+		}
+
+		vars := map[string]any{}
+		err = json.Unmarshal(f, &vars)
+		if err != nil {
+			return diag.FromErr(fmt.Errorf("failed to parse variables file: %w", err))
+		}
+
+		if len(vars) > 0 {
+			err = b.Config.InitializeAnyTypeVariables(vars)
+			if err != nil {
+				return diag.FromErr(err)
+			}
+		}
+
+		return nil
+	})
+}
+
 func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diagnostics) {
 	// Load bundle config and apply target
 	b, diags := root.MustConfigureBundle(cmd)
@@ -27,9 +54,20 @@ func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diag
 	if err != nil {
 		return b, diag.FromErr(err)
 	}
+	variableFilePath, err := cmd.Flags().GetString("vars-file-path")
+	if err != nil {
+		return b, diag.FromErr(err)
+	}
 
-	// Initialize variables by assigning them values passed as command line flags
-	diags = diags.Extend(configureVariables(cmd, b, variables))
+	if len(variables) > 0 && variableFilePath != "" {
+		return b, diag.Errorf("cannot specify both --var and --vars-file-path flags")
+	} else if len(variables) > 0 {
+		// Initialize variables by assigning them values passed as command line flags
+		diags = diags.Extend(configureVariables(cmd, b, variables))
+	} else if variableFilePath != "" {
+		// Initialize variables by loading them from a file
+		diags = diags.Extend(configureVariablesFromFile(cmd, b, variableFilePath))
+	}
 
 	return b, diags
 }

cmd/bundle/variables.go

@@ -6,4 +6,5 @@ import (
 
 func initVariableFlag(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`)
+	cmd.PersistentFlags().String("vars-file-path", "", `file path to a JSON file containing variables. Example: --vars-file-path="/path/to/vars.json"`)
 }