From ed038ea1a6cb0c1f55bd242b52717c575c551c83 Mon Sep 17 00:00:00 2001 From: Fabian Jakobs Date: Mon, 17 Jul 2023 14:22:25 +0200 Subject: [PATCH 1/5] first look for databricks.yaml before falling back to bundle.yml --- bundle/bundle.go | 6 +- .../config/mutator/process_root_includes.go | 6 +- bundle/config/root.go | 32 +++++++++- bundle/config/root_test.go | 58 +++++++++++++++++++ bundle/root.go | 12 ++-- bundle/root_test.go | 2 +- 6 files changed, 105 insertions(+), 11 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index 02d0eaac9a..81fdfd4a85 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -45,7 +45,11 @@ type Bundle struct { func Load(path string) (*Bundle, error) { bundle := &Bundle{} - err := bundle.Config.Load(filepath.Join(path, config.FileName)) + configFile, err := config.FileNames.FindInPath(path) + if err != nil { + return nil, err + } + err = bundle.Config.Load(configFile) if err != nil { return nil, err } diff --git a/bundle/config/mutator/process_root_includes.go b/bundle/config/mutator/process_root_includes.go index 454e3a9870..f3717ce01e 100644 --- a/bundle/config/mutator/process_root_includes.go +++ b/bundle/config/mutator/process_root_includes.go @@ -27,8 +27,10 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) error var out []bundle.Mutator // Map with files we've already seen to avoid loading them twice. - var seen = map[string]bool{ - config.FileName: true, + var seen = map[string]bool{} + + for _, file := range config.FileNames { + seen[file] = true } // Maintain list of files in order of files being loaded. diff --git a/bundle/config/root.go b/bundle/config/root.go index 5ee337d30e..6d53264b9d 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -11,8 +11,31 @@ import ( "github.com/imdario/mergo" ) -// FileName is the name of bundle configuration file. -const FileName = "bundle.yml" +type ConfigFileNames [4]string + +// FileNames is the name of bundle configuration file. +var FileNames = ConfigFileNames{"databricks.yaml", "databricks.yml", "bundle.yaml", "bundle.yml"} + +func (c ConfigFileNames) FindInPath(path string) (string, error) { + result := "" + var err error = nil + + for _, file := range c { + filePath := filepath.Join(path, file) + _, err = os.Stat(filePath) + if err == nil { + if result != "" { + return "", fmt.Errorf("multiple bundle configuration files found in %s", path) + } + result = filePath + } + } + + if result == "" { + return "", err + } + return result, nil +} type Root struct { // Path contains the directory path to the root of the bundle. @@ -62,7 +85,10 @@ func Load(path string) (*Root, error) { // If we were given a directory, assume this is the bundle root. if stat.IsDir() { - path = filepath.Join(path, FileName) + path, err = FileNames.FindInPath(path) + if err != nil { + return nil, err + } } if err := r.Load(path); err != nil { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 818e89a2db..e6d1aeeb29 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -2,7 +2,11 @@ package config import ( "encoding/json" + "os" + "path/filepath" "reflect" + "runtime" + "strings" "testing" "github.com/databricks/cli/bundle/config/variable" @@ -163,3 +167,57 @@ func TestRootMergeEnvironmentWithMode(t *testing.T) { require.NoError(t, root.MergeEnvironment(env)) assert.Equal(t, Development, root.Bundle.Mode) } + +func TestConfigFileNames_FindInPath(t *testing.T) { + testCases := []struct { + name string + files []string + expected string + err string + }{ + { + name: "file found", + files: []string{"databricks.yaml"}, + expected: "BASE/databricks.yaml", + err: "", + }, + { + name: "multiple files found", + files: []string{"databricks.yaml", "bundle.yml"}, + expected: "", + err: "multiple bundle configuration files found", + }, + { + name: "file not found", + files: []string{}, + expected: "", + err: "no such file or directory", + }, + } + + if runtime.GOOS == "windows" { + testCases[2].err = "The system cannot find the file specified." + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + projectDir := t.TempDir() + for _, file := range tc.files { + f1, _ := os.Create(filepath.Join(projectDir, file)) + f1.Close() + } + + cfn := ConfigFileNames{"databricks.yaml", "databricks.yml", "bundle.yaml", "bundle.yml"} + result, err := cfn.FindInPath(projectDir) + + expected := strings.Replace(tc.expected, "BASE/", projectDir+string(os.PathSeparator), 1) + if result != expected { + t.Errorf("expected %s, but got %s", expected, result) + } + + if err != nil && !strings.Contains(err.Error(), tc.err) { + t.Errorf("expected error %v, but got %v", tc.err, err) + } + }) + } +} diff --git a/bundle/root.go b/bundle/root.go index 70d778e152..46f63e1342 100644 --- a/bundle/root.go +++ b/bundle/root.go @@ -36,11 +36,15 @@ func getRootWithTraversal() (string, error) { if err != nil { return "", err } - path, err := folders.FindDirWithLeaf(wd, config.FileName) - if err != nil { - return "", fmt.Errorf(`unable to locate bundle root: %s not found`, config.FileName) + + for _, file := range config.FileNames { + path, err := folders.FindDirWithLeaf(wd, file) + if err == nil { + return path, nil + } } - return path, nil + + return "", fmt.Errorf(`unable to locate bundle root: %s not found`, config.FileNames[0]) } // mustGetRoot returns a bundle root or an error if one cannot be found. diff --git a/bundle/root_test.go b/bundle/root_test.go index dab0022567..9a3cfcb08a 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -77,7 +77,7 @@ func TestRootLookup(t *testing.T) { chdir(t, t.TempDir()) // Create bundle.yml file. - f, err := os.Create(config.FileName) + f, err := os.Create(config.FileNames[0]) require.NoError(t, err) defer f.Close() From 2d0ead8350a603a9d08253255a3c7f891e7f9bfa Mon Sep 17 00:00:00 2001 From: Fabian Jakobs Date: Mon, 17 Jul 2023 16:33:59 +0200 Subject: [PATCH 2/5] Update bundle/config/root.go Co-authored-by: Pieter Noordhuis --- bundle/config/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 6d53264b9d..2423c1856a 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -13,7 +13,7 @@ import ( type ConfigFileNames [4]string -// FileNames is the name of bundle configuration file. +// FileNames contains possible names of bundle configuration files. var FileNames = ConfigFileNames{"databricks.yaml", "databricks.yml", "bundle.yaml", "bundle.yml"} func (c ConfigFileNames) FindInPath(path string) (string, error) { From dcb4e6655e5747e3e9addd43adcf78cbfdb03606 Mon Sep 17 00:00:00 2001 From: Fabian Jakobs Date: Mon, 17 Jul 2023 16:36:29 +0200 Subject: [PATCH 3/5] Update bundle/config/root.go Co-authored-by: Pieter Noordhuis --- bundle/config/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 2423c1856a..ea8af1f260 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -25,7 +25,7 @@ func (c ConfigFileNames) FindInPath(path string) (string, error) { _, err = os.Stat(filePath) if err == nil { if result != "" { - return "", fmt.Errorf("multiple bundle configuration files found in %s", path) + return "", fmt.Errorf("multiple bundle root configuration files found in %s", path) } result = filePath } From 2d87c780a7e22c8917b265a27f929df2cfa999e4 Mon Sep 17 00:00:00 2001 From: Fabian Jakobs Date: Mon, 17 Jul 2023 16:56:00 +0200 Subject: [PATCH 4/5] Address PR feedback --- bundle/bundle_test.go | 4 +-- .../mutator/process_root_includes_test.go | 2 +- bundle/config/root.go | 16 +++++---- bundle/config/root_test.go | 33 +++++++++++-------- .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 bundle/root_test.go | 2 +- .../{bundle.yml => databricks.yaml} | 0 .../basic/{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 bundle/tests/conflicting_resource_ids_test.go | 4 +-- .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 bundle/tests/job_and_pipeline_test.go | 8 ++--- .../{bundle.yml => databricks.yaml} | 0 .../vanilla/{bundle.yml => databricks.yaml} | 0 .../{bundle.yml => databricks.yaml} | 0 cmd/sync/sync.go | 2 +- 26 files changed, 40 insertions(+), 31 deletions(-) rename bundle/config/testdata/duplicate_resource_name_in_subconfiguration/{bundle.yml => databricks.yaml} (100%) rename bundle/config/testdata/duplicate_resource_names_in_root/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/autoload_git/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/basic/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/conflicting_resource_ids/no_subconfigurations/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/conflicting_resource_ids/one_subconfiguration/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/conflicting_resource_ids/two_subconfigurations/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/environment_empty/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/environment_overrides/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/include_default/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/include_invalid/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/include_override/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/include_with_glob/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/interpolation/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/job_and_pipeline/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/variables/env_overrides/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/variables/vanilla/{bundle.yml => databricks.yaml} (100%) rename bundle/tests/yaml_anchors/{bundle.yml => databricks.yaml} (100%) diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 5a26d35082..9de84f3c5c 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -23,7 +23,7 @@ func TestLoadExists(t *testing.T) { func TestBundleCacheDir(t *testing.T) { projectDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "bundle.yml")) + f1, err := os.Create(filepath.Join(projectDir, "databricks.yaml")) require.NoError(t, err) f1.Close() @@ -47,7 +47,7 @@ func TestBundleCacheDir(t *testing.T) { func TestBundleCacheDirOverride(t *testing.T) { projectDir := t.TempDir() bundleTmpDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "bundle.yml")) + f1, err := os.Create(filepath.Join(projectDir, "databricks.yaml")) require.NoError(t, err) f1.Close() diff --git a/bundle/config/mutator/process_root_includes_test.go b/bundle/config/mutator/process_root_includes_test.go index c7d00d88bd..0869bfe03c 100644 --- a/bundle/config/mutator/process_root_includes_test.go +++ b/bundle/config/mutator/process_root_includes_test.go @@ -61,7 +61,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { }, } - touch(t, bundle.Config.Path, "bundle.yml") + touch(t, bundle.Config.Path, "databricks.yaml") touch(t, bundle.Config.Path, "a.yml") touch(t, bundle.Config.Path, "b.yml") diff --git a/bundle/config/root.go b/bundle/config/root.go index ea8af1f260..f11821bcba 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -13,33 +13,37 @@ import ( type ConfigFileNames [4]string -// FileNames contains possible names of bundle configuration files. +// FileNames contains allowed names of bundle configuration files. var FileNames = ConfigFileNames{"databricks.yaml", "databricks.yml", "bundle.yaml", "bundle.yml"} func (c ConfigFileNames) FindInPath(path string) (string, error) { result := "" - var err error = nil + var firstErr error for _, file := range c { filePath := filepath.Join(path, file) - _, err = os.Stat(filePath) + _, err := os.Stat(filePath) if err == nil { if result != "" { return "", fmt.Errorf("multiple bundle root configuration files found in %s", path) } result = filePath + } else { + if firstErr == nil { + firstErr = err + } } } if result == "" { - return "", err + return "", firstErr } return result, nil } type Root struct { // Path contains the directory path to the root of the bundle. - // It is set when loading `bundle.yml`. + // It is set when loading `databricks.yaml`. Path string `json:"-" bundle:"readonly"` // Contains user defined variables @@ -50,7 +54,7 @@ type Root struct { Bundle Bundle `json:"bundle"` // Include specifies a list of patterns of file names to load and - // merge into the this configuration. If not set in `bundle.yml`, + // merge into the this configuration. If not set in `databricks.yaml`, // it defaults to loading `*.yml` and `*/*.yml`. // // Also see [mutator.DefineDefaultInclude]. diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index e6d1aeeb29..225ab49db9 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -30,7 +30,7 @@ func TestRootMarshalUnmarshal(t *testing.T) { func TestRootLoad(t *testing.T) { root := &Root{} - err := root.Load("../tests/basic/bundle.yml") + err := root.Load("../tests/basic/databricks.yaml") require.NoError(t, err) assert.Equal(t, "basic", root.Bundle.Name) } @@ -82,13 +82,13 @@ func TestRootMergeMap(t *testing.T) { func TestDuplicateIdOnLoadReturnsError(t *testing.T) { root := &Root{} - err := root.Load("./testdata/duplicate_resource_names_in_root/bundle.yml") - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/bundle.yml, pipeline at ./testdata/duplicate_resource_names_in_root/bundle.yml)") + err := root.Load("./testdata/duplicate_resource_names_in_root/databricks.yaml") + assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yaml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yaml)") } func TestDuplicateIdOnMergeReturnsError(t *testing.T) { root := &Root{} - err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/bundle.yml") + err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml") require.NoError(t, err) other := &Root{} @@ -96,7 +96,7 @@ func TestDuplicateIdOnMergeReturnsError(t *testing.T) { require.NoError(t, err) err = root.Merge(other) - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/bundle.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") + assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") } func TestInitializeVariables(t *testing.T) { @@ -181,11 +181,17 @@ func TestConfigFileNames_FindInPath(t *testing.T) { expected: "BASE/databricks.yaml", err: "", }, + { + name: "file found", + files: []string{"bundle.yml"}, + expected: "BASE/bundle.yml", + err: "", + }, { name: "multiple files found", - files: []string{"databricks.yaml", "bundle.yml"}, + files: []string{"databricks.yaml", "bundle.yaml"}, expected: "", - err: "multiple bundle configuration files found", + err: "multiple bundle root configuration files found", }, { name: "file not found", @@ -207,16 +213,15 @@ func TestConfigFileNames_FindInPath(t *testing.T) { f1.Close() } - cfn := ConfigFileNames{"databricks.yaml", "databricks.yml", "bundle.yaml", "bundle.yml"} - result, err := cfn.FindInPath(projectDir) + result, err := FileNames.FindInPath(projectDir) expected := strings.Replace(tc.expected, "BASE/", projectDir+string(os.PathSeparator), 1) - if result != expected { - t.Errorf("expected %s, but got %s", expected, result) - } + assert.Equal(t, expected, result) - if err != nil && !strings.Contains(err.Error(), tc.err) { - t.Errorf("expected error %v, but got %v", tc.err, err) + if tc.err != "" { + assert.ErrorContains(t, err, tc.err) + } else { + assert.NoError(t, err) } }) } diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/bundle.yml b/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml similarity index 100% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/bundle.yml rename to bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml diff --git a/bundle/config/testdata/duplicate_resource_names_in_root/bundle.yml b/bundle/config/testdata/duplicate_resource_names_in_root/databricks.yaml similarity index 100% rename from bundle/config/testdata/duplicate_resource_names_in_root/bundle.yml rename to bundle/config/testdata/duplicate_resource_names_in_root/databricks.yaml diff --git a/bundle/root_test.go b/bundle/root_test.go index 9a3cfcb08a..b3fa4e76c9 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -76,7 +76,7 @@ func TestRootLookup(t *testing.T) { chdir(t, t.TempDir()) - // Create bundle.yml file. + // Create databricks.yaml file. f, err := os.Create(config.FileNames[0]) require.NoError(t, err) defer f.Close() diff --git a/bundle/tests/autoload_git/bundle.yml b/bundle/tests/autoload_git/databricks.yaml similarity index 100% rename from bundle/tests/autoload_git/bundle.yml rename to bundle/tests/autoload_git/databricks.yaml diff --git a/bundle/tests/basic/bundle.yml b/bundle/tests/basic/databricks.yaml similarity index 100% rename from bundle/tests/basic/bundle.yml rename to bundle/tests/basic/databricks.yaml diff --git a/bundle/tests/conflicting_resource_ids/no_subconfigurations/bundle.yml b/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yaml similarity index 100% rename from bundle/tests/conflicting_resource_ids/no_subconfigurations/bundle.yml rename to bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yaml diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/bundle.yml b/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yaml similarity index 100% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/bundle.yml rename to bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yaml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/bundle.yml b/bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yaml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/bundle.yml rename to bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yaml diff --git a/bundle/tests/conflicting_resource_ids_test.go b/bundle/tests/conflicting_resource_ids_test.go index 12f460fde9..6922149252 100644 --- a/bundle/tests/conflicting_resource_ids_test.go +++ b/bundle/tests/conflicting_resource_ids_test.go @@ -14,7 +14,7 @@ import ( func TestConflictingResourceIdsNoSubconfig(t *testing.T) { _, err := bundle.Load("./conflicting_resource_ids/no_subconfigurations") - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/bundle.yml") + bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yaml") assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, bundleConfigPath)) } @@ -22,7 +22,7 @@ func TestConflictingResourceIdsOneSubconfig(t *testing.T) { b, err := bundle.Load("./conflicting_resource_ids/one_subconfiguration") require.NoError(t, err) err = bundle.Apply(context.Background(), b, bundle.Seq(mutator.DefaultMutators()...)) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/bundle.yml") + bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yaml") resourcesConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/resources.yml") assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, resourcesConfigPath)) } diff --git a/bundle/tests/environment_empty/bundle.yml b/bundle/tests/environment_empty/databricks.yaml similarity index 100% rename from bundle/tests/environment_empty/bundle.yml rename to bundle/tests/environment_empty/databricks.yaml diff --git a/bundle/tests/environment_overrides/bundle.yml b/bundle/tests/environment_overrides/databricks.yaml similarity index 100% rename from bundle/tests/environment_overrides/bundle.yml rename to bundle/tests/environment_overrides/databricks.yaml diff --git a/bundle/tests/include_default/bundle.yml b/bundle/tests/include_default/databricks.yaml similarity index 100% rename from bundle/tests/include_default/bundle.yml rename to bundle/tests/include_default/databricks.yaml diff --git a/bundle/tests/include_invalid/bundle.yml b/bundle/tests/include_invalid/databricks.yaml similarity index 100% rename from bundle/tests/include_invalid/bundle.yml rename to bundle/tests/include_invalid/databricks.yaml diff --git a/bundle/tests/include_override/bundle.yml b/bundle/tests/include_override/databricks.yaml similarity index 100% rename from bundle/tests/include_override/bundle.yml rename to bundle/tests/include_override/databricks.yaml diff --git a/bundle/tests/include_with_glob/bundle.yml b/bundle/tests/include_with_glob/databricks.yaml similarity index 100% rename from bundle/tests/include_with_glob/bundle.yml rename to bundle/tests/include_with_glob/databricks.yaml diff --git a/bundle/tests/interpolation/bundle.yml b/bundle/tests/interpolation/databricks.yaml similarity index 100% rename from bundle/tests/interpolation/bundle.yml rename to bundle/tests/interpolation/databricks.yaml diff --git a/bundle/tests/job_and_pipeline/bundle.yml b/bundle/tests/job_and_pipeline/databricks.yaml similarity index 100% rename from bundle/tests/job_and_pipeline/bundle.yml rename to bundle/tests/job_and_pipeline/databricks.yaml diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 775f415c2c..868785a16e 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -15,7 +15,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +29,7 @@ func TestJobAndPipelineStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +42,14 @@ func TestJobAndPipelineProduction(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(j.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(j.ConfigFilePath)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/variables/env_overrides/bundle.yml b/bundle/tests/variables/env_overrides/databricks.yaml similarity index 100% rename from bundle/tests/variables/env_overrides/bundle.yml rename to bundle/tests/variables/env_overrides/databricks.yaml diff --git a/bundle/tests/variables/vanilla/bundle.yml b/bundle/tests/variables/vanilla/databricks.yaml similarity index 100% rename from bundle/tests/variables/vanilla/bundle.yml rename to bundle/tests/variables/vanilla/databricks.yaml diff --git a/bundle/tests/yaml_anchors/bundle.yml b/bundle/tests/yaml_anchors/databricks.yaml similarity index 100% rename from bundle/tests/yaml_anchors/bundle.yml rename to bundle/tests/yaml_anchors/databricks.yaml diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index d13a85d033..62f9b19a64 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -72,7 +72,7 @@ var syncCmd = &cobra.Command{ // // To be uncommented and used once our VS Code extension is bundle aware. - // Until then, this could interfere with extension usage where a `bundle.yml` file is present. + // Until then, this could interfere with extension usage where a `databricks.yaml` file is present. // See https://github.com/databricks/cli/pull/207. // // b := bundle.GetOrNil(cmd.Context()) From 3d3c2c97a966a13fddcf070cf7c916a2fdb9bf91 Mon Sep 17 00:00:00 2001 From: Fabian Jakobs Date: Tue, 18 Jul 2023 11:17:23 +0200 Subject: [PATCH 5/5] use databricks.yml by default --- bundle/bundle_test.go | 4 ++-- .../mutator/process_root_includes_test.go | 2 +- bundle/config/root.go | 8 ++++---- bundle/config/root_test.go | 18 +++++++++--------- .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 bundle/root_test.go | 2 +- .../{databricks.yaml => databricks.yml} | 0 .../basic/{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 bundle/tests/conflicting_resource_ids_test.go | 4 ++-- .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 bundle/tests/job_and_pipeline_test.go | 8 ++++---- .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 .../{databricks.yaml => databricks.yml} | 0 cmd/sync/sync.go | 2 +- 26 files changed, 24 insertions(+), 24 deletions(-) rename bundle/config/testdata/duplicate_resource_name_in_subconfiguration/{databricks.yaml => databricks.yml} (100%) rename bundle/config/testdata/duplicate_resource_names_in_root/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/autoload_git/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/basic/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/conflicting_resource_ids/no_subconfigurations/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/conflicting_resource_ids/one_subconfiguration/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/conflicting_resource_ids/two_subconfigurations/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/environment_empty/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/environment_overrides/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/include_default/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/include_invalid/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/include_override/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/include_with_glob/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/interpolation/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/job_and_pipeline/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/variables/env_overrides/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/variables/vanilla/{databricks.yaml => databricks.yml} (100%) rename bundle/tests/yaml_anchors/{databricks.yaml => databricks.yml} (100%) diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 9de84f3c5c..18550f4f2d 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -23,7 +23,7 @@ func TestLoadExists(t *testing.T) { func TestBundleCacheDir(t *testing.T) { projectDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "databricks.yaml")) + f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) require.NoError(t, err) f1.Close() @@ -47,7 +47,7 @@ func TestBundleCacheDir(t *testing.T) { func TestBundleCacheDirOverride(t *testing.T) { projectDir := t.TempDir() bundleTmpDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "databricks.yaml")) + f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) require.NoError(t, err) f1.Close() diff --git a/bundle/config/mutator/process_root_includes_test.go b/bundle/config/mutator/process_root_includes_test.go index 0869bfe03c..9ca5335ac5 100644 --- a/bundle/config/mutator/process_root_includes_test.go +++ b/bundle/config/mutator/process_root_includes_test.go @@ -61,7 +61,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { }, } - touch(t, bundle.Config.Path, "databricks.yaml") + touch(t, bundle.Config.Path, "databricks.yml") touch(t, bundle.Config.Path, "a.yml") touch(t, bundle.Config.Path, "b.yml") diff --git a/bundle/config/root.go b/bundle/config/root.go index f11821bcba..28b1a6158e 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -11,10 +11,10 @@ import ( "github.com/imdario/mergo" ) -type ConfigFileNames [4]string +type ConfigFileNames []string // FileNames contains allowed names of bundle configuration files. -var FileNames = ConfigFileNames{"databricks.yaml", "databricks.yml", "bundle.yaml", "bundle.yml"} +var FileNames = ConfigFileNames{"databricks.yml", "databricks.yaml", "bundle.yml", "bundle.yaml"} func (c ConfigFileNames) FindInPath(path string) (string, error) { result := "" @@ -43,7 +43,7 @@ func (c ConfigFileNames) FindInPath(path string) (string, error) { type Root struct { // Path contains the directory path to the root of the bundle. - // It is set when loading `databricks.yaml`. + // It is set when loading `databricks.yml`. Path string `json:"-" bundle:"readonly"` // Contains user defined variables @@ -54,7 +54,7 @@ type Root struct { Bundle Bundle `json:"bundle"` // Include specifies a list of patterns of file names to load and - // merge into the this configuration. If not set in `databricks.yaml`, + // merge into the this configuration. If not set in `databricks.yml`, // it defaults to loading `*.yml` and `*/*.yml`. // // Also see [mutator.DefineDefaultInclude]. diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 225ab49db9..531ffcec1d 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -30,7 +30,7 @@ func TestRootMarshalUnmarshal(t *testing.T) { func TestRootLoad(t *testing.T) { root := &Root{} - err := root.Load("../tests/basic/databricks.yaml") + err := root.Load("../tests/basic/databricks.yml") require.NoError(t, err) assert.Equal(t, "basic", root.Bundle.Name) } @@ -82,13 +82,13 @@ func TestRootMergeMap(t *testing.T) { func TestDuplicateIdOnLoadReturnsError(t *testing.T) { root := &Root{} - err := root.Load("./testdata/duplicate_resource_names_in_root/databricks.yaml") - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yaml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yaml)") + err := root.Load("./testdata/duplicate_resource_names_in_root/databricks.yml") + assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yml)") } func TestDuplicateIdOnMergeReturnsError(t *testing.T) { root := &Root{} - err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml") + err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml") require.NoError(t, err) other := &Root{} @@ -96,7 +96,7 @@ func TestDuplicateIdOnMergeReturnsError(t *testing.T) { require.NoError(t, err) err = root.Merge(other) - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") + assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") } func TestInitializeVariables(t *testing.T) { @@ -177,8 +177,8 @@ func TestConfigFileNames_FindInPath(t *testing.T) { }{ { name: "file found", - files: []string{"databricks.yaml"}, - expected: "BASE/databricks.yaml", + files: []string{"databricks.yml"}, + expected: "BASE/databricks.yml", err: "", }, { @@ -189,7 +189,7 @@ func TestConfigFileNames_FindInPath(t *testing.T) { }, { name: "multiple files found", - files: []string{"databricks.yaml", "bundle.yaml"}, + files: []string{"databricks.yaml", "bundle.yml"}, expected: "", err: "multiple bundle root configuration files found", }, @@ -202,7 +202,7 @@ func TestConfigFileNames_FindInPath(t *testing.T) { } if runtime.GOOS == "windows" { - testCases[2].err = "The system cannot find the file specified." + testCases[3].err = "The system cannot find the file specified." } for _, tc := range testCases { diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml b/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml rename to bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml diff --git a/bundle/config/testdata/duplicate_resource_names_in_root/databricks.yaml b/bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_names_in_root/databricks.yaml rename to bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml diff --git a/bundle/root_test.go b/bundle/root_test.go index b3fa4e76c9..2f8304921e 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -76,7 +76,7 @@ func TestRootLookup(t *testing.T) { chdir(t, t.TempDir()) - // Create databricks.yaml file. + // Create databricks.yml file. f, err := os.Create(config.FileNames[0]) require.NoError(t, err) defer f.Close() diff --git a/bundle/tests/autoload_git/databricks.yaml b/bundle/tests/autoload_git/databricks.yml similarity index 100% rename from bundle/tests/autoload_git/databricks.yaml rename to bundle/tests/autoload_git/databricks.yml diff --git a/bundle/tests/basic/databricks.yaml b/bundle/tests/basic/databricks.yml similarity index 100% rename from bundle/tests/basic/databricks.yaml rename to bundle/tests/basic/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yaml b/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yaml rename to bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yaml b/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yaml rename to bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yaml b/bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yaml rename to bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids_test.go b/bundle/tests/conflicting_resource_ids_test.go index 6922149252..b75e3753f1 100644 --- a/bundle/tests/conflicting_resource_ids_test.go +++ b/bundle/tests/conflicting_resource_ids_test.go @@ -14,7 +14,7 @@ import ( func TestConflictingResourceIdsNoSubconfig(t *testing.T) { _, err := bundle.Load("./conflicting_resource_ids/no_subconfigurations") - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yaml") + bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yml") assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, bundleConfigPath)) } @@ -22,7 +22,7 @@ func TestConflictingResourceIdsOneSubconfig(t *testing.T) { b, err := bundle.Load("./conflicting_resource_ids/one_subconfiguration") require.NoError(t, err) err = bundle.Apply(context.Background(), b, bundle.Seq(mutator.DefaultMutators()...)) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yaml") + bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yml") resourcesConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/resources.yml") assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, resourcesConfigPath)) } diff --git a/bundle/tests/environment_empty/databricks.yaml b/bundle/tests/environment_empty/databricks.yml similarity index 100% rename from bundle/tests/environment_empty/databricks.yaml rename to bundle/tests/environment_empty/databricks.yml diff --git a/bundle/tests/environment_overrides/databricks.yaml b/bundle/tests/environment_overrides/databricks.yml similarity index 100% rename from bundle/tests/environment_overrides/databricks.yaml rename to bundle/tests/environment_overrides/databricks.yml diff --git a/bundle/tests/include_default/databricks.yaml b/bundle/tests/include_default/databricks.yml similarity index 100% rename from bundle/tests/include_default/databricks.yaml rename to bundle/tests/include_default/databricks.yml diff --git a/bundle/tests/include_invalid/databricks.yaml b/bundle/tests/include_invalid/databricks.yml similarity index 100% rename from bundle/tests/include_invalid/databricks.yaml rename to bundle/tests/include_invalid/databricks.yml diff --git a/bundle/tests/include_override/databricks.yaml b/bundle/tests/include_override/databricks.yml similarity index 100% rename from bundle/tests/include_override/databricks.yaml rename to bundle/tests/include_override/databricks.yml diff --git a/bundle/tests/include_with_glob/databricks.yaml b/bundle/tests/include_with_glob/databricks.yml similarity index 100% rename from bundle/tests/include_with_glob/databricks.yaml rename to bundle/tests/include_with_glob/databricks.yml diff --git a/bundle/tests/interpolation/databricks.yaml b/bundle/tests/interpolation/databricks.yml similarity index 100% rename from bundle/tests/interpolation/databricks.yaml rename to bundle/tests/interpolation/databricks.yml diff --git a/bundle/tests/job_and_pipeline/databricks.yaml b/bundle/tests/job_and_pipeline/databricks.yml similarity index 100% rename from bundle/tests/job_and_pipeline/databricks.yaml rename to bundle/tests/job_and_pipeline/databricks.yml diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 868785a16e..d92eabd3bd 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -15,7 +15,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +29,7 @@ func TestJobAndPipelineStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +42,14 @@ func TestJobAndPipelineProduction(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(j.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/variables/env_overrides/databricks.yaml b/bundle/tests/variables/env_overrides/databricks.yml similarity index 100% rename from bundle/tests/variables/env_overrides/databricks.yaml rename to bundle/tests/variables/env_overrides/databricks.yml diff --git a/bundle/tests/variables/vanilla/databricks.yaml b/bundle/tests/variables/vanilla/databricks.yml similarity index 100% rename from bundle/tests/variables/vanilla/databricks.yaml rename to bundle/tests/variables/vanilla/databricks.yml diff --git a/bundle/tests/yaml_anchors/databricks.yaml b/bundle/tests/yaml_anchors/databricks.yml similarity index 100% rename from bundle/tests/yaml_anchors/databricks.yaml rename to bundle/tests/yaml_anchors/databricks.yml diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 62f9b19a64..51d71ea2f8 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -72,7 +72,7 @@ var syncCmd = &cobra.Command{ // // To be uncommented and used once our VS Code extension is bundle aware. - // Until then, this could interfere with extension usage where a `databricks.yaml` file is present. + // Until then, this could interfere with extension usage where a `databricks.yml` file is present. // See https://github.com/databricks/cli/pull/207. // // b := bundle.GetOrNil(cmd.Context())