Skip to content

Commit

Permalink
use databricks.yml by default
Browse files Browse the repository at this point in the history
  • Loading branch information
fjakobs committed Jul 18, 2023
1 parent 2d87c78 commit 3d3c2c9
Show file tree
Hide file tree
Showing 26 changed files with 24 additions and 24 deletions.
4 changes: 2 additions & 2 deletions bundle/bundle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestLoadExists(t *testing.T) {

func TestBundleCacheDir(t *testing.T) {
projectDir := t.TempDir()
f1, err := os.Create(filepath.Join(projectDir, "databricks.yaml"))
f1, err := os.Create(filepath.Join(projectDir, "databricks.yml"))
require.NoError(t, err)
f1.Close()

Expand All @@ -47,7 +47,7 @@ func TestBundleCacheDir(t *testing.T) {
func TestBundleCacheDirOverride(t *testing.T) {
projectDir := t.TempDir()
bundleTmpDir := t.TempDir()
f1, err := os.Create(filepath.Join(projectDir, "databricks.yaml"))
f1, err := os.Create(filepath.Join(projectDir, "databricks.yml"))
require.NoError(t, err)
f1.Close()

Expand Down
2 changes: 1 addition & 1 deletion bundle/config/mutator/process_root_includes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) {
},
}

touch(t, bundle.Config.Path, "databricks.yaml")
touch(t, bundle.Config.Path, "databricks.yml")
touch(t, bundle.Config.Path, "a.yml")
touch(t, bundle.Config.Path, "b.yml")

Expand Down
8 changes: 4 additions & 4 deletions bundle/config/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ import (
"github.com/imdario/mergo"
)

type ConfigFileNames [4]string
type ConfigFileNames []string

// FileNames contains allowed names of bundle configuration files.
var FileNames = ConfigFileNames{"databricks.yaml", "databricks.yml", "bundle.yaml", "bundle.yml"}
var FileNames = ConfigFileNames{"databricks.yml", "databricks.yaml", "bundle.yml", "bundle.yaml"}

func (c ConfigFileNames) FindInPath(path string) (string, error) {
result := ""
Expand Down Expand Up @@ -43,7 +43,7 @@ func (c ConfigFileNames) FindInPath(path string) (string, error) {

type Root struct {
// Path contains the directory path to the root of the bundle.
// It is set when loading `databricks.yaml`.
// It is set when loading `databricks.yml`.
Path string `json:"-" bundle:"readonly"`

// Contains user defined variables
Expand All @@ -54,7 +54,7 @@ type Root struct {
Bundle Bundle `json:"bundle"`

// Include specifies a list of patterns of file names to load and
// merge into the this configuration. If not set in `databricks.yaml`,
// merge into the this configuration. If not set in `databricks.yml`,
// it defaults to loading `*.yml` and `*/*.yml`.
//
// Also see [mutator.DefineDefaultInclude].
Expand Down
18 changes: 9 additions & 9 deletions bundle/config/root_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func TestRootMarshalUnmarshal(t *testing.T) {

func TestRootLoad(t *testing.T) {
root := &Root{}
err := root.Load("../tests/basic/databricks.yaml")
err := root.Load("../tests/basic/databricks.yml")
require.NoError(t, err)
assert.Equal(t, "basic", root.Bundle.Name)
}
Expand Down Expand Up @@ -82,21 +82,21 @@ func TestRootMergeMap(t *testing.T) {

func TestDuplicateIdOnLoadReturnsError(t *testing.T) {
root := &Root{}
err := root.Load("./testdata/duplicate_resource_names_in_root/databricks.yaml")
assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yaml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yaml)")
err := root.Load("./testdata/duplicate_resource_names_in_root/databricks.yml")
assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yml)")
}

func TestDuplicateIdOnMergeReturnsError(t *testing.T) {
root := &Root{}
err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml")
err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml")
require.NoError(t, err)

other := &Root{}
err = other.Load("./testdata/duplicate_resource_name_in_subconfiguration/resources.yml")
require.NoError(t, err)

err = root.Merge(other)
assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yaml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)")
assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)")
}

func TestInitializeVariables(t *testing.T) {
Expand Down Expand Up @@ -177,8 +177,8 @@ func TestConfigFileNames_FindInPath(t *testing.T) {
}{
{
name: "file found",
files: []string{"databricks.yaml"},
expected: "BASE/databricks.yaml",
files: []string{"databricks.yml"},
expected: "BASE/databricks.yml",
err: "",
},
{
Expand All @@ -189,7 +189,7 @@ func TestConfigFileNames_FindInPath(t *testing.T) {
},
{
name: "multiple files found",
files: []string{"databricks.yaml", "bundle.yaml"},
files: []string{"databricks.yaml", "bundle.yml"},
expected: "",
err: "multiple bundle root configuration files found",
},
Expand All @@ -202,7 +202,7 @@ func TestConfigFileNames_FindInPath(t *testing.T) {
}

if runtime.GOOS == "windows" {
testCases[2].err = "The system cannot find the file specified."
testCases[3].err = "The system cannot find the file specified."
}

for _, tc := range testCases {
Expand Down
2 changes: 1 addition & 1 deletion bundle/root_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func TestRootLookup(t *testing.T) {

chdir(t, t.TempDir())

// Create databricks.yaml file.
// Create databricks.yml file.
f, err := os.Create(config.FileNames[0])
require.NoError(t, err)
defer f.Close()
Expand Down
File renamed without changes.
File renamed without changes.
4 changes: 2 additions & 2 deletions bundle/tests/conflicting_resource_ids_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ import (

func TestConflictingResourceIdsNoSubconfig(t *testing.T) {
_, err := bundle.Load("./conflicting_resource_ids/no_subconfigurations")
bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yaml")
bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yml")
assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, bundleConfigPath))
}

func TestConflictingResourceIdsOneSubconfig(t *testing.T) {
b, err := bundle.Load("./conflicting_resource_ids/one_subconfiguration")
require.NoError(t, err)
err = bundle.Apply(context.Background(), b, bundle.Seq(mutator.DefaultMutators()...))
bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yaml")
bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yml")
resourcesConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/resources.yml")
assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, resourcesConfigPath))
}
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
8 changes: 4 additions & 4 deletions bundle/tests/job_and_pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) {
assert.Len(t, b.Config.Resources.Pipelines, 1)

p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath))
assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
assert.Equal(t, b.Config.Bundle.Mode, config.Development)
assert.True(t, p.Development)
require.Len(t, p.Libraries, 1)
Expand All @@ -29,7 +29,7 @@ func TestJobAndPipelineStaging(t *testing.T) {
assert.Len(t, b.Config.Resources.Pipelines, 1)

p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath))
assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
assert.False(t, p.Development)
require.Len(t, p.Libraries, 1)
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
Expand All @@ -42,14 +42,14 @@ func TestJobAndPipelineProduction(t *testing.T) {
assert.Len(t, b.Config.Resources.Pipelines, 1)

p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"]
assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(p.ConfigFilePath))
assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath))
assert.False(t, p.Development)
require.Len(t, p.Libraries, 1)
assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path)
assert.Equal(t, "nyc_taxi_production", p.Target)

j := b.Config.Resources.Jobs["pipeline_schedule"]
assert.Equal(t, "job_and_pipeline/databricks.yaml", filepath.ToSlash(j.ConfigFilePath))
assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath))
assert.Equal(t, "Daily refresh of production pipeline", j.Name)
require.Len(t, j.Tasks, 1)
assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId)
Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion cmd/sync/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ var syncCmd = &cobra.Command{

//
// To be uncommented and used once our VS Code extension is bundle aware.
// Until then, this could interfere with extension usage where a `databricks.yaml` file is present.
// Until then, this could interfere with extension usage where a `databricks.yml` file is present.
// See https://github.com/databricks/cli/pull/207.
//
// b := bundle.GetOrNil(cmd.Context())
Expand Down

0 comments on commit 3d3c2c9

Please sign in to comment.