Add data sources for retrieving UC tables (databricks#1155)

Add data sources for retrieving UC entity names:

```hcl
data "databricks_schemas" "sandbox" {
  catalog_name = "sandbox"
}

output "all_sandbox_schemas" {
  value = data.databricks_schemas.sandbox
}

data "databricks_tables" "things" {
  catalog_name = "sandbox"
  schema_name = "things"
}

output "all_things_tables" {
  value = data.databricks_tables.things
}

data "databricks_catalogs" "all" {}

output "all_catalogs" {
  value = data.databricks_catalogs.all
}
```

Fixes databricks#1105

.gitignore

@@ -338,4 +338,6 @@ tf.log
 *.env
 *~
 
-scripts/tt
+scripts/tt
+
+.metals

README.md

@@ -12,6 +12,7 @@
 | [databricks_aws_bucket_policy](docs/data-sources/aws_bucket_policy.md) data
 | [databricks_aws_crossaccount_policy](docs/data-sources/aws_crossaccount_policy.md) data
 | [databricks_catalog](docs/resources/catalog.md)
+| [databricks_catalogs](docs/data-sources/catalog.md) data
 | [databricks_cluster](docs/resources/cluster.md)
 | [databricks_clusters](docs/data-sources/clusters.md) data
 | [databricks_cluster_policy](docs/resources/cluster_policy.md)
@@ -51,6 +52,7 @@
 | [databricks_pipeline](docs/resources/pipeline.md)
 | [databricks_repo](docs/resources/repo.md)
 | [databricks_schema](docs/resources/schema.md)
+| [databricks_schemas](docs/data-sources/schema.md) data
 | [databricks_secret](docs/resources/secret.md)
 | [databricks_secret_acl](docs/resources/secret_acl.md)
 | [databricks_secret_scope](docs/resources/secret_scope.md)
@@ -64,6 +66,7 @@
 | [databricks_sql_widget](docs/resources/sql_widget.md)
 | [databricks_storage_credential](docs/resources/storage_credential.md)
 | [databricks_table](docs/resources/table.md)
+| [databricks_tables](docs/data-sources/table.md) data
 | [databricks_token](docs/resources/token.md)
 | [databricks_user](docs/resources/user.md)
 | [databricks_user_instance_profile](docs/resources/user_instance_profile.md)

catalog/data_catalogs.go

@@ -0,0 +1,25 @@
+package catalog
+
+import (
+	"context"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+func DataSourceCatalogs() *schema.Resource {
+	var data struct {
+		Ids []string `json:"ids,omitempty" tf:"computed,slice_set"`
+	}
+	return common.DataResource(&data, func(ctx context.Context, c *common.DatabricksClient) error {
+		catalogsAPI := NewCatalogsAPI(ctx, c)
+		catalogs, err := catalogsAPI.list()
+		if err != nil {
+			return err
+		}
+		for _, v := range catalogs.Catalogs {
+			data.Ids = append(data.Ids, v.Name)
+		}
+		return nil
+	})
+}

catalog/data_catalogs_test.go

@@ -0,0 +1,42 @@
+package catalog
+
+import (
+	"testing"
+
+	"github.com/databrickslabs/terraform-provider-databricks/qa"
+)
+
+func TestCatalogsData(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/catalogs",
+				Response: Catalogs{
+					Catalogs: []CatalogInfo{
+						{
+							Name: "b",
+						},
+						{
+							Name: "a",
+						},
+					},
+				},
+			},
+		},
+		Resource:    DataSourceCatalogs(),
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+	}.ApplyNoError(t)
+}
+
+func TestCatalogsData_Error(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures:    qa.HTTPFailures,
+		Resource:    DataSourceCatalogs(),
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+	}.ExpectError(t, "I'm a teapot")
+}

catalog/data_schemas.go

@@ -0,0 +1,26 @@
+package catalog
+
+import (
+	"context"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+func DataSourceSchemas() *schema.Resource {
+	var data struct {
+		CatalogName string   `json:"catalog_name"`
+		Ids         []string `json:"ids,omitempty" tf:"computed,slice_set"`
+	}
+	return common.DataResource(&data, func(ctx context.Context, c *common.DatabricksClient) error {
+		schemasAPI := NewSchemasAPI(ctx, c)
+		schemas, err := schemasAPI.listByCatalog(data.CatalogName)
+		if err != nil {
+			return err
+		}
+		for _, v := range schemas.Schemas {
+			data.Ids = append(data.Ids, v.FullName)
+		}
+		return nil
+	})
+}

catalog/data_schemas_test.go

@@ -0,0 +1,43 @@
+package catalog
+
+import (
+	"testing"
+
+	"github.com/databrickslabs/terraform-provider-databricks/qa"
+)
+
+func TestSchemasData(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/schemas?catalog_name=a",
+				Response: Schemas{
+					Schemas: []SchemaInfo{
+						{
+							Name: "a.c",
+						},
+						{
+							Name: "a.d",
+						},
+					},
+				},
+			},
+		},
+		Resource:    DataSourceSchemas(),
+		HCL:         `catalog_name = "a"`,
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+	}.ApplyNoError(t)
+}
+
+func TestSchemasData_Error(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures:    qa.HTTPFailures,
+		Resource:    DataSourceSchemas(),
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+	}.ExpectError(t, "I'm a teapot")
+}

catalog/data_tables.go

@@ -0,0 +1,27 @@
+package catalog
+
+import (
+	"context"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+func DataSourceTables() *schema.Resource {
+	var data struct {
+		CatalogName string   `json:"catalog_name"`
+		SchemaName  string   `json:"schema_name"`
+		Ids         []string `json:"ids,omitempty" tf:"computed,slice_set"`
+	}
+	return common.DataResource(&data, func(ctx context.Context, c *common.DatabricksClient) error {
+		tablesAPI := NewTablesAPI(ctx, c)
+		tables, err := tablesAPI.listTables(data.CatalogName, data.SchemaName)
+		if err != nil {
+			return err
+		}
+		for _, v := range tables.Tables {
+			data.Ids = append(data.Ids, v.FullName())
+		}
+		return nil
+	})
+}

catalog/data_tables_test.go

@@ -0,0 +1,45 @@
+package catalog
+
+import (
+	"testing"
+
+	"github.com/databrickslabs/terraform-provider-databricks/qa"
+)
+
+func TestTablesData(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/tables/?catalog_name=a&schema_name=b",
+				Response: Tables{
+					Tables: []TableInfo{
+						{
+							Name: "a.b.c",
+						},
+						{
+							Name: "a.b.d",
+						},
+					},
+				},
+			},
+		},
+		Resource: DataSourceTables(),
+		HCL: `
+		catalog_name = "a"
+		schema_name = "b"`,
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+	}.ApplyNoError(t)
+}
+
+func TestTablesData_Error(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures:    qa.HTTPFailures,
+		Resource:    DataSourceTables(),
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+	}.ExpectError(t, "I'm a teapot")
+}

catalog/resource_catalog.go

@@ -25,6 +25,15 @@ type CatalogInfo struct {
 	MetastoreID string            `json:"metastore_id,omitempty" tf:"computed"`
 }
 
+type Catalogs struct {
+	Catalogs []CatalogInfo `json:"catalogs"`
+}
+
+func (a CatalogsAPI) list() (catalogs Catalogs, err error) {
+	err = a.client.Get(a.context, "/unity-catalog/catalogs", nil, &catalogs)
+	return
+}
+
 func (a CatalogsAPI) createCatalog(ci *CatalogInfo) error {
 	return a.client.Post(a.context, "/unity-catalog/catalogs", ci, ci)
 }

catalog/resource_schema.go

@@ -26,6 +26,17 @@ type SchemaInfo struct {
 	FullName    string            `json:"full_name,omitempty" tf:"computed"`
 }
 
+type Schemas struct {
+	Schemas []SchemaInfo `json:"schemas"`
+}
+
+func (a SchemasAPI) listByCatalog(catalogName string) (schemas Schemas, err error) {
+	err = a.client.Get(a.context, "/unity-catalog/schemas", map[string]string{
+		"catalog_name": catalogName,
+	}, &schemas)
+	return
+}
+
 func (a SchemasAPI) createSchema(si *SchemaInfo) error {
 	return a.client.Post(a.context, "/unity-catalog/schemas", si, si)
 }

catalog/resource_table.go

@@ -50,6 +50,18 @@ func (ti TableInfo) FullName() string {
 	return fmt.Sprintf("%s.%s.%s", ti.CatalogName, ti.SchemaName, ti.Name)
 }
 
+type Tables struct {
+	Tables []TableInfo `json:"tables"`
+}
+
+func (a TablesAPI) listTables(catalogName, schemaName string) (tables Tables, err error) {
+	err = a.client.Get(a.context, "/unity-catalog/tables/", map[string]string{
+		"catalog_name": catalogName,
+		"schema_name":  schemaName,
+	}, &tables)
+	return
+}
+
 func (a TablesAPI) createTable(ti *TableInfo) error {
 	return a.client.Post(a.context, "/unity-catalog/tables", ti, ti)
 }

common/reflect_resource_test.go

@@ -584,7 +584,7 @@ func TestDataToReflectValueBypass(t *testing.T) {
 func TestDataResource(t *testing.T) {
 	r := func() *schema.Resource {
 		var dto struct {
-			In string `json:"in"`
+			In  string `json:"in"`
 			Out string `json:"out,omitempty" tf:"computed"`
 		}
 		return DataResource(&dto, func(ctx context.Context, c *DatabricksClient) error {
@@ -608,4 +608,4 @@ func TestDataResource(t *testing.T) {
 	d.Set("in", "fail")
 	diags = r.ReadContext(context.Background(), d, &DatabricksClient{})
 	assert.Len(t, diags, 1)
-}
+}

common/version.go

@@ -3,7 +3,7 @@ package common
 import "context"
 
 var (
-	version = "0.5.1"
+	version = "0.5.2"
 	// ResourceName is resource name without databricks_ prefix
 	ResourceName contextKey = 1
 	// Provider is the current instance of provider

docs/data-sources/catalogs.md

@@ -0,0 +1,34 @@
+---
+subcategory: "Unity Catalog"
+---
+# databricks_catalogs Data Source
+
+-> **Note** If you have a fully automated setup with workspaces created by [databricks_mws_workspaces](../resources/mws_workspaces.md) or [azurerm_databricks_workspace](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/databricks_workspace), please make sure to add [depends_on attribute](../index.md#data-resources-and-authentication-is-not-configured-errors) in order to prevent _authentication is not configured for provider_ errors.
+
+Retrieves a list of [databricks_catalog](../resources/catalog.md) ids, that were created by Terraform or manually, so that special handling could be applied.
+
+## Example Usage
+
+Listing all catalogs:
+
+```hcl
+data "databricks_catalogs" "all" {}
+
+output "all_catalogs" {
+  value = data.databricks_catalogs.all
+}
+```
+
+## Attribute Reference
+
+This data source exports the following attributes:
+
+* `ids` - set of [databricks_catalog](../resources/catalog.md) names
+
+## Related Resources
+
+The following resources are used in the same context:
+
+* [databricks_table](../resources/table.md) to manage tables within Unity Catalog.
+* [databricks_schema](../resources/schema.md) to manage schemas within Unity Catalog.
+* [databricks_catalog](../resources/catalog.md) to manage catalogs within Unity Catalog.