Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added azure_managed_identity block to databricks_storage_credential and databricks_metastore_data_access resources #1354

Merged
merged 4 commits into from
Jun 7, 2022
Merged

Added azure_managed_identity block to databricks_storage_credential and databricks_metastore_data_access resources #1354

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
43f7488
add mi support for credential & dac
nkvuong Jun 4, 2022
5b1d1e8
update doc
nkvuong Jun 4, 2022
0867144
update docs
nkvuong Jun 4, 2022
d2e8c26
Merge branch 'master' into feature/add_uc_mi_support
nfx Jun 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion catalog/resource_metastore_data_access.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,17 @@ type AzureServicePrincipal struct {
ClientSecret string `json:"client_secret"`
}

type AzureManagedIdentity struct {
AccessConnectorID string `json:"access_connector_id"`
}

type DataAccessConfiguration struct {
ID string `json:"id,omitempty" tf:"computed"`
Name string `json:"name"`
ConfigurationType string `json:"configuration_type,omitempty" tf:"computed"`
Aws *AwsIamRole `json:"aws_iam_role,omitempty" tf:"group:access"`
Azure *AzureServicePrincipal `json:"azure_service_principal,omitempty" tf:"group:access"`
AzMI *AzureManagedIdentity `json:"azure_managed_identity,omitempty" tf:"group:access"`
}

func (a DataAccessConfigurationsAPI) Create(metastoreID string, dac *DataAccessConfiguration) error {
Expand Down Expand Up @@ -65,9 +70,10 @@ func ResourceDataAccessConfiguration() *schema.Resource {
Type: schema.TypeBool,
Optional: true,
}
alof := []string{"aws_iam_role", "azure_service_principal"}
alof := []string{"aws_iam_role", "azure_service_principal", "azure_managed_identity"}
m["aws_iam_role"].AtLeastOneOf = alof
m["azure_service_principal"].AtLeastOneOf = alof
m["azure_managed_identity"].AtLeastOneOf = alof
return m
})
p := common.NewPairID("metastore_id", "id")
Expand Down
54 changes: 54 additions & 0 deletions catalog/resource_metastore_data_access_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,57 @@ func TestCreateDac(t *testing.T) {
`,
}.ApplyNoError(t)
}

func TestCreateDacWithAzMI(t *testing.T) {
qa.ResourceFixture{
Fixtures: []qa.HTTPFixture{
{
Method: "POST",
Resource: "/api/2.0/unity-catalog/metastores/abc/data-access-configurations",
ExpectedRequest: DataAccessConfiguration{
Name: "bcd",
AzMI: &AzureManagedIdentity{
AccessConnectorID: "def",
},
},
Response: DataAccessConfiguration{
ID: "efg",
},
},
{
Method: "PATCH",
Resource: "/api/2.0/unity-catalog/metastores/abc",
ExpectedRequest: map[string]interface{}{
"default_data_access_config_id": "efg",
},
},
{
Method: "GET",
Resource: "/api/2.0/unity-catalog/metastores/abc/data-access-configurations/efg",
Response: DataAccessConfiguration{
Name: "bcd",
AzMI: &AzureManagedIdentity{
AccessConnectorID: "def",
},
},
},
{
Method: "GET",
Resource: "/api/2.0/unity-catalog/metastores/abc",
Response: MetastoreInfo{
DefaultDacID: "efg",
},
},
},
Create: true,
Resource: ResourceDataAccessConfiguration(),
HCL: `
metastore_id = "abc"
name = "bcd"
is_default = true
azure_managed_identity {
access_connector_id = "def"
}
`,
}.ApplyNoError(t)
}
4 changes: 3 additions & 1 deletion catalog/resource_storage_credentials.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type StorageCredentialInfo struct {
Comment string `json:"comment,omitempty"`
Aws *AwsIamRole `json:"aws_iam_role,omitempty" tf:"group:access"`
Azure *AzureServicePrincipal `json:"azure_service_principal,omitempty" tf:"group:access"`
AzMI *AzureManagedIdentity `json:"azure_managed_identity,omitempty" tf:"group:access"`
MetastoreID string `json:"metastore_id,omitempty" tf:"computed"`
}

Expand All @@ -41,9 +42,10 @@ func (a StorageCredentialsAPI) delete(id string) error {
func ResourceStorageCredential() *schema.Resource {
s := common.StructToSchema(StorageCredentialInfo{},
func(m map[string]*schema.Schema) map[string]*schema.Schema {
alof := []string{"aws_iam_role", "azure_service_principal"}
alof := []string{"aws_iam_role", "azure_service_principal", "azure_managed_identity"}
m["aws_iam_role"].AtLeastOneOf = alof
m["azure_service_principal"].AtLeastOneOf = alof
m["azure_managed_identity"].AtLeastOneOf = alof
return m
})
update := updateFunctionFactory("/unity-catalog/storage-credentials", []string{"owner", "comment", "aws_iam_role", "azure_service_principal"})
Expand Down
41 changes: 41 additions & 0 deletions catalog/resource_storage_credentials_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,3 +137,44 @@ func TestUpdateStorageCredentials(t *testing.T) {
`,
}.ApplyNoError(t)
}

func TestCreateStorageCredentialWithAzMI(t *testing.T) {
qa.ResourceFixture{
Fixtures: []qa.HTTPFixture{
{
Method: "POST",
Resource: "/api/2.0/unity-catalog/storage-credentials",
ExpectedRequest: StorageCredentialInfo{
Name: "a",
AzMI: &AzureManagedIdentity{
AccessConnectorID: "def",
},
Comment: "c",
},
Response: StorageCredentialInfo{
Name: "a",
},
},
{
Method: "GET",
Resource: "/api/2.0/unity-catalog/storage-credentials/a",
Response: StorageCredentialInfo{
Name: "a",
AzMI: &AzureManagedIdentity{
AccessConnectorID: "def",
},
MetastoreID: "d",
},
},
},
Resource: ResourceStorageCredential(),
Create: true,
HCL: `
name = "a"
azure_managed_identity {
access_connector_id = "def"
}
comment = "c"
`,
}.ApplyNoError(t)
}
29 changes: 27 additions & 2 deletions docs/resources/metastore_data_access.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ resource "databricks_metastore_data_access" "this" {
}
```

For Azure
For Azure using service principal as credential

```hcl
resource "databricks_metastore" "this" {
Expand All @@ -43,7 +43,7 @@ resource "databricks_metastore" "this" {

resource "databricks_metastore_data_access" "this" {
metastore_id = databricks_metastore.this.id
name = aws_iam_role.metastore_data_access.name
name = "sp_dac"
azure_service_principal {
directory_id = var.tenant_id
application_id = azuread_application.unity_catalog.application_id
Expand All @@ -53,6 +53,28 @@ resource "databricks_metastore_data_access" "this" {
}
```

For Azure using managed identity as credential (Private Preview)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you get a green light from @annashres to document it publicly for private preview?

If not - please remove this section from registry docs just yet, create a google doc in GTM private preview folder and move it there. And also link it on internal wiki for tf provider and azmi preview.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Anna said it should be fine since the UC is gated as well and we'll ungate MIs when the UC is ungated

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then it goes to this release :)


```hcl
resource "databricks_metastore" "this" {
name = "primary"
storage_root = format("abfss://%s@%s.dfs.core.windows.net/",
azurerm_storage_account.unity_catalog.name,
azurerm_storage_container.unity_catalog.name)
owner = "uc admins"
force_destroy = true
}

resource "databricks_metastore_data_access" "this" {
metastore_id = databricks_metastore.this.id
name = "mi_dac"
azure_managed_identity {
access_connector_id = var.access_connector_id
}
is_default = true
}
```

## Argument Reference

The following arguments are required:
Expand All @@ -68,6 +90,9 @@ The following arguments are required:
* `application_id` - The application ID of the application registration within the referenced AAD tenant
* `client_secret` - The client secret generated for the above app ID in AAD. **This field is redacted on output**

`azure_managed_identity` optional configuration block for using managed identity as credential details for Azure:
* `access_connector_id` - The Resource ID of the Azure Databricks Access Connector resource, of the form `/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-name/providers/Microsoft.Databricks/accessConnectors/connector-name`

## Import

-> **Note** Importing this resource is not currently supported.
18 changes: 14 additions & 4 deletions docs/resources/storage_credential.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,22 @@ resource "databricks_grants" "external_creds" {
For Azure

```hcl
resource "databricks_storage_credential" "external" {
resource "databricks_storage_credential" "external_sp" {
name = azuread_application.ext_cred.display_name
azure_service_principal {
directory_id = var.tenant_id
application_id = azuread_application.ext_cred.application_id
client_secret = azuread_application_password.ext_cred.value
}
comment = "Managed by TF"
comment = "SP credential managed by TF"
}

resource "databricks_storage_credential" "external_mi" {
name = "mi_credential"
azure_managed_identity {
access_connector_id = var.access_connector_id
}
comment = "Managed identity credential managed by TF"
}

resource "databricks_grants" "external_creds" {
Expand All @@ -62,11 +70,13 @@ The following arguments are required:
`aws_iam_role` optional configuration block for credential details for AWS:
* `role_arn` - The Amazon Resource Name (ARN) of the AWS IAM role for S3 data access, of the form `arn:aws:iam::1234567890:role/MyRole-AJJHDSKSDF`

`azure_service_principal` optional configuration block for credential details for Azure:
`azure_service_principal` optional configuration block to use service principal as credential details for Azure:
* `directory_id` - The directory ID corresponding to the Azure Active Directory (AAD) tenant of the application
* `application_id` - The application ID of the application registration within the referenced AAD tenant
* `client_secret` - The client secret generated for the above app ID in AAD. **This field is redacted on output**
* `owner` - (Optional) Username/groupname/sp application_id storage credential owner.

`azure_managed_identity` optional configuration block for using managed identity as credential details for Azure:
* `access_connector_id` - The Resource ID of the Azure Databricks Access Connector resource, of the form `/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-name/providers/Microsoft.Databricks/accessConnectors/connector-name`

## Import

Expand Down