diff --git a/.github/ISSUE_TEMPLATE/docs-issue.md b/.github/ISSUE_TEMPLATE/docs-issue.md index bc320960fc..e35ed90342 100644 --- a/.github/ISSUE_TEMPLATE/docs-issue.md +++ b/.github/ISSUE_TEMPLATE/docs-issue.md @@ -2,9 +2,7 @@ name: Docs Issue about: Use this to suggest enhancements or point out bugs for the documentation website. title: "[DOC] Issue Short Description" -labels: '' -assignees: stikkireddy - +labels: documentation --- Hi there, @@ -30,4 +28,4 @@ Any links to external documentation that may prove your case, i.e Databricks pub ### References Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: -- GH-1234 +- #158 diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 2ab5bf8d12..0000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: "[FEATURE] Feature Short Description" -labels: enhancement -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/feedback.md b/.github/ISSUE_TEMPLATE/feedback.md deleted file mode 100644 index a1212c7d37..0000000000 --- a/.github/ISSUE_TEMPLATE/feedback.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Feedback -about: Use this to provide feedback or ask questions regarding the provider. -title: "[FEEDBACK/QUESTION] Short Description of feedback" -labels: question -assignees: stikkireddy - ---- - -**Please provide feedback in regards to user experience.** -A clear and concise summary of the situation whether it is positive or negative. - -**Describe any questions you may have** -A clear and concise set of questions to be answered by the maintainers. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/provider-issue.md b/.github/ISSUE_TEMPLATE/provider-issue.md index 3d7c8bd2b1..7d5a484855 100644 --- a/.github/ISSUE_TEMPLATE/provider-issue.md +++ b/.github/ISSUE_TEMPLATE/provider-issue.md @@ -2,9 +2,7 @@ name: Provider Issue about: Use this to identify a issue or a bug with the provider. title: "[ISSUE] Issue Short Description" -labels: '' -assignees: stikkireddy - +labels: bug --- Hi there, @@ -45,8 +43,4 @@ Please list the steps required to reproduce the issue, for example: 1. `terraform apply` ### Important Factoids -Are there anything atypical about your accounts that we should know? For example: Running in EC2 Classic? Custom version of OpenStack? Tight ACLs? - -### References -Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: -- GH-1234 +Are there anything atypical about your accounts that we should know? \ No newline at end of file diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml deleted file mode 100644 index 40eb66c196..0000000000 --- a/.github/workflows/gh-pages.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: github pages - -on: - push: - branches: - - master - paths: - - 'website/**' - - -jobs: - deploy: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - with: - submodules: true # Fetch Hugo themes - fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod - - - name: Setup Hugo - uses: peaceiris/actions-hugo@v2 - with: - hugo-version: '0.68.3' - extended: true - - - name: Build - run: cd website && hugo --minify - - - name: Deploy - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.token }} - publish_branch: gh-pages - publish_dir: ./website/public \ No newline at end of file diff --git a/.gitignore b/.gitignore index cc55344994..58d8a9d6ca 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ # Created by https://www.gitignore.io/api/go,python,terraform,virtualenv,pycharm+iml,intellij+all,visualstudiocode # Edit at https://www.gitignore.io/?templates=go,python,terraform,virtualenv,pycharm+iml,intellij+all,visualstudiocode +common/testdata/.azure + ### Go ### # Binaries for programs and plugins *.exe diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32e2fab32b..dcc3f42bce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,9 +14,6 @@ Contributing to Databricks Terraform Provider - [Random naming anywhere](#random-naming-anywhere) - [Integration Testing](#integration-testing) - [Pre-release procedure](#pre-release-procedure) -- [Project Components](#project-components) - - [Databricks Terraform Provider Resources State](#databricks-terraform-provider-resources-state) - - [Databricks Terraform Data Sources State](#databricks-terraform-data-sources-state) We happily welcome contributions to databricks-terraform. We use GitHub Issues to track community reported issues and GitHub Pull Requests for accepting changes. @@ -191,47 +188,4 @@ crucial for making sure that the provider behaves as expected on all supported c 2. `make test-mws` if MWS related code changed given release. 3. Create release notes. 4. Perfrom backwards-compatibility checks and make proper notes. - -## Project Components - -### Databricks Terraform Provider Resources State - -| Resource | Implemented | Import Support | Acceptance Tests | Documentation | Reviewed | Finalize Schema | -|----------------------------------|--------------------|----------------------|----------------------|----------------------|----------------------|----------------------| -| databricks_token | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_secret_scope | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_secret | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_secret_acl | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_instance_pool | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_scim_user | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_scim_group | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_notebook | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_cluster | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_job | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file_sync | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_instance_profile | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_aws_s3_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_azure_blob_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_azure_adls_gen1_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_azure_adls_gen2_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | - -### Databricks Terraform Data Sources State - -| Data Source | Implemented | Acceptance Tests | Documentation | Reviewed | -|-----------------------------|----------------------|----------------------|----------------------|----------------------| -| databricks_notebook | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_notebook_paths | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file_paths | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_zones | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_runtimes | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_instance_pool | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_scim_user | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_scim_group | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_cluster | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_job | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_mount | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_instance_profile | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_database | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_table | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +5. \ No newline at end of file diff --git a/README.md b/README.md index bbcbb1512f..7db0d21a47 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,32 @@ [![Build Status](https://travis-ci.org/databrickslabs/terraform-provider-databricks.svg?branch=master)](https://travis-ci.org/databrickslabs/terraform-provider-databricks) [![codecov](https://codecov.io/gh/databrickslabs/terraform-provider-databricks/branch/master/graph/badge.svg)](https://codecov.io/gh/databrickslabs/terraform-provider-databricks) -[Documentation](https://databrickslabs.github.io/terraform-provider-databricks/provider/) | [Contributing and Development Guidelines](CONTRIBUTING.md) +[Authentication](docs/index.md) +| [databricks_aws_s3_mount](docs/resources/aws_s3_mount.md) +| [databricks_azure_adls_gen1_mount](docs/resources/azure_adls_gen1_mount.md) +| [databricks_azure_adls_gen2_mount](docs/resources/azure_adls_gen2_mount.md) +| [databricks_azure_blob_mount](docs/resources/azure_blob_mount.md) +| [databricks_cluster](docs/resources/cluster.md) +| [databricks_cluster_policy](docs/resources/cluster_policy.md) +| [databricks_dbfs_file](docs/resources/dbfs_file.md) +| [databricks_group](docs/resources/group.md) +| [databricks_group_instance_profile](docs/resources/group_instance_profile.md) +| [databricks_group_member](docs/resources/group_member.md) +| [databricks_instance_pool](docs/resources/instance_pool.md) +| [databricks_instance_profile](docs/resources/instance_profile.md) +| [databricks_job](docs/resources/job.md) +| [databricks_mws_credentials](docs/resources/mws_credentials.md) +| [databricks_mws_networks](docs/resources/mws_networks.md) +| [databricks_mws_storage_configurations](docs/resources/mws_storage_configurations.md) +| [databricks_mws_workspaces](docs/resources/mws_workspaces.md) +| [databricks_notebook](docs/resources/notebook.md) +| [databricks_permissions](docs/resources/permissions.md) +| [databricks_scim_user](docs/resources/scim_user.md) +| [databricks_secret](docs/resources/secret.md) +| [databricks_secret_acl](docs/resources/secret_acl.md) +| [databricks_secret_scope](docs/resources/secret_scope.md) +| [databricks_token](docs/resources/token.md) +| [Contributing and Development Guidelines](CONTRIBUTING.md) To quickly install the binary please execute the following curl command in your shell or [install provider from source](CONTRIBUTING.md#installing-from-source). @@ -31,7 +56,7 @@ resource "databricks_cluster" "shared_autoscaling" { } ``` -Then run `terraform init` then `terraform apply` to apply the hcl code to your Databricks workspace. Please refer to the [end-user documentation](https://databrickslabs.github.io/terraform-provider-databricks/provider/) for detailed use of the provider. Also refer to these [examples](examples/) for more scenarios. +Then run `terraform init` then `terraform apply` to apply the hcl code to your Databricks workspace. ## Project Support Please note that all projects in the /databrickslabs github account are provided for your exploration only, and are not formally supported by Databricks with Service Level Agreements (SLAs). They are provided AS-IS and we do not make any guarantees of any kind. Please do not submit a support ticket relating to any issues arising from the use of these projects. diff --git a/codecov.yml b/codecov.yml index e6322690c3..0b1b76f116 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,4 @@ ignore: - - "client/**/*test.go" - - "databricks/*test.go" - "vendor/**/*" - "website/**/*" - "dist/**/*" @@ -8,28 +6,13 @@ ignore: coverage: status: - project: - client: - target: auto - flags: - - client - provider: - target: auto - flags: - - provider + project: yes + patch: yes + changes: yes comment: layout: "reach, diff, flags, files" behavior: default require_changes: false # if true: only post the comment if coverage changes require_base: yes # [yes :: must have a base report to post] require_head: yes # [yes :: must have a head report to post] - branches: null # branch names that can post comment -flags: - client: - paths: - - "client/**/*.go" - carryforward: false - provider: - paths: - - "databricks/*.go" - carryforward: true \ No newline at end of file + branches: null # branch names that can post comment \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 1a732f6b9e..8d25846073 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,9 +8,7 @@ description: |- # Databricks Provider -The Databricks provider is what is used to interact with the Databricks resources. This needs to be configured so that -terraform can provision resources in your Databricks workspace on your behalf. - +The Databricks provider is what is used to interact with the Databricks resources. This needs to be configured so that terraform can provision resources in your Databricks workspace on your behalf. ## Example Usage @@ -46,8 +44,8 @@ is `pat_token_duration_seconds` which will be deprecated and after AAD support i There are currently three supported methods [to authenticate into](https://docs.databricks.com/dev-tools/api/latest/authentication.html) the Databricks platform to create resources: * [PAT Tokens](https://docs.databricks.com/dev-tools/api/latest/authentication.html) -* Username+Password pair -* Azure Active Directory Tokens via Azure Service Principal +* Username and password pair +* Azure Active Directory Tokens via Azure Service Principal or Azure CLI ### Authenticating with Databricks CLI credentials @@ -110,7 +108,7 @@ provider "databricks" { -> **Note** **Azure Service Principal Authentication** will only work on Azure Databricks where as the API Token authentication will work on both **Azure** and **AWS**. Internally `azure_auth` will generate a session-based PAT token. -``` hcl +```hcl provider "azurerm" { client_id = var.client_id client_secret = var.client_secret @@ -122,21 +120,14 @@ resource "azurerm_databricks_workspace" "demo_test_workspace" { location = "centralus" name = "my-workspace-name" resource_group_name = var.resource_group - managed_resource_group_name = var.managed_resource_group_name sku = "premium" } provider "databricks" { - azure_auth = { - managed_resource_group = azurerm_databricks_workspace.demo_test_workspace.managed_resource_group_name - azure_region = azurerm_databricks_workspace.demo_test_workspace.location - workspace_name = azurerm_databricks_workspace.demo_test_workspace.name - resource_group = azurerm_databricks_workspace.demo_test_workspace.resource_group_name - client_id = var.client_id - client_secret = var.client_secret - tenant_id = var.tenant_id - subscription_id = var.subscription_id - } + azure_workspace_resource_id = azurerm_databricks_workspace.demo_test_workspace.id + azure_client_id = var.client_id + azure_client_secret = var.client_secret + azure_tenant_id = var.tenant_id } resource "databricks_scim_user" "my-user" { @@ -145,136 +136,100 @@ resource "databricks_scim_user" "my-user" { } ``` -## Argument Reference - -The following arguments are supported by the db provider block: - -* `host` - (optional) This is the host of the Databricks workspace. This is will be a url that you use to login to your workspace. -Alternatively you can provide this value as an environment variable `DATABRICKS_HOST`. - -* `token` - (optional) This is the api token to authenticate into the workspace. Alternatively you can provide this value as an -environment variable `DATABRICKS_TOKEN`. +### Authenticating with Azure CLI -* `basic_auth` - (optional) This is a basic_auth block ([documented below](#basic_auth-configuration-block)) to authenticate to the Databricks via basic auth through a user -that has access to the workspace. This is optional as you can use the api token based auth. - -* `config_file` - (optional) Location of the Databricks CLI credentials file, that is created, by `databricks configure --token` command. -By default, it is located in ~/.databrickscfg. Check https://docs.databricks.com/dev-tools/cli/index.html#set-up-authentication -for docs. Config file credentials will only be used when host/token/basic_auth/azure_auth are not provided. -Alternatively you can provide this value as an environment variable `DATABRICKS_CONFIG_FILE`. This field defaults to -`~/.databrickscfg`. - -* `profile` - (optional) Connection profile specified within ~/.databrickscfg. Please check -https://docs.databricks.com/dev-tools/cli/index.html#connection-profiles for documentation. This field defaults to -`DEFAULT`. - -* `azure_auth` - (optional) This is a azure_auth block ([documented below]((#azure_auth-configuration-block))) required to authenticate to the Databricks via an azure service -principal that has access to the workspace. This is optional as you can use the api token based auth. - -### basic_auth Configuration Block - -Example: +-> **Note** **Azure Service Principal Authentication** will only work on Azure Databricks where as the API Token authentication will work on both **Azure** and **AWS**. Internally `azure_auth` will generate a session-based PAT token. ```hcl -basic_auth = { - username = "user" - password = "mypass-123" +provider "azurerm" { + features {} } -``` - -The basic_auth block contains the following arguments: - -* `username` - (required) This is the username of the user that can log into the workspace. -Alternatively you can provide this value as an environment variable `DATABRICKS_USERNAME`. - -* `password` - (required) This is the password of the user that can log into the workspace. -Alternatively you can provide this value as an environment variable `DATABRICKS_PASSWORD`. +resource "azurerm_databricks_workspace" "demo_test_workspace" { + location = "centralus" + name = "my-workspace-name" + resource_group_name = var.resource_group + sku = "premium" +} -### azure_auth Configuration Block - -Example: +provider "databricks" { + azure_workspace_resource_id = azurerm_databricks_workspace.demo_test_workspace.id +} -```hcl -azure_auth = { - azure_region = "centralus" - managed_resource_group = "my-databricks-managed-rg" - workspace_name = "test-managed-workspace" - resource_group = "1-test-rg" - client_id = var.client_id - client_secret = var.client_secret - tenant_id = var.tenant_id - subscription_id = var.subscription_id +resource "databricks_scim_user" "my-user" { + user_name = "test-user@databricks.com" + display_name = "Test User" } ``` -This is the authentication required to authenticate to the Databricks via an azure service -principal that has access to the workspace. This is optional as you can use the api token based auth. -The azure_auth block contains the following arguments: +## Argument Reference + +The following arguments are supported by the db provider block: -* `managed_resource_group` - (required) This is the managed resource group id when the Databricks workspace is provisioned. -Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP`. +* `host` - (optional) This is the host of the Databricks workspace. This is will be a url that you use to login to your workspace. +Alternatively you can provide this value as an environment variable `DATABRICKS_HOST`. +* `token` - (optional) This is the api token to authenticate into the workspace. Alternatively you can provide this value as an +environment variable `DATABRICKS_TOKEN`. +* `username` - (optional) This is the username of the user that can log into the workspace. Alternatively you can provide this value as an environment variable `DATABRICKS_USERNAME`. Recommended only for [creating workspaced in AWS](resources/mws_workspaces.md). +* `password` - (optional) This is the password of the user that can log into the workspace. Alternatively you can provide this value as an environment variable `DATABRICKS_PASSWORD`. Recommended only for [creating workspaced in AWS](resources/mws_workspaces.md). +* `config_file` - (optional) Location of the Databricks CLI credentials file, that is created, by `databricks configure --token` command. By default, it is located in ~/.databrickscfg. Check [databricks cli documentation](https://docs.databricks.com/dev-tools/cli/index.html#set-up-authentication) for more details. Config file credentials will only be used when host/token/basic_auth/azure_auth are not provided. Alternatively you can provide this value as an environment variable `DATABRICKS_CONFIG_FILE`. This field defaults to `~/.databrickscfg`. +* `profile` - (optional) Connection profile specified within ~/.databrickscfg. Please check [connection profiles section](https://docs.databricks.com/dev-tools/cli/index.html#connection-profiles) for more details. This field defaults to +`DEFAULT`. -* `azure_region` - (required) This is the azure region in which your workspace is deployed. -Alternatively you can provide this value as an environment variable `AZURE_REGION`. +## Special configurations for Azure -* `workspace_name` - (required) This is the name of your Azure Databricks Workspace. -Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_WORKSPACE_NAME`. +In order to work with Azure Databricks workspace, provider has to know it's `id`. Or construct it from `azure_subscription_id`, `azure_workspace_name` and `azure_workspace_name`. -* `resource_group` - (required) This is the resource group in which your Azure Databricks Workspace resides in. -Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_RESOURCE_GROUP`. +* `azure_workspace_resource_id` - (optional) `id` attribute of [azurerm_databricks_workspace](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/databricks_workspace) resource. Combination of subscription id, resource group name and workspace name. +* `azure_workspace_name` - (optional) This is the name of your Azure Databricks Workspace. Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_WORKSPACE_NAME`. Not needed with `azure_workspace_resource_id` is set. +* `azure_resource_group` - (optional) This is the resource group in which your Azure Databricks Workspace resides in. Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_RESOURCE_GROUP`. Not needed with `azure_workspace_resource_id` is set. +* `azure_subscription_id` - (optional) This is the Azure Subscription id in which your Azure Databricks Workspace resides in. Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_SUBSCRIPTION_ID` or `ARM_SUBSCRIPTION_ID`. Not needed with `azure_workspace_resource_id` is set. -* `subscription_id` - (required) This is the Azure Subscription id in which your Azure Databricks Workspace resides in. -Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_SUBSCRIPTION_ID` or `ARM_SUBSCRIPTION_ID`. - -* `client_secret` - (required) This is the Azure Enterprise Application (Service principal) client secret. This service -principal requires contributor access to your Azure Databricks deployment. Alternatively you can provide this -value as an environment variable `DATABRICKS_AZURE_CLIENT_SECRET` or `ARM_CLIENT_SECRET`. -* `client_id` - (required) This is the Azure Enterprise Application (Service principal) client id. This service principal -requires contributor access to your Azure Databricks deployment. Alternatively you can provide this value as an -environment variable `DATABRICKS_AZURE_CLIENT_ID` or `ARM_CLIENT_ID`. +Provider works with [Azure CLI authentication](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest) to facilitate local development workflows, though for automated scenarios a service principal auth is necessary: -* `tenant_id` - (required) This is the Azure Active Directory Tenant id in which the Enterprise Application (Service Principal) +* `azure_client_secret` - (optional) This is the Azure Enterprise Application (Service principal) client secret. This service principal requires contributor access to your Azure Databricks deployment. Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_CLIENT_SECRET` or `ARM_CLIENT_SECRET`. +* `azure_client_id` - (optional) This is the Azure Enterprise Application (Service principal) client id. This service principal requires contributor access to your Azure Databricks deployment. Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_CLIENT_ID` or `ARM_CLIENT_ID`. +* `azure_tenant_id` - (optional) This is the Azure Active Directory Tenant id in which the Enterprise Application (Service Principal) resides in. Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_TENANT_ID` or `ARM_TENANT_ID`. +* `pat_token_duration_seconds` - The current implementation of the azure auth via sp requires the provider to create a temporary personal access token within Databricks. The current AAD implementation does not cover all the APIs for Authentication. This field determines the duration in which that temporary PAT token is alive for. It is measured in seconds and will default to `3600` seconds. -* `pat_token_duration_seconds` - The current implementation of the azure auth via sp requires the provider to create a temporary -personal access token within Databricks. The current AAD implementation does not cover all the APIs for Authentication. This -field determines the duration in which that temporary PAT token is alive for. It is measured in seconds and will default to -`3600` seconds. - -Where there are multiple environment variable options, the `DATABRICKS_AZURE_*` environment variables takes precedence -and the `ARM_*` environment variables provide a way to share authentication configuration when using the `databricks-terraform` -provider alongside the `azurerm` provider. +Where there are multiple environment variable options, the `DATABRICKS_AZURE_*` environment variables takes precedence and the `ARM_*` environment variables provide a way to share authentication configuration when using the `databricks` provider alongside the `azurerm` provider. ## Environment variables -The following variables can be passed via environment variables: +The following configuration attributes can be passed via environment variables: + +| Argument | Environment variable | +| --: | --- | +| `host` | `DATABRICKS_HOST` | +| `token` | `DATABRICKS_TOKEN` | +| `username` | `DATABRICKS_USERNAME` | +| `password` | `DATABRICKS_PASSWORD` | +| `config_file` | `DATABRICKS_CONFIG_FILE` | +| `profile` | `DATABRICKS_CONFIG_PROFILE` | +| `azure_workspace_resource_id` | `DATABRICKS_AZURE_WORKSPACE_RESOURCE_ID` | +| `azure_workspace_name` | `DATABRICKS_AZURE_WORKSPACE_NAME` | +| `azure_resource_group` | `DATABRICKS_AZURE_RESOURCE_GROUP` | +| `azure_subscription_id` | `DATABRICKS_AZURE_SUBSCRIPTION_ID` or `ARM_SUBSCRIPTION_ID` | +| `azure_client_secret` | `DATABRICKS_AZURE_CLIENT_SECRET` or `ARM_CLIENT_SECRET` | +| `azure_client_id` | `DATABRICKS_AZURE_CLIENT_ID` or `ARM_CLIENT_ID` | +| `azure_tenant_id` | `DATABRICKS_AZURE_TENANT_ID` or `ARM_TENANT_ID` | -* `host` → `DATABRICKS_HOST` -* `token` → `DATABRICKS_TOKEN` -* `basic_auth.username` → `DATABRICKS_USERNAME` -* `basic_auth.password` → `DATABRICKS_PASSWORD` -* `config_file` → `DATABRICKS_CONFIG_FILE` -* `managed_resource_group` → `DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP` -* `azure_region` → `AZURE_REGION` -* `workspace_name` → `DATABRICKS_AZURE_WORKSPACE_NAME` -* `resource_group` → `DATABRICKS_AZURE_RESOURCE_GROUP` -* `subscription_id` → `DATABRICKS_AZURE_SUBSCRIPTION_ID` or `ARM_SUBSCRIPTION_ID` -* `client_secret` → `DATABRICKS_AZURE_CLIENT_SECRET` or `ARM_CLIENT_SECRET` -* `client_id` → `DATABRICKS_AZURE_CLIENT_ID` or `ARM_CLIENT_ID` -* `tenant_id` → `DATABRICKS_AZURE_TENANT_ID` or `ARM_TENANT_ID` +## Empty provider block -For example you can have the following provider definition: +For example, with the following zero-argument configuration ... ``` hcl provider "databricks" {} ``` -Then run the following code and the following environment variables will be injected into the provider. - -``` bash -$ export DATABRICKS_HOST="http://databricks.domain.com" -$ export DATABRICKS_TOKEN="dapitokenhere" -$ terraform plan -``` \ No newline at end of file +1. Provider will check all of the supported environment variables and set values of relevant arguments. +2. In case of any conflicting arguments are present, plan will end with error. +3. Will check for presence of `host` + `token` pair, continue trying otherwise. +4. Will check for `host` + `username` + `password` presence, continue trying otherwise. +5. Will check for Azure workspace ID, `azure_client_secret` + `azure_client_id` + `azure_tenant_id` presence, continue trying otherwise. +6. Will check for Azure workspace ID presence and if `az cli` is authenticated, continue trying otherwise. +7. Will check for `~/.databrickscfg` file in the home directory, will fail otherwise. +8. Will check for `profile` presence and try picking from that file, will fail otherwise. +9. Will check for `host` and `token` or `username`+`password` combination, will fail if nothing of these exist. \ No newline at end of file diff --git a/docs/resources/aws_s3_mount.md b/docs/resources/aws_s3_mount.md new file mode 100644 index 0000000000..2d9440943c --- /dev/null +++ b/docs/resources/aws_s3_mount.md @@ -0,0 +1,49 @@ +# databricks_aws_s3_mount Resource + +**This resource has evolving API, which may change in future versions of provider.** + +This resource will mount your S3 bucket on `dbfs:/mnt/yourname`. It is important to understand that this will start up the cluster if the cluster is terminated. The read and refresh terraform command will require a cluster and make take some time to validate mount. If cluster_id is not specified, it will create the smallest possible cluster called `terraform-mount` for shortest possible amount of time. + +## Example Usage + +```hcl +// will create AWS S3 bucket +resource "aws_s3_bucket" "this" { + bucket = "${var.prefix}-rootbucket" + acl = "private" + versioning { + enabled = false + } +} + +// now you can do `%fs ls /mnt/experiments` in notebooks +resource "databricks_s3_mount" "this" { + s3_bucket_name = aws_s3_bucket.this.bucket + mount_name = "experiments" +} +``` + +## Argument Reference + +The following arguments are required: + +* `cluster_id` - (Optional) (String) Cluster to use for mounting. If no cluster is specified, new cluster will be created and will mount the bucket for all of the clusters in this workspace. If cluster is specified, mount will be visible for all clusters with the same [instance profile](./instance_profile.md). If cluster is not running - it's going to be started, so be aware to set autotermination rules on it. +* `mount_name` - (Required) (String) Name, under which mount will be accessible in `dbfs:/mnt/`. +* `s3_bucket_name` - (Required) (String) S3 bucket name to be mounted. + + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - mount name +* `source` - (String) HDFS-compatible S3 bucket url `s3a://` + + +## Import + +The resource aws s3 mount can be imported using it's mount name + +```bash +$ terraform import databricks_aws_s3_mount.this +``` \ No newline at end of file diff --git a/docs/resources/azure_adls_gen1_mount.md b/docs/resources/azure_adls_gen1_mount.md new file mode 100644 index 0000000000..3232999d95 --- /dev/null +++ b/docs/resources/azure_adls_gen1_mount.md @@ -0,0 +1,72 @@ +# databricks_azure_adls_gen1_mount Resource + +**This resource has evolving API, which may change in future versions of provider.** + +This resource will mount your ADLS v1 bucket on `dbfs:/mnt/yourname`. It is important to understand that this will start up the cluster if the cluster is terminated. The read and refresh terraform command will require a cluster and make take some time to validate mount. If cluster_id is not specified, it will create the smallest possible cluster called `terraform-mount` for shortest possible amount of time. + + +## Example Usage + +```hcl +resource "azurerm_data_lake_store" "gen1" { + name = "${local.prefix}gen1" + resource_group_name = var.resource_group_name + location = var.resource_group_location +} + +// azurerm_data_lake_store.gen1.name + +resource "databricks_secret_scope" "this" { + name = "application" + initial_manage_principal = "users" +} + +resource "databricks_secret" "service_principal_key" { + key = "service_principal_key" + string_value = "{env.TEST_STORAGE_ACCOUNT_KEY}" + scope = databricks_secret_scope.terraform.name +} + +resource "databricks_azure_adls_gen1_mount" "mount" { + container_name = "dev" + storage_account_name = "{env.TEST_STORAGE_ACCOUNT_NAME}" + mount_name = "{var.RANDOM}" + auth_type = "ACCESS_KEY" + token_secret_scope = databricks_secret_scope.terraform.name + token_secret_key = databricks_secret.service_principal_key.key +} + +``` + +## Argument Reference + +The following arguments are required: + +* `client_id` - (Required) (String) This is the client_id for the enterprise application for the service principal. +* `tenant_id` - (Required) (String) This is your azure directory tenant id. This is required for creating the mount. +* `client_secret_key` - (Required) (String) This is the secret key in which your service principal/enterprise app client secret will be stored. +* `client_secret_scope` - (Required) (String) This is the secret scope in which your service principal/enterprise app client secret will be stored. + +* `cluster_id` - (Optional) (String) Cluster to use for mounting. If no cluster is specified, new cluster will be created and will mount the bucket for all of the clusters in this workspace. If cluster is not running - it's going to be started, so be aware to set autotermination rules on it. +* `mount_name` - (Required) (String) Name, under which mount will be accessible in `dbfs:/mnt/`. +* `storage_resource_name` - (Required) (String) The name of the storage resource in which the data is for ADLS gen 1. This is what you are trying to mount. +* `spark_conf_prefix` - (Optional) (String) This is the spark configuration prefix for adls gen 1 mount. The options are `fs.adl`, `dfs.adls`. Use `fs.adl` for runtime 6.0 and above for the clusters. Otherwise use `dfs.adls`. The default value is: `fs.adl`. +* `directory` - (Computed) (String) This is optional if you want to add an additional directory that you wish to mount. This must start with a "/". + + + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - mount name +* `source` - (String) HDFS-compatible url `adl://` + + +## Import + +The resource can be imported using it's mount name + +```bash +$ terraform import databricks_azure_adls_gen1_mount.this +``` \ No newline at end of file diff --git a/docs/resources/azure_adls_gen2_mount.md b/docs/resources/azure_adls_gen2_mount.md new file mode 100644 index 0000000000..5b576e3257 --- /dev/null +++ b/docs/resources/azure_adls_gen2_mount.md @@ -0,0 +1,89 @@ +# databricks_azure_adls_gen2_mount Resource + +**This resource has evolving API, which may change in future versions of provider.** + +This resource will mount your ADLS v2 bucket on `dbfs:/mnt/yourname`. It is important to understand that this will start up the cluster if the cluster is terminated. The read and refresh terraform command will require a cluster and make take some time to validate mount. If cluster_id is not specified, it will create the smallest possible cluster called `terraform-mount` for shortest possible amount of time. + +## Example Usage + +```hcl +resource "databricks_secret_scope" "terraform" { + name = "application" + initial_manage_principal = "users" +} + +resource "databricks_secret" "service_principal_key" { + key = "service_principal_key" + string_value = "${var.ARM_CLIENT_SECRET}" + scope = databricks_secret_scope.terraform.name +} + +data "azurerm_client_config" "current" { +} + +resource "azurerm_storage_account" "this" { + name = "${var.prefix}datalake" + resource_group_name = var.resource_group_name + location = var.resource_group_location + account_tier = "Standard" + account_replication_type = "GRS" + account_kind = "StorageV2" + is_hns_enabled = true +} + +resource "azurerm_role_assignment" "this" { + scope = azurerm_storage_account.this.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_storage_container" "this" { + name = "marketing" + storage_account_name = azurerm_storage_account.this.name + container_access_type = "private" +} + +resource "databricks_azure_adls_gen2_mount" "marketing" { + container_name = azurerm_storage_container.adlsexample.name + storage_account_name = azurerm_storage_account.this.name + mount_name = "marketing" + tenant_id = data.azurerm_client_config.current.tenant_id + client_id = data.azurerm_client_config.current.client_id + client_secret_scope = databricks_secret_scope.terraform.name + client_secret_key = databricks_secret.client_secret.key + initialize_file_system = true +} +``` + +## Argument Reference + +The following arguments are required: + +* `client_id` - (Required) (String) This is the client_id for the enterprise application for the service principal. +* `tenant_id` - (Required) (String) This is your azure directory tenant id. This is required for creating the mount. +* `client_secret_key` - (Required) (String) This is the secret key in which your service principal/enterprise app client secret will be stored. +* `client_secret_scope` - (Required) (String) This is the secret scope in which your service principal/enterprise app client secret will be stored. + +* `cluster_id` - (Optional) (String) Cluster to use for mounting. If no cluster is specified, new cluster will be created and will mount the bucket for all of the clusters in this workspace. If cluster is not running - it's going to be started, so be aware to set autotermination rules on it. + +* `container_name` - (Required) (String) ADLS gen2 container name +* `storage_account_name` - (Required) (String) The name of the storage resource in which the data is. +* `mount_name` - (Required) (String) Name, under which mount will be accessible in `dbfs:/mnt/`. +* `directory` - (Computed) (String) This is optional if you want to add an additional directory that you wish to mount. This must start with a "/". +* `initialize_file_system` - (Required) (Bool) either or not intialize FS for the first use + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - mount name +* `source` - (String) HDFS-compatible url `abfss://` + + +## Import + +The resource can be imported using it's mount name + +```bash +$ terraform import databricks_azure_adls_gen2_mount.this +``` \ No newline at end of file diff --git a/docs/resources/azure_blob_mount.md b/docs/resources/azure_blob_mount.md new file mode 100644 index 0000000000..87c3eb8368 --- /dev/null +++ b/docs/resources/azure_blob_mount.md @@ -0,0 +1,74 @@ +# databricks_azure_blob_mount Resource + +**This resource has evolving API, which may change in future versions of provider.** + +This resource will mount your Azure Blob Storage bucket on `dbfs:/mnt/yourname`. It is important to understand that this will start up the cluster if the cluster is terminated. The read and refresh terraform command will require a cluster and make take some time to validate mount. If cluster_id is not specified, it will create the smallest possible cluster called `terraform-mount` for shortest possible amount of time. This resource will help you create, get and delete a azure blob storage mount using SAS token or storage account access keys. + + +## Example Usage + +```hcl +resource "azurerm_storage_account" "blobaccount" { + name = "${var.prefix}blob" + resource_group_name = var.resource_group_name + location = var.resource_group_location + account_tier = "Standard" + account_replication_type = "LRS" + account_kind = "StorageV2" +} + +resource "azurerm_storage_container" "marketing" { + name = "marketing" + storage_account_name = azurerm_storage_account.blobaccount.name + container_access_type = "private" +} + +resource "databricks_secret_scope" "terraform" { + name = "application" + initial_manage_principal = "users" +} + +resource "databricks_secret" "storage_key" { + key = "blob_storage_key" + string_value = azurerm_storage_account.blobaccount.primary_access_key + scope = databricks_secret_scope.terraform.name +} + +resource "databricks_azure_blob_mount" "marketing" { + container_name = azurerm_storage_container.marketing.name + storage_account_name = azurerm_storage_account.blobaccount.name + mount_name = "marketing" + auth_type = "ACCESS_KEY" + token_secret_scope = databricks_secret_scope.terraform.name + token_secret_key = databricks_secret.storage_key.key +} +``` + +## Argument Reference + +The following arguments are required: + +* `auth_type` - (Required) (String) This is the auth type for blob storage. This can either be SAS tokens or account access keys. +* `token_secret_scope` - (Required) (String) This is the secret scope in which your auth type token exists in. +* `token_secret_key` - (Required) (String) This is the secret key in which your auth type token exists in. +* `container_name` - (Required) (String) The container in which the data is. This is what you are trying to mount. +* `storage_account_name` - (Required) (String) The name of the storage resource in which the data is. +* `cluster_id` - (Optional) (String) Cluster to use for mounting. If no cluster is specified, new cluster will be created and will mount the bucket for all of the clusters in this workspace. If cluster is not running - it's going to be started, so be aware to set autotermination rules on it. +* `mount_name` - (Required) (String) Name, under which mount will be accessible in `dbfs:/mnt/`. +* `directory` - (Computed) (String) This is optional if you want to add an additional directory that you wish to mount. This must start with a "/". + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - mount name +* `source` - (String) HDFS-compatible url `wasbs://` + + +## Import + +The resource can be imported using it's mount name + +```bash +$ terraform import databricks_azure_blob_mount.this +``` \ No newline at end of file diff --git a/docs/resources/cluster.md b/docs/resources/cluster.md index f28e6c4716..9462fd41a3 100644 --- a/docs/resources/cluster.md +++ b/docs/resources/cluster.md @@ -65,6 +65,118 @@ When you [create a Databricks cluster](https://docs.databricks.com/clusters/conf * `min_workers` - (Optional) The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. * `max_workers` - (Optional) The maximum number of workers to which the cluster can scale up when overloaded. max_workers must be strictly greater than min_workers. +### libraries Configuration Block + +In order to install libraries, one must specify each library in own configuration block. Each different type of library has slightly different syntax. It's possible to specify only one type of library within one config block, otherwise plan will fail with error. + +Installing JAR artifacts on a cluster. Location can be anyling, that is DBFS or mounted object store (s3, adls, ...) +```hcl +libraries { + jar = "dbfs://FileStore/app-0.0.1.jar" +} +``` + +Installing Python EGG artifacts. Location can be anyling, that is DBFS or mounted object store (s3, adls, ...) +```hcl +libraries { + egg = "dbfs://FileStore/foo.egg" +} +``` + +Installing Python Wheel artifacts. Location can be anyling, that is DBFS or mounted object store (s3, adls, ...) +```hcl +libraries { + whl = "dbfs://FileStore/baz.whl" +} +``` + +Installing Python PyPI artifacts. You can also optionally also specify `repo` parameter for custom PyPI mirror, that should be accessible without any authentication for the network, that cluster runs in. +```hcl +libraries { + pypi { + package = "fbprophet==0.6" + // repo can also be specified here + } +} +``` + +Installing artifacts from Maven repository. You can also optionally also specify `repo` parameter for custom Maven-style repository, that should be accessible without any authentication for the network, that cluster runs in. It can even be properly configured [maven s3 wagon](https://github.com/seahen/maven-s3-wagon), [AWS CodeArtifact](https://aws.amazon.com/codeartifact/) or [Azure Artifacts](https://azure.microsoft.com/en-us/services/devops/artifacts/). +```hcl +libraries { + maven { + coordinates = "com.amazon.deequ:deequ:1.0.4" + // exlusions block is optional + exclusions = ["org.apache.avro:avro"] + } +} +``` + +Installing artifacts from CRan. You can also optionally also specify `repo` parameter for custom cran mirror. +```hcl +libraries { + cran { + package = "rkeops" + } +} +``` + +## cluster_log_conf + +Example of pushing all cluster logs to DBFS: +```hcl +cluster_log_conf { + dbfs { + destination = "dbfs://cluster-logs" + } +} +``` + +Example of pushing all cluster logs to S3: +```hcl +cluster_log_conf { + s3 { + destination = "s3a://acmecorp-main/cluster-logs" + region = "us-east-1" + } +} +``` + +There are few more advanced attributes for S3 log delivery: + +* `destination` - S3 destination, e.g. `s3://my-bucket/some-prefix` You must configure the cluster with an instance profile and the instance profile must have write access to the destination. You cannot use AWS keys. +* `region` - (Optional) S3 region, e.g. `us-west-2`. Either region or endpoint must be set. If both are set, endpoint is used. +* `endpoint` - (Optional) S3 endpoint, e.g. https://s3-us-west-2.amazonaws.com. Either region or endpoint needs to be set. If both are set, endpoint is used. +* `enable_encryption` - (Optional) Enable server side encryption, false by default. +* `encryption_type` - (Optional) The encryption type, it could be sse-s3 or sse-kms. It is used only when encryption is enabled and the default type is sse-s3. +* `kms_key` - (Optional) KMS key used if encryption is enabled and encryption type is set to sse-kms. +* `canned_acl` - (Optional) Set canned access control list, e.g. bucket-owner-full-control. If canned_cal is set, the cluster instance profile must have s3:PutObjectAcl permission on the destination bucket and prefix. The full list of possible canned ACL can be found [here](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl). By default only the object owner gets full control. If you are using cross account role for writing data, you may + want to set bucket-owner-full-control to make bucket owner able to read the logs. + +## init_scripts + +You can specify up to 10 different init scripts for cluster. + +Example of taking init script from DBFS: +```hcl +init_scripts { + dbfs { + destination = "dbfs://init-scripts/install-elk.sh" + } +} +``` + +Example of taking init script from S3: +```hcl +init_scripts { + s3 { + destination = "s3a://acmecorp-main/init-scripts/install-elk.sh" + region = "us-east-1" + } +} +``` + +Attributes are the same as for `cluster_log_conf` configuration block. + ## aws_attributes `aws_attributes` optional configuration block contains attributes related to [clusters running on Amazon Web Services](https://docs.databricks.com/clusters/configure.html#aws-configurations). If not specified at cluster creation, a set of default values will be used. It is advised to keep all common configurations in [Cluster Policies](cluster_policy.md) to maintain control of the environments launched. @@ -105,12 +217,15 @@ The following options are available: In addition to all arguments above, the following attributes are exported: -* `id` - The id for the cluster object. -* `cluster_id` - Canonical identifier for the cluster. -* `default_tags` - Tags that are added by Databricks by default, regardless of any custom_tags that may have been added. These include: Vendor: Databricks, Creator: , ClusterName: , ClusterId: , Name: -* `state` - State of the cluster. -* `state_message` - A message associated with the most recent state transition (e.g., the reason why the cluster entered a TERMINATED state). This field is unstructured, and its exact format is subject to change. +* `id` - Canonical unique identifier for the cluster. +* `default_tags` - (map) Tags that are added by Databricks by default, regardless of any custom_tags that may have been added. These include: Vendor: Databricks, Creator: , ClusterName: , ClusterId: , Name: +* `state` - (string) State of the cluster. + ## Import -Importing this resource is not currently supported. \ No newline at end of file +The resource cluster can be imported using cluster id + +```bash +$ terraform import databricks_cluster.this +``` \ No newline at end of file diff --git a/docs/resources/cluster_policy.md b/docs/resources/cluster_policy.md index b41b079087..d4ee08162c 100644 --- a/docs/resources/cluster_policy.md +++ b/docs/resources/cluster_policy.md @@ -52,7 +52,6 @@ resource "databricks_cluster_policy" "external_metastore" { The following arguments are required: * `name` - (Required) Cluster policy name. This must be unique. Length must be between 1 and 100 characters. - * `definition` - (Required) Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policies.html#cluster-policy-definition). ## Attribute Reference @@ -60,9 +59,12 @@ The following arguments are required: In addition to all arguments above, the following attributes are exported: * `id` - Canonical unique identifier for the cluster policy. This equal to policy_id. - * `policy_id` - Canonical unique identifier for the cluster policy. ## Import --> **Note** Importing this resource is not currently supported. +The resource cluster policy can be imported using the policy id: + +```bash +$ terraform import databricks_cluster_policy.this +``` \ No newline at end of file diff --git a/docs/resources/dbfs_file.md b/docs/resources/dbfs_file.md index 2dc8afc3eb..4027caa1f9 100644 --- a/docs/resources/dbfs_file.md +++ b/docs/resources/dbfs_file.md @@ -36,17 +36,11 @@ resource "databricks_dbfs_file" "my_dbfs_file" { The following arguments are supported: * `content` - (Optional) The content of the file as a base64 encoded string. - * `source` - (Optional) The full absolute path to the file. Please use [pathexpand](https://www.terraform.io/docs/configuration/functions/pathexpand.html). - * `content_b64_md5` - (Required) The checksum for the content please use the [md5](https://www.terraform.io/docs/configuration/functions/md5.html) and [filebase64](https://www.terraform.io/docs/configuration/functions/filebase64.html) functions in terraform to retrieve the checksum. - * `path` - (Required) The path of the file in which you wish to save. - * `overwrite` - (Optional) This is used to determine whether it should delete the existing file when with the same name when it writes. The default is set to false. - * `mkdirs` - (Optional) When the resource is created, this field is used to determine if it needs to make the parent directories. The default value is set to true. - * `validate_remote_file` - (Optional) This is used to compare the actual contents of the file to determine if the remote file is valid or not. If the base64 content is different it will attempt to do a delete, create. @@ -56,10 +50,13 @@ it will attempt to do a delete, create. In addition to all arguments above, the following attributes are exported: * `id` - The id for the dbfs file object. - * `file_size` - The file size of the file that is being tracked by this resource in bytes. ## Import --> **Note** Importing this resource is not currently supported. +The resource dbfs file can be imported using the `object`, e.g. + +```bash +$ terraform import databricks_dbfs_file.object +``` \ No newline at end of file diff --git a/docs/resources/group_instance_profile.md b/docs/resources/group_instance_profile.md index 2727f2bcbe..4ef6d8d7ed 100644 --- a/docs/resources/group_instance_profile.md +++ b/docs/resources/group_instance_profile.md @@ -1,5 +1,7 @@ # databricks_group_instance_profile Resource +**This resource has evolving API, which may change in future versions of provider.** + This resource allows you to attach instance profiles to groups created by the `databricks_group` resource. -> **Note** Please only use this resource in conjunction with the `databricks_group` resource and **not** the `databricks_scim_group` resource. diff --git a/docs/resources/instance_pool.md b/docs/resources/instance_pool.md new file mode 100644 index 0000000000..44f3c1d6cd --- /dev/null +++ b/docs/resources/instance_pool.md @@ -0,0 +1,82 @@ +# databricks_instance_pool Resource + +This resource allows you to manage instance pools on Databricks. An instance pool reduces cluster start and auto-scaling times by maintaining a set of idle, ready-to-use cloud instances. When a cluster attached to a pool needs an instance, it first attempts to allocate one of the pool’s idle instances. If the pool has no idle instances, it expands by allocating a new instance from the instance provider in order to accommodate the cluster’s request. When a cluster releases an instance, it returns to the pool and is free for another cluster to use. Only clusters attached to a pool can use that pool’s idle instances. + +**It is important to know what that different cloud service providers have different `node_type_id`, `disk_specs` and potentially other configurations. ** + +## Example Usage + +```hcl +resource "databricks_instance_pool" "my-pool" { + instance_pool_name = "reserved-i3.xlarge-pool" + min_idle_instances = 0 + max_capacity = 300 + node_type_id = "i3.xlarge" + aws_attributes { + availability = "ON_DEMAND" + zone_id = "us-east-1a" + spot_bid_price_percent = "100" + } + idle_instance_autotermination_minutes = 10 + disk_spec { + ebs_volume_type = "GENERAL_PURPOSE_SSD" + disk_size = 80 + disk_count = 1 + } +} +``` + +## Argument Reference + +The following arguments are required: + +* `instance_pool_name` - (Required) (String) The name of the instance pool. This is required for create and edit operations. It must be unique, non-empty, and less than 100 characters. +* `min_idle_instances` - (Required) (Integer) The minimum number of idle instances maintained by the pool. This is in addition to any instances in use by active clusters. +* `max_capacity` - (Required) (Integer) The maximum number of instances the pool can contain, including both idle instances and ones in use by clusters. Once the maximum capacity is reached, you cannot create new clusters from the pool and existing clusters cannot autoscale up until some instances are made idle in the pool via cluster termination or down-scaling. +* `idle_instance_autotermination_minutes` - (Required) (Integer) The number of minutes that idle instances in excess of the min_idle_instances are maintained by the pool before being terminated. If not specified, excess idle instances are terminated automatically after a default timeout period. If specified, the time must be between 0 and 10000 minutes. If you specify 0, excess idle instances are removed as soon as possible. +* `node_type_id` - (Required) (String) The node type for the instances in the pool. All clusters attached to the pool inherit this node type and the pool’s idle instances are allocated based on this type. You can retrieve a list of available node types by using the [List Node Types API](https://docs.databricks.com/dev-tools/api/latest/clusters.html#clusterclusterservicelistnodetypes) call. +* `custom_tags` - (Optional) (Map) Additional tags for instance pool resources. Databricks tags all pool resources (e.g. AWS & Azure instances and Disk volumes) with these tags in addition to default_tags. **Databricks allows at most 43 custom tags.** +* `enable_elastic_disk` - (Optional) (Bool) Autoscaling Local Storage: when enabled, the instances in the pool dynamically acquire additional disk space when they are running low on disk space. + +* `preloaded_spark_versions` - (Optional) (List) A list with the runtime version the pool installs on each instance. Pool clusters that use a preloaded runtime version start faster as they do have to wait for the image to download. You can retrieve a list of available runtime versions by using the [Runtime Versions API](https://docs.databricks.com/dev-tools/api/latest/clusters.html#clusterclusterservicelistsparkversions) call. + +### aws_attributes Configuration Block + +* `spot_bid_price_percent` - (Optional) (Integer) The max price for AWS spot instances, as a percentage of the corresponding instance type’s on-demand price. For example, if this field is set to 50, and the instance pool needs a new i3.xlarge spot instance, then the max price is half of the price of on-demand i3.xlarge instances. Similarly, if this field is set to 200, the max price is twice the price of on-demand i3.xlarge instances. If not specified, the **default value is 100**. When spot instances are requested for this instance pool, only spot instances whose max price percentage matches this field are considered. *For safety, this field cannot be greater than 10000.* +* `availability` - (Optional) (String) Availability type used for all instances in the pool. Only `ON_DEMAND` and `SPOT` are supported. +* `zone_id` - (Required) (String) Identifier for the availability zone/datacenter in which the instance pool resides. This string is of a form like `"us-west-2a"`. The provided availability zone must be in the same region as the Databricks deployment. For example, `"us-west-2a"` is not a valid zone ID if the Databricks deployment resides in the `"us-east-1"` region. This is an optional field. If not specified, a default zone is used. You can find the list of available zones as well as the default value by using the [List Zones API](https://docs.databricks.com/dev-tools/api/latest/clusters.html#clusterclusterservicelistavailablezones). + + +### disk_spec Configuration Block + +For disk_spec make sure to use **ebs_volume_type** only on AWS deployment of Databricks and **azure_disk_volume_type** only on a Azure deployment of Databricks. + +* `ebs_volume_type` - (Optional) (String) The EBS volume type to use. Options are: `GENERAL_PURPOSE_SSD` (Provision extra storage using AWS gp2 EBS volumes) or `THROUGHPUT_OPTIMIZED_HDD` (Provision extra storage using AWS st1 volumes). +* `azure_disk_volume_type` - (Optional) (String) The type of Azure disk to use. Options are: `PREMIUM_LRS` (Premium storage tier, backed by SSDs) or `"STANDARD_LRS"` (Standard storage tier, backed by HDDs). +* `disk_count` - (Optional) (Integer) The number of disks to attach to each instance. This feature is only enabled for supported node types. Users can choose up to the limit of the disks supported by the node type. For node types with no local disk, at least one disk needs to be specified. +* `disk_size` - (Optional) (Integer) The size of each disk (in GiB) to attach. Values must fall into the supported range +for a particular instance type: + +**AWS (ebs)**: + * General Purpose SSD: `100 - 4096` GiB + * Throughput Optimized HDD: `500 - 4096` GiB + +**Azure (disk volume)**: + * Premium LRS (SSD): `1 - 1023` GiB + * Standard LRS (HDD): `1- 1023` GiB + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the instance pool. +* `default_tags` - default tag map. + + +## Import + +The resource instance pool can be imported using it's id: + +```bash +$ terraform import databricks_instance_pool.this +``` \ No newline at end of file diff --git a/docs/resources/instance_profile.md b/docs/resources/instance_profile.md index a8ffd33930..eb05dccbe9 100644 --- a/docs/resources/instance_profile.md +++ b/docs/resources/instance_profile.md @@ -96,4 +96,8 @@ In addition to all arguments above, the following attributes are exported: ## Import -Importing this resource is not currently supported. \ No newline at end of file +The resource instance profile can be imported using the ARN of it + +```bash +$ terraform import databricks_instance_profile.this +``` \ No newline at end of file diff --git a/docs/resources/job.md b/docs/resources/job.md new file mode 100644 index 0000000000..c453b38592 --- /dev/null +++ b/docs/resources/job.md @@ -0,0 +1,91 @@ +# databricks_job Resource + +The databricks_job resource allows you to create, edit, and delete jobs, which run on either new or existing [clusters](cluster.md). + +## Example Usage + +```hcl +resource "databricks_job" "this" { + name = "Featurization" + timeout_seconds = 3600 + max_retries = 1 + max_concurrent_runs = 1 + + new_cluster { + num_workers = 300 + spark_version = "6.6.x-scala2.11 + node_type_id = "i3.xlarge" + } + + notebook_task { + notebook_path = "/Production/MakeFeatures" + } + + libraries { + pypi { + package = "fbprophet==0.6" + } + } + + email_notifications { + no_alert_for_skipped_runs = true + } +} +``` + +## Argument Reference + +The following arguments are required: + +* `name` - (Optional) (String) An optional name for the job. The default value is Untitled. +* `new_cluster` - (Optional) (List) Same set of parameters as for [databricks_cluster](cluster.md) resource. This field is a block and is documented below. +* `existing_cluster_id` - (Optional) (String) If existing_cluster_id, the ID of an existing cluster that will be used for all runs of this job. When running jobs on an existing cluster, you may need to manually restart the cluster if it stops responding. We strongly suggest to use `new_cluster` for greater reliability. +* `libraries` - (Optional) (Set) An optional list of libraries to be installed on the cluster that will execute the job. Please consult [libraries section](cluster.md#libraries) for [databricks_cluster](cluster.md) resource. +* `retry_on_timeout` - (Optional) (Bool) An optional policy to specify whether to retry a job when it times out. The default behavior is to not retry on timeout. +* `max_retries` - (Optional) (Integer) An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with a FAILED result_state or INTERNAL_ERROR life_cycle_state. The value -1 means to retry indefinitely and the value 0 means to never retry. The default behavior is to never retry. +* `timeout_seconds` - (Optional) (Integer) An optional timeout applied to each run of this job. The default behavior is to have no timeout. +* `min_retry_interval_millis` - (Optional) (Integer) An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried. +* `max_concurrent_runs` - (Optional) (Integer) An optional maximum allowed number of concurrent runs of the job. +* `email_notifications` - (Optional) (List) An optional set of email addresses notified when runs of this job begin and complete and when this job is deleted. The default behavior is to not send any emails. This field is a block and is documented below. +* `schedule` - (Optional) (List) An optional periodic schedule for this job. The default behavior is that the job runs when triggered by clicking Run Now in the Jobs UI or sending an API request to runNow. This field is a block and is documented below. + +### schedule Configuration Block + +* `quartz_cron_expression` - (Required) (String) A Cron expression using Quartz syntax that describes the schedule for a job. This field is required. +* `timezone_id` - (Required) (String) A Java timezone ID. The schedule for a job will be resolved with respect to this timezone. See Java TimeZone for details. This field is required. + +### spark_jar_task Configuration Block + +* `parameters` - (Optional) (List) Parameters passed to the main method. +* `main_class_name` - (Optional) (String) The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library. The code should use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job will fail. + +### spark_submit_task Configuration Block + +You can invoke Spark submit tasks only on new clusters. In the new_cluster specification, libraries and spark_conf are not supported. Instead, use --jars and --py-files to add Java and Python libraries and --conf to set the Spark configuration. By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set --driver-memory, and --executor-memory to a smaller value to leave some room for off-heap usage. **Please use `spark_jar_task`, `spark_python_task` or `notebook_task` wherever possible**. + +* `parameters` - (Optional) (List) Command-line parameters passed to spark submit. + +### spark_python_task Configuration Block + +* `python_file` - (Required) (String) The URI of the Python file to be executed. DBFS and S3 paths are supported. This field is required. +* `parameters` - (Optional) (List) Command line parameters passed to the Python file. + +### notebook_task Configuration Block + +* `base_parameters` - (Optional) (Map) Base parameters to be used for each run of this job. If the run is initiated by a call to run-now with parameters specified, the two parameters maps will be merged. If the same key is specified in base_parameters and in run-now, the value from run-now will be used. If the notebook takes a parameter that is not specified in the job’s base_parameters or the run-now override parameters, the default value from the notebook will be used. Retrieve these parameters in a notebook using `dbutils.widgets.get`. +* `notebook_path` - (Required) (String) The absolute path of the notebook to be run in the Databricks workspace. This path must begin with a slash. This field is required. + +### email_notifications Configuration Block + +* `on_failure` - (Optional) (List) list of emails to notify on failure +* `no_alert_for_skipped_runs` - (Optional) (Bool) don't send alert for skipped runs +* `on_start` - (Optional) (List) list of emails to notify on failure +* `on_success` - (Optional) (List) list of emails to notify on failure + +## Import + +The resource job can be imported using the id of the job + +```bash +$ terraform import databricks_job.this +``` \ No newline at end of file diff --git a/docs/resources/mws_credentials.md b/docs/resources/mws_credentials.md new file mode 100644 index 0000000000..8a2b1ef93d --- /dev/null +++ b/docs/resources/mws_credentials.md @@ -0,0 +1,61 @@ +# databricks_mws_credentials Resource + +This resource to configure the cross-account role for creation of new workspaces within AWS. + +It is important to understand that this will require you to configure your provider separately for the multiple workspaces resources. This will point to https://accounts.cloud.databricks.com for the HOST and it will use basic auth as that is the only authentication method available for multiple workspaces api. + +Please follow this [complete runnable example](https://github.com/databrickslabs/terraform-provider-databricks/blob/master/scripts/awsmt-integration/main.tf) with new VPC and new workspace setup. Please pay special attention to the fact that there you have two different instances of a databricks provider - one for deploying workspaces (with host=https://accounts.cloud.databricks.com/) and another for the workspace you've created with databricks_mws_workspaces resource. If you want both creation of workspaces & clusters within workspace within the same terraform module (essentially same directory), you should use the provider aliasing feature of Terraform. We strongly recommend having one terraform module for creation of workspace + PAT token and the rest in different modules. + +## Example Usage + +**This resource has evolving API, which may change in future versions of provider.** + +```hcl +resource "aws_iam_role" "cross_account_role" { + name = "${var.prefix}-crossaccount" + assume_role_policy = data.template_file.cross_account_role_assume_policy.rendered + tags = var.tags +} + +resource "aws_iam_policy" "cross_account_role_policy" { + name = "${var.prefix}-policy" + description = "E2 Workspace Cross account role policy policy" + policy = data.template_file.cross_account_role_policy.rendered +} + +resource "aws_iam_role_policy_attachment" "cross_account_role_policy_attach" { + role = aws_iam_role.cross_account_role.name + policy_arn = aws_iam_policy.cross_account_role_policy.arn +} + +provider "databricks" { + alias = "mws" + host = "https://accounts.cloud.databricks.com" +} + +// register cross-account ARN +resource "databricks_mws_credentials" "this" { + provider = databricks.mws + account_id = var.account_id + credentials_name = "${var.prefix}-creds" + role_arn = aws_iam_role.cross_account_role.arn +} +``` + +## Argument Reference + +The following arguments are required: + +* `account_id` - (Required) (String) master account id (also used for `sts:ExternalId` of `sts:AssumeRole`) +* `credentials_name` - (Required) (String) name of credentials to register +* `role_arn` - (Required) (String) ARN of cross-account role + + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the mws credentials. +* `creation_time` - (Integer) time of credentials registration +* `external_id` - (String) master account id +* `credentials_id` - (String) identifier of credentials \ No newline at end of file diff --git a/docs/resources/mws_networks.md b/docs/resources/mws_networks.md new file mode 100644 index 0000000000..4670aaf63a --- /dev/null +++ b/docs/resources/mws_networks.md @@ -0,0 +1,108 @@ +# databricks_mws_networks Resource + +**This resource has evolving API, which may change in future versions of provider.** + +This resource to configure VPC & subnets for new workspaces within AWS. + +It is important to understand that this will require you to configure your provider separately for the multiple workspaces resources. This will point to https://accounts.cloud.databricks.com for the HOST and it will use basic auth as that is the only authentication method available for multiple workspaces api. + +Please follow this [complete runnable example](https://github.com/databrickslabs/terraform-provider-databricks/blob/master/scripts/awsmt-integration/main.tf) with new VPC and new workspace setup. Please pay special attention to the fact that there you have two different instances of a databricks provider - one for deploying workspaces (with host=https://accounts.cloud.databricks.com/) and another for the workspace you've created with databricks_mws_workspaces resource. If you want both creation of workspaces & clusters within workspace within the same terraform module (essentially same directory), you should use the provider aliasing feature of Terraform. We strongly recommend having one terraform module for creation of workspace + PAT token and the rest in different modules. + +## Example Usage + +```hcl +resource "aws_vpc" "main" { + cidr_block = data.external.env.result.TEST_CIDR + enable_dns_hostnames = true + + tags = merge(var.tags, { + Name = "${var.prefix}-vpc" + }) +} + +resource "aws_subnet" "public" { + vpc_id = aws_vpc.main.id + cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 3, 0) + availability_zone = "${data.external.env.result.TEST_REGION}b" + + tags = merge(var.tags, { + Name = "${var.prefix}-public-sn" + }) +} + +resource "aws_subnet" "private" { + vpc_id = aws_vpc.main.id + cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 3, 1) + availability_zone = "${data.external.env.result.TEST_REGION}a" + + tags = merge(var.tags, { + Name = "${var.prefix}-private-sn" + }) +} + +resource "aws_internet_gateway" "gw" { + vpc_id = aws_vpc.main.id + tags = merge(var.tags, { + Name = "${var.prefix}-igw" + }) +} + +resource "aws_route" "r" { + route_table_id = aws_vpc.main.default_route_table_id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.gw.id +} + +resource "aws_security_group" "test_sg" { + name = "all all" + description = "Allow inbound traffic" + vpc_id = aws_vpc.main.id + + ingress { + description = "All" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = merge(var.tags, { + Name = "${var.prefix}-sg" + }) +} + +resource "databricks_mws_networks" "this" { + provider = databricks.mws + account_id = var.account_id + network_name = "${var.prefix}-network" + vpc_id = aws_vpc.main.id + subnet_ids = [aws_subnet.public.id, aws_subnet.private.id] + security_group_ids = [aws_security_group.test_sg.id] +} +``` + +## Argument Reference + +The following arguments are required: + +* `account_id` - (Required) (String) master account id (also used for `sts:ExternaId` of `sts:AssumeRole`) +* `network_name` - (Required) (String) name under which this network is regisstered +* `vpc_id` - (Required) (String) AWS VPC id +* `subnet_ids` - (Required) (Set) ids of AWS VPC subnets +* `security_group_ids` - (Required) (Set) ids of AWS Security Groups + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the mws networks. +* `network_id` - (String) id of network to be used for `databricks_mws_workspace` resource. +* `vpc_status` - (String) VPC attachment status +* `workspace_id` - (Integer) id of associated workspace diff --git a/docs/resources/mws_storage_configurations.md b/docs/resources/mws_storage_configurations.md new file mode 100644 index 0000000000..9806de7b32 --- /dev/null +++ b/docs/resources/mws_storage_configurations.md @@ -0,0 +1,43 @@ +# databricks_mws_storage_configurations Resource + +**This resource has evolving API, which may change in future versions of provider.** + +This resource to configure root bucket new workspaces within AWS. + +It is important to understand that this will require you to configure your provider separately for the multiple workspaces resources. This will point to https://accounts.cloud.databricks.com for the HOST and it will use basic auth as that is the only authentication method available for multiple workspaces api. + +Please follow this [complete runnable example](https://github.com/databrickslabs/terraform-provider-databricks/blob/master/scripts/awsmt-integration/main.tf) with new VPC and new workspace setup. Please pay special attention to the fact that there you have two different instances of a databricks provider - one for deploying workspaces (with host=https://accounts.cloud.databricks.com/) and another for the workspace you've created with databricks_mws_workspaces resource. If you want both creation of workspaces & clusters within workspace within the same terraform module (essentially same directory), you should use the provider aliasing feature of Terraform. We strongly recommend having one terraform module for creation of workspace + PAT token and the rest in different modules. + +## Example Usage + +```hcl +resource "aws_s3_bucket" "root_storage_bucket" { + bucket = "${var.prefix}-rootbucket" + acl = "private" + versioning { + enabled = false + } +} + +resource "databricks_mws_storage_configurations" "this" { + provider = databricks.mws + account_id = var.account_id + storage_configuration_name = "${var.prefix}-storage" + bucket_name = aws_s3_bucket.root_storage_bucket.bucket +} +``` + +## Argument Reference + +The following arguments are required: + +* `bucket_name` - (Required) (String) name of AWS S3 bucket +* `account_id` - (Required) (String) master account id (also used for `sts:ExternaId` of `sts:AssumeRole`) +* `storage_configuration_name` - (Required) (String) name under which this storage configuration is stored + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the mws storage configurations. +* `storage_configuration_id` - (String) id of storage config to be used for `databricks_mws_workspace` resource. diff --git a/docs/resources/mws_workspaces.md b/docs/resources/mws_workspaces.md new file mode 100644 index 0000000000..5bede461ed --- /dev/null +++ b/docs/resources/mws_workspaces.md @@ -0,0 +1,98 @@ +# databricks_mws_workspaces Resource + +**This resource has evolving API, which may change in future versions of provider.** + +This resource to configure new workspaces within AWS. + +It is important to understand that this will require you to configure your provider separately for the multiple workspaces resources. This will point to https://accounts.cloud.databricks.com for the HOST and it will use basic auth as that is the only authentication method available for multiple workspaces api. + +Please follow this [complete runnable example](https://github.com/databrickslabs/terraform-provider-databricks/blob/master/scripts/awsmt-integration/main.tf) with new VPC and new workspace setup. Please pay special attention to the fact that there you have two different instances of a databricks provider - one for deploying workspaces (with host=https://accounts.cloud.databricks.com/) and another for the workspace you've created with databricks_mws_workspaces resource. If you want both creation of workspaces & clusters within workspace within the same terraform module (essentially same directory), you should use the provider aliasing feature of Terraform. We strongly recommend having one terraform module for creation of workspace + PAT token and the rest in different modules. + +## Example Usage + +```hcl +provider "databricks" { + alias = "mws" + host = "https://accounts.cloud.databricks.com" +} + +// register cross-account ARN +resource "databricks_mws_credentials" "this" { + provider = databricks.mws + account_id = var.account_id + credentials_name = "${var.prefix}-creds" + role_arn = var.crossaccount_arn +} + +// register root bucket +resource "databricks_mws_storage_configurations" "this" { + provider = databricks.mws + account_id = var.account_id + storage_configuration_name = "${var.prefix}-storage" + bucket_name = var.root_bucket +} + +// register VPC +resource "databricks_mws_networks" "this" { + provider = databricks.mws + account_id = var.account_id + network_name = "${var.prefix}-network" + vpc_id = var.vpc_id + subnet_ids = [var.subnet_public, var.subnet_private] + security_group_ids = [var.security_group] +} + +// create workspace in given VPC with DBFS on root bucket +resource "databricks_mws_workspaces" "this" { + provider = databricks.mws + account_id = var.account_id + workspace_name = var.prefix + deployment_name = var.prefix + aws_region = var.region + + credentials_id = databricks_mws_credentials.this.credentials_id + storage_configuration_id = databricks_mws_storage_configurations.this.storage_configuration_id + network_id = databricks_mws_networks.this.network_id + verify_workspace_runnning = true +} + +provider "databricks" { + // in normal scenario you won't have to give providers aliases + alias = "created_workspace" + + host = databricks_mws_workspaces.this.workspace_url +} + +// create PAT token to provision entities within workspace +resource "databricks_token" "pat" { + provider = databricks.created_workspace + comment = "Terraform Provisioning" + // 1 day token + lifetime_seconds = 86400 +} +``` + +## Argument Reference + +The following arguments are required: + +* `network_id` - (Optional) (String) `network_id` from [networks](mws_networks.md) +* `is_no_public_ip_enabled` - (Optional) (Bool) disable Public IP addresses for nodes within cluster +* `account_id` - (Required) (String) master account id (also used for `sts:ExternaId` of `sts:AssumeRole`) +* `credentials_id` - (Required) (String) `credentials_id` from [credentials](mws_credentials.md) +* `deployment_name` - (Required) (String) part of URL: `https://.cloud.databricks.com` +* `workspace_name` - (Required) (String) name of the workspace, will appear on UI +* `aws_region` - (Required) (String) AWS region of VPC +* `storage_configuration_id` - (Required) (String) `storage_configuration_id` from [storage configuration](mws_storage_configurations.md) +* `verify_workspace_runnning` - (Required) (Bool) wait until workspace is running. + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the workspace. +* `workspace_status_message` - (String) updates on workspace status +* `workspace_status` - (String) workspace status +* `creation_time` - (Integer) time when workspace was created +* `workspace_url` - (String) URL of the workspace +* `workspace_id` - (Integer) same as `id` diff --git a/docs/resources/notebook.md b/docs/resources/notebook.md index ffbc43aae4..02c9ffa4fa 100644 --- a/docs/resources/notebook.md +++ b/docs/resources/notebook.md @@ -1,5 +1,7 @@ # databricks_notebook Resource +**This resource has evolving API, which may change in future versions of provider.** + This resource allows you to manage the import, export, and delete notebooks. The maximum allowed size of a request to resource is 10MB. @@ -27,36 +29,25 @@ resource "databricks_notebook" "notebook" { The following arguments are supported: -* `content` - (Required) The base64-encoded content. If the limit (10MB) is exceeded, -exception with error code MAX_NOTEBOOK_SIZE_EXCEEDED will be thrown. - -* `path` - (Required) The absolute path of the notebook or directory, beginning with "/", e.g. "/mynotebook". -This field is **required**. - -* `language` - (Required) The language. If format is set to SOURCE, -this field is required; otherwise, it will be ignored. Possible choices are SCALA, PYTHON, SQL, R. - -* `overwrite` - (Required) The flag that specifies whether to overwrite existing object. -It is false by default. - -* `mkdirs` - (Required) Create the given directory and necessary parent directories -if they do not exists. If there exists an object (not a directory) at any prefix of the input path, this call -returns an error RESOURCE_ALREADY_EXISTS. If this operation fails it may have succeeded in creating some of the necessary parent directories. - -* `format` - (Required) This specifies the format of the file to be imported. -This resource currently only supports SOURCE. The value is case sensitive. SOURCE is suitable for .scala, .py, .r, .sql extension based files, HTML for .html files, JUPYTER for .ipynb files. -Though the API supports DBC, HTML, and JUPYTER currently we do not support them as effectively identifying DIFF is currently not feasible. +* `content` - (Required) The base64-encoded content. If the limit (10MB) is exceeded, exception with error code MAX_NOTEBOOK_SIZE_EXCEEDED will be thrown. +* `path` - (Required) The absolute path of the notebook or directory, beginning with "/", e.g. "/mynotebook". This field is **required**. +* `language` - (Required) The language. If format is set to SOURCE, this field is required; otherwise, it will be ignored. Possible choices are SCALA, PYTHON, SQL, R. +* `overwrite` - (Required) The flag that specifies whether to overwrite existing object. It is false by default. +* `mkdirs` - (Required) Create the given directory and necessary parent directories if they do not exists. If there exists an object (not a directory) at any prefix of the input path, this call returns an error RESOURCE_ALREADY_EXISTS. If this operation fails it may have succeeded in creating some of the necessary parent directories. +* `format` - (Required) This specifies the format of the file to be imported. This resource currently only supports SOURCE. The value is case sensitive. SOURCE is suitable for .scala, .py, .r, .sql extension based files, HTML for .html files, JUPYTER for .ipynb files. Though the API supports DBC, HTML, and JUPYTER currently we do not support them as effectively identifying DIFF is currently not feasible. ## Attribute Reference In addition to all arguments above, the following attributes are exported: * `id` - The id for the notebook object. - * `object_id` - Unique identifier for a NOTEBOOK or DIRECTORY. - * `object_type` - The type of the object. It could be NOTEBOOK, DIRECTORY or LIBRARY. ## Import --> **Note** Importing this resource is not currently supported. +The resource notebook can be imported using notebook id + +```bash +$ terraform import databricks_notebook.this +``` \ No newline at end of file diff --git a/docs/resources/permissions.md b/docs/resources/permissions.md new file mode 100644 index 0000000000..77502afb07 --- /dev/null +++ b/docs/resources/permissions.md @@ -0,0 +1,74 @@ +# databricks_permissions Resource + +**This resource has evolving API, which may change in future versions of provider.** + +**This feature is not available to all customers. Please contact [sales@databricks.com](mailto:sales@databricks.com) in order to enable this feature.** + +This resource allows you to generically manage permissions for other resources in Databricks workspace. + +## Example Usage + +```hcl +resource "databricks_group" "datascience" { + display_name = "Data scientists" + allow_cluster_create = false + allow_instance_pool_create = false +} + +resource "databricks_cluster_policy" "something_simple" { + name = "Some simple policy" + definition = jsonencode({ + "spark_conf.spark.hadoop.javax.jdo.option.ConnectionURL": { + "type": "forbidden" + }, + "spark_conf.spark.secondkey": { + "type": "forbidden" + } + }) +} + +resource "databricks_permissions" "grant policy usage" { + cluster_policy_id = databricks_cluster_policy.something_simple.id + + access_control { + group_name = databricks_scim_group.datascience.display_name + permission_level = "CAN_USE" + } +} +``` + +## Argument Reference + +Exactly one of the following attribues is required: + +* `cluster_id` - [cluster](cluster.md) id +* `job_id` - [job](job.md) id +* `directory_id` - [directory](notebook.md) id +* `directory_path` - path of directory +* `notebook_id` - ID of [notebook](notebook.md) within workspace +* `notebook_path` - path of notebook +* `cluster_policy_id` - [cluster policy](cluster_policy.md) id +* `instance_pool_id` - [instance pool](instance_pool.md) id + +One or more `access_control` blocks are required to actually set the permission levels: + +```hcl +access_control { + group_name = databricks_scim_group.datascience.display_name + permission_level = "CAN_USE" +} +``` + +Attributes are: + +* `permission_level` - (Required) (String) permission level according to [specific resource](https://docs.databricks.com/security/access-control/workspace-acl.html) +* `user_name` - (Optional) (String) name of the user, which should be used if group name is not used +* `group_name` - (Optional) (String) name of the group, which should be used if user name is not used. We recommend setting permissions on groups. + + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the permissions. +* `object_type` - (String) type of permissions. \ No newline at end of file diff --git a/docs/resources/scim_user.md b/docs/resources/scim_user.md new file mode 100644 index 0000000000..9b1ff8a9f1 --- /dev/null +++ b/docs/resources/scim_user.md @@ -0,0 +1,48 @@ +# databricks_scim_user Resource + +**This resource has evolving API, which may change in future versions of provider.** + +Directly creates user within databricks workspace. We're not recommending extensive use of this resource, because it's way more manageable to create few [databricks_group](group.md) instances with all related permissions to them and let Identity provider use SCIM provisioning to populate users into those groups: + +* [Azure Active Directory](https://docs.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/scim/aad) +* [Okta](https://docs.databricks.com/administration-guide/users-groups/scim/okta.html) +* [OneLogin](https://docs.databricks.com/administration-guide/users-groups/scim/onelogin.html) + +This resource allows you to create users in Databricks and give them the proper level of access, as well as remove access for users (deprovision them) when they leave your organization or no longer need access to Databricks. This resource is heavily reliant on inherited group information and the default_roles object, to determine deltas. What this means is that, even if you change the roles field, if it is inherited it will ignore the change as it is inherited by parent group. It will only detect delta when it is a net new role or a net new delete not covered by inherited roles or default roles. + + +## Example Usage + +```hcl +resource "databricks_scim_user" "admin" { + user_name = "me@example.com" + display_name = "Example user" + set_admin = true +} +``` + +## Argument Reference + +The following arguments are required: + +* `user_name` - (Required) This is the username of the given user and will be their form of access and identity. +* `display_name` - (Optional) This is an alias for the username can be the full name of the user. +* `roles` - (Optional) (Set) This is a list of roles assigned to the user, specific to the AWS environment for user to assume roles on clusters. +* `entitlements` - (Optional) (Set) Entitlements for the user to be able to have the ability to create clusters and pools. Current options are: `"allow-cluster-create", "allow-instance-pool-create"`. +* `default_roles` - (Required) (Set) Set of roles that are assigned to the `all_users` group in Databricks. You can use the default_user_roles data source to fetch the values for this. +* `set_admin` - (Optional) (Bool) Setting this to true will patch this user to include the admin group id as a group item and if false, it will patch remove this user from the admin group. + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the user. +* `inherited_roles` - (Set) The list of roles inherited by parent and all_users groups. This is used to determine when there are no changes. + +## Import + +The resource scim user can be imported using id: + +```bash +$ terraform import databricks_scim_user.me +``` \ No newline at end of file diff --git a/docs/resources/secret.md b/docs/resources/secret.md new file mode 100644 index 0000000000..92adef4c8d --- /dev/null +++ b/docs/resources/secret.md @@ -0,0 +1,41 @@ +# databricks_secret Resource + +With this resource you can insert a secret under the provided scope with the given name. If a secret already exists with the same name, this command overwrites the existing secret’s value. The server encrypts the secret using the secret scope’s encryption settings before storing it. You must have WRITE or MANAGE permission on the secret scope. The secret key must consist of alphanumeric characters, dashes, underscores, and periods, and cannot exceed 128 characters. The maximum allowed secret value size is 128 KB. The maximum number of secrets in a given scope is 1000. You can read a secret value only from within a command on a cluster (for example, through a notebook); there is no API to read a secret value outside of a cluster. The permission applied is based on who is invoking the command and you must have at least READ permission. Please consult [Secrets User Guide](https://docs.databricks.com/security/secrets/index.html#secrets-user-guide) for more details. + +## Example Usage + +```hcl +resource "databricks_secret_scope" "app" { + name = "Application Secret Scope" +} +resource "databricks_secret" "publishing_api" { + key = "publishing_api" + string_value = data.azurerm_key_vault_secret.example.value + scope = databricks_secret_scope.app.name +} +``` + +## Argument Reference + +The following arguments are required: + +* `string_value` - (Required) (String) super secret sensitive value. +* `scope` - (Required) (String) name of databricks secret scope +* `key` - (Required) (String) key within secret scope + + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the secret. +* `last_updated_timestamp` - (Integer) time secret was updated + + +## Import + +The resource secret can be imported using `scopeName|||secretKey` combination. **This may change in future versions.** + +```bash +$ terraform import databricks_secret.app `scopeName|||secretKey` +``` \ No newline at end of file diff --git a/docs/resources/secret_acl.md b/docs/resources/secret_acl.md new file mode 100644 index 0000000000..5a389a9a68 --- /dev/null +++ b/docs/resources/secret_acl.md @@ -0,0 +1,46 @@ +# databricks_secret_acl Resource + +Create or overwrite the ACL associated with the given principal (user or group) on the specified scope point. Please consult [Secrets User Guide](https://docs.databricks.com/security/secrets/index.html#secrets-user-guide) for more details. + +## Example Usage + +This way data scientists can read Publishing API key that is synchronized from example Azure Key Vault. + +```hcl +resource "databricks_group" "ds" { + display_name = "data-scientists" +} + +resource "databricks_secret_scope" "app" { + name = "Application Secret Scope" +} + +resource "databricks_secret_acl" "my_secret_acl" { + principal = databricks_group.ds.display_name + permission = "READ" + scope = databricks_secret_scope.app.name +} + +resource "databricks_secret" "publishing_api" { + key = "publishing_api" + // replace it with secret management solution of your choice :-) + string_value = data.azurerm_key_vault_secret.example.value + scope = databricks_secret_scope.app.name +} +``` + +## Argument Reference + +The following arguments are required: + +* `scope` - (Required) name of the scope +* `principal` - (Required) name of the principals. It can be `users` for all users or name or `display_name` of [databricks_group](group.md) +* `permission` - (Required) `READ`, `WRITE` or `MANAGE`. + +## Import + +The resource secret acl can be imported using `scopeName|||principalName` combination. **This may change in future versions.** + +```bash +$ terraform import databricks_secret_acl.object `scopeName|||principalName` +``` \ No newline at end of file diff --git a/docs/resources/secret_scope.md b/docs/resources/secret_scope.md index 47db852ad4..9c9f116b11 100644 --- a/docs/resources/secret_scope.md +++ b/docs/resources/secret_scope.md @@ -1,13 +1,14 @@ # databricks_secret_scope Resource -This resource creates a Databricks-backed secret scope in which secrets are stored in Databricks-managed storage and -encrypted with a cloud-based specific encryption key. +This resource creates a Databricks-backed secret scope in which secrets are stored in Databricks-managed storage and encrypted with a cloud-based specific encryption key. The scope name: * Must be unique within a workspace. * Must consist of alphanumeric characters, dashes, underscores, and periods, and may not exceed 128 characters. +Please consult [Secrets User Guide](https://docs.databricks.com/security/secrets/index.html#secrets-user-guide) for more details. + ## Example Usage ```hcl @@ -22,9 +23,7 @@ resource "databricks_secret_scope" "my-scope" { The following arguments are supported: * `name` - (Required) Scope name requested by the user. Scope names are unique. This field is required. - -* `initial_manage_principal` - (Optional) The principal that is initially granted -MANAGE permission to the created scope. +* `initial_manage_principal` - (Optional) The principal that is initially granted MANAGE permission to the created scope. ## Attribute Reference @@ -34,4 +33,8 @@ In addition to all arguments above, the following attributes are exported: ## Import --> **Note** Importing this resource is not currently supported. +The resource secret scope can be imported using the scope name: + +```bash +$ terraform import databricks_secret_scope.object +``` \ No newline at end of file diff --git a/docs/resources/token.md b/docs/resources/token.md new file mode 100644 index 0000000000..0d1d65a547 --- /dev/null +++ b/docs/resources/token.md @@ -0,0 +1,43 @@ +# databricks_token Resource + +This resource creates an api token that can be used to create Databricks resources. This will create an API token for the user that has authenticated on the provider. So if you have used an admin user to setup the provider then you will be making API tokens for that admin user. + +## Example Usage + +```hcl +// initialize provider in normal mode +provider "databricks" { + alias = "created_workspace" + + host = databricks_mws_workspaces.this.workspace_url + basic_auth {} +} + +// create PAT token to provision entities within workspace +resource "databricks_token" "pat" { + provider = databricks.created_workspace + comment = "Terraform Provisioning" + // 100 day token + lifetime_seconds = 8640000 +} + +// output token for other modules +output "databricks_token" { + value = databricks_token.pat.token_value + sensitive = true +} +``` + +## Argument Reference + +The following arguments are available: + +* `lifetime_seconds` - (Optional) (Integer) The lifetime of the token, in seconds. If no lifetime is specified, the token remains valid indefinitely. +* `comment` - (Optional) (String) Comment that will appear on user settings page for this token. + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - Canonical unique identifier for the token. +* `token_value` - **Sensitive** value of the newly-created token. \ No newline at end of file diff --git a/docs/stubs/aws_s3_mount.md b/docs/stubs/aws_s3_mount.md deleted file mode 100644 index d0ade4d7d5..0000000000 --- a/docs/stubs/aws_s3_mount.md +++ /dev/null @@ -1,35 +0,0 @@ -# databricks_aws_s3_mount Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `cluster_id` - (Optional) (String) - -* `mount_name` - (Required) (String) - -* `s3_bucket_name` - (Required) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the aws s3 mount. - -* `source` - (String) - - -## Import - -The resource aws s3 mount can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_aws_s3_mount.object -``` \ No newline at end of file diff --git a/docs/stubs/azure_adls_gen1_mount.md b/docs/stubs/azure_adls_gen1_mount.md deleted file mode 100644 index 9ddafae1ec..0000000000 --- a/docs/stubs/azure_adls_gen1_mount.md +++ /dev/null @@ -1,47 +0,0 @@ -# databricks_azure_adls_gen1_mount Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `storage_resource_name` - (Required) (String) - -* `client_id` - (Required) (String) - -* `client_secret_key` - (Required) (String) - -* `cluster_id` - (Optional) (String) - -* `mount_name` - (Required) (String) - -* `directory` - (Computed) (String) - -* `tenant_id` - (Required) (String) - -* `client_secret_scope` - (Required) (String) - -* `spark_conf_prefix` - (Optional) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the azure adls gen1 mount. - -* `source` - (String) - - -## Import - -The resource azure adls gen1 mount can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_azure_adls_gen1_mount.object -``` \ No newline at end of file diff --git a/docs/stubs/azure_adls_gen2_mount.md b/docs/stubs/azure_adls_gen2_mount.md deleted file mode 100644 index b5a3e9f866..0000000000 --- a/docs/stubs/azure_adls_gen2_mount.md +++ /dev/null @@ -1,49 +0,0 @@ -# databricks_azure_adls_gen2_mount Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `cluster_id` - (Optional) (String) - -* `container_name` - (Required) (String) - -* `storage_account_name` - (Required) (String) - -* `client_secret_key` - (Required) (String) - -* `mount_name` - (Required) (String) - -* `directory` - (Computed) (String) - -* `tenant_id` - (Required) (String) - -* `client_id` - (Required) (String) - -* `client_secret_scope` - (Required) (String) - -* `initialize_file_system` - (Required) (Bool) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the azure adls gen2 mount. - -* `source` - (String) - - -## Import - -The resource azure adls gen2 mount can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_azure_adls_gen2_mount.object -``` \ No newline at end of file diff --git a/docs/stubs/azure_blob_mount.md b/docs/stubs/azure_blob_mount.md deleted file mode 100644 index 45901178c3..0000000000 --- a/docs/stubs/azure_blob_mount.md +++ /dev/null @@ -1,45 +0,0 @@ -# databricks_azure_blob_mount Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `directory` - (Optional) (String) - -* `auth_type` - (Required) (String) - -* `token_secret_scope` - (Required) (String) - -* `container_name` - (Required) (String) - -* `storage_account_name` - (Required) (String) - -* `token_secret_key` - (Required) (String) - -* `cluster_id` - (Optional) (String) - -* `mount_name` - (Required) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the azure blob mount. - -* `source` - (String) - - -## Import - -The resource azure blob mount can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_azure_blob_mount.object -``` \ No newline at end of file diff --git a/docs/stubs/cluster.md b/docs/stubs/cluster.md deleted file mode 100644 index f71be0ed85..0000000000 --- a/docs/stubs/cluster.md +++ /dev/null @@ -1,306 +0,0 @@ -# databricks_cluster Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `ssh_public_keys` - (Optional) (List) - -* `init_scripts` - (Optional) (List) This field is a block and is documented below. - -* `idempotency_token` - (Optional) (String) - -* `library_jar` - (Optional) (Set) This field is a block and is documented below. - -* `enable_elastic_disk` - (Computed) (Bool) - -* `single_user_name` - (Optional) (String) - -* `libraries` - (Optional) (Set) This field is a block and is documented below. - -* `library_maven` - (Optional) (Set) This field is a block and is documented below. - -* `cluster_id` - (Computed) (String) - -* `spark_version` - (Required) (String) - -* `instance_pool_id` - (Optional) (String) - -* `policy_id` - (Optional) (String) - -* `spark_env_vars` - (Optional) (Map) - -* `num_workers` - (Optional) (Integer) - -* `driver_node_type_id` - (Computed) (String) - -* `custom_tags` - (Optional) (Map) - -* `cluster_log_conf` - (Optional) (List) This field is a block and is documented below. - -* `library_egg` - (Optional) (Set) This field is a block and is documented below. - -* `library_whl` - (Optional) (Set) This field is a block and is documented below. - -* `aws_attributes` - (Optional) (List) This field is a block and is documented below. - -* `autotermination_minutes` - (Optional) (Integer) - -* `spark_conf` - (Optional) (Map) - -* `docker_image` - (Optional) (List) This field is a block and is documented below. - -* `library_pypi` - (Optional) (Set) This field is a block and is documented below. - -* `library_cran` - (Optional) (Set) This field is a block and is documented below. - -* `cluster_name` - (Optional) (String) - -* `autoscale` - (Optional) (List) This field is a block and is documented below. - -* `node_type_id` - (Computed) (String) - - - -### autoscale Configuration Block - - -* `min_workers` - (Optional) (Integer) - -* `max_workers` - (Optional) (Integer) - - -### library_cran Configuration Block - - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `package` - (Optional) (String) - -* `repo` - (Optional) (String) - - -### library_pypi Configuration Block - - -* `repo` - (Optional) (String) - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `package` - (Optional) (String) - - -### docker_image Configuration Block - - -* `basic_auth` - (Optional) (List) This field is a block and is documented below. - -* `url` - (Required) (String) - - -### basic_auth for docker_image Configuration Block - - -* `password` - (Required) (String) - -* `username` - (Required) (String) - - -### aws_attributes Configuration Block - - -* `first_on_demand` - (Computed) (Integer) - -* `availability` - (Computed) (String) - -* `zone_id` - (Computed) (String) - -* `instance_profile_arn` - (Optional) (String) - -* `spot_bid_price_percent` - (Computed) (Integer) - -* `ebs_volume_type` - (Computed) (String) - -* `ebs_volume_count` - (Computed) (Integer) - -* `ebs_volume_size` - (Computed) (Integer) - - -### library_whl Configuration Block - - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `path` - (Optional) (String) - - -### library_egg Configuration Block - - -* `status` - (Computed) (String) - -* `path` - (Optional) (String) - -* `messages` - (Computed) (List) - - -### cluster_log_conf Configuration Block - - -* `dbfs` - (Optional) (List) This field is a block and is documented below. - -* `s3` - (Optional) (List) This field is a block and is documented below. - - -### s3 for cluster_log_conf Configuration Block - - -* `encryption_type` - (Optional) (String) - -* `kms_key` - (Optional) (String) - -* `canned_acl` - (Optional) (String) - -* `destination` - (Required) (String) - -* `region` - (Optional) (String) - -* `endpoint` - (Optional) (String) - -* `enable_encryption` - (Optional) (Bool) - - -### dbfs for cluster_log_conf Configuration Block - - -* `destination` - (Required) (String) - - -### library_maven Configuration Block - - -* `status` - (Computed) (String) - -* `coordinates` - (Optional) (String) - -* `repo` - (Optional) (String) - -* `exclusions` - (Optional) (List) - -* `messages` - (Computed) (List) - - -### libraries Configuration Block - - -* `pypi` - (Optional) (List) This field is a block and is documented below. - -* `maven` - (Optional) (List) This field is a block and is documented below. - -* `cran` - (Optional) (List) This field is a block and is documented below. - -* `jar` - (Optional) (String) - -* `egg` - (Optional) (String) - -* `whl` - (Optional) (String) - - -### cran for libraries Configuration Block - - -* `repo` - (Optional) (String) - -* `package` - (Required) (String) - - -### maven for libraries Configuration Block - - -* `coordinates` - (Required) (String) - -* `repo` - (Optional) (String) - -* `exclusions` - (Optional) (List) - - -### pypi for libraries Configuration Block - - -* `package` - (Required) (String) - -* `repo` - (Optional) (String) - - -### library_jar Configuration Block - - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `path` - (Optional) (String) - - -### init_scripts Configuration Block - - -* `s3` - (Optional) (List) This field is a block and is documented below. - -* `dbfs` - (Optional) (List) This field is a block and is documented below. - - -### dbfs for init_scripts Configuration Block - - -* `destination` - (Required) (String) - - -### s3 for init_scripts Configuration Block - - -* `canned_acl` - (Optional) (String) - -* `destination` - (Required) (String) - -* `region` - (Optional) (String) - -* `endpoint` - (Optional) (String) - -* `enable_encryption` - (Optional) (Bool) - -* `encryption_type` - (Optional) (String) - -* `kms_key` - (Optional) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the cluster. - -* `state` - (String) - -* `default_tags` - (Map) - - -## Import - -The resource cluster can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_cluster.object -``` \ No newline at end of file diff --git a/docs/stubs/cluster_policy.md b/docs/stubs/cluster_policy.md deleted file mode 100644 index 110d0ced70..0000000000 --- a/docs/stubs/cluster_policy.md +++ /dev/null @@ -1,35 +0,0 @@ -# databricks_cluster_policy Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `name` - (Required) (String) Cluster policy name. This must be unique. -Length must be between 1 and 100 characters. - -* `definition` - (Optional) (String) Policy definition JSON document expressed in -Databricks Policy Definition Language. - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the cluster policy. - -* `policy_id` - (String) - - -## Import - -The resource cluster policy can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_cluster_policy.object -``` \ No newline at end of file diff --git a/docs/stubs/dbfs_file.md b/docs/stubs/dbfs_file.md deleted file mode 100644 index bbae7a7f4f..0000000000 --- a/docs/stubs/dbfs_file.md +++ /dev/null @@ -1,43 +0,0 @@ -# databricks_dbfs_file Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `path` - (Required) (String) - -* `overwrite` - (Optional) (Bool) - -* `mkdirs` - (Optional) (Bool) - -* `validate_remote_file` - (Optional) (Bool) - -* `content` - (Optional) (String) - -* `source` - (Optional) (String) - -* `content_b64_md5` - (Required) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the dbfs file. - -* `file_size` - (Integer) - - -## Import - -The resource dbfs file can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_dbfs_file.object -``` \ No newline at end of file diff --git a/docs/stubs/group.md b/docs/stubs/group.md deleted file mode 100644 index b3cbf89685..0000000000 --- a/docs/stubs/group.md +++ /dev/null @@ -1,28 +0,0 @@ -# databricks_group Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `display_name` - (Required) (String) - -* `allow_cluster_create` - (Optional) (Bool) - -* `allow_instance_pool_create` - (Optional) (Bool) - - - - - - -## Import - -The resource group can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_group.object -``` \ No newline at end of file diff --git a/docs/stubs/group_instance_profile.md b/docs/stubs/group_instance_profile.md deleted file mode 100644 index 9540095a69..0000000000 --- a/docs/stubs/group_instance_profile.md +++ /dev/null @@ -1,26 +0,0 @@ -# databricks_group_instance_profile Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `group_id` - (Required) (String) - -* `instance_profile_id` - (Required) (String) - - - - - - -## Import - -The resource group instance profile can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_group_instance_profile.object -``` \ No newline at end of file diff --git a/docs/stubs/group_member.md b/docs/stubs/group_member.md deleted file mode 100644 index b1fe09a8b6..0000000000 --- a/docs/stubs/group_member.md +++ /dev/null @@ -1,26 +0,0 @@ -# databricks_group_member Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `group_id` - (Required) (String) - -* `member_id` - (Required) (String) - - - - - - -## Import - -The resource group member can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_group_member.object -``` \ No newline at end of file diff --git a/docs/stubs/instance_pool.md b/docs/stubs/instance_pool.md deleted file mode 100644 index edd840d270..0000000000 --- a/docs/stubs/instance_pool.md +++ /dev/null @@ -1,74 +0,0 @@ -# databricks_instance_pool Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `enable_elastic_disk` - (Optional) (Bool) - -* `disk_spec` - (Optional) (List) This field is a block and is documented below. - -* `preloaded_spark_versions` - (Optional) (List) - -* `state` - (Computed) (String) - -* `aws_attributes` - (Optional) (List) This field is a block and is documented below. - -* `custom_tags` - (Optional) (Map) - -* `max_capacity` - (Required) (Integer) - -* `idle_instance_autotermination_minutes` - (Required) (Integer) - -* `node_type_id` - (Required) (String) - -* `instance_pool_name` - (Required) (String) - -* `min_idle_instances` - (Required) (Integer) - - - -### aws_attributes Configuration Block - - -* `spot_bid_price_percent` - (Optional) (Integer) - -* `availability` - (Optional) (String) - -* `zone_id` - (Required) (String) - - -### disk_spec Configuration Block - - -* `ebs_volume_type` - (Optional) (String) - -* `azure_disk_volume_type` - (Optional) (String) - -* `disk_count` - (Optional) (Integer) - -* `disk_size` - (Optional) (Integer) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the instance pool. - -* `default_tags` - (Map) - - -## Import - -The resource instance pool can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_instance_pool.object -``` \ No newline at end of file diff --git a/docs/stubs/instance_profile.md b/docs/stubs/instance_profile.md deleted file mode 100644 index 95e49c4caa..0000000000 --- a/docs/stubs/instance_profile.md +++ /dev/null @@ -1,26 +0,0 @@ -# databricks_instance_profile Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `instance_profile_arn` - (Required) (String) - -* `skip_validation` - (Required) (Bool) - - - - - - -## Import - -The resource instance profile can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_instance_profile.object -``` \ No newline at end of file diff --git a/docs/stubs/job.md b/docs/stubs/job.md deleted file mode 100644 index c21f57cd19..0000000000 --- a/docs/stubs/job.md +++ /dev/null @@ -1,399 +0,0 @@ -# databricks_job Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `new_cluster` - (Optional) (List) Same set of parameters as for [databricks_cluster](cluster.md) resource. This field is a block and is documented below. - -* `library_cran` - (Optional) (Set) This field is a block and is documented below. - -* `name` - (Optional) (String) An optional name for the job. The default value is Untitled. - -* `python_file` - (Optional) (String) Deprecated. Please use `spark_python_task`. - -* `library_maven` - (Optional) (Set) This field is a block and is documented below. - -* `libraries` - (Optional) (Set) An optional list of libraries to be installed on the cluster that will execute the job. The default value is an empty list. This field is a block and is documented below. - -* `retry_on_timeout` - (Optional) (Bool) An optional policy to specify whether to retry a job when it times out. The default behavior is to not retry on timeout. - -* `jar_main_class_name` - (Optional) (String) Deprecated. Please use `spark_jar_task`. - -* `notebook_task` - (Optional) (List) This field is a block and is documented below. - -* `notebook_path` - (Optional) (String) Deprecated. Please use `notebook_task`. - -* `notebook_base_parameters` - (Optional) (Map) Deprecated. Please use `notebook_task`. - -* `existing_cluster_id` - (Optional) (String) If existing_cluster_id, the ID of an existing cluster that will be used for all runs of this job. When running jobs on an existing cluster, you may need to manually restart the cluster if it stops responding. We strongly suggest to use `new_cluster` for greater reliability. - -* `spark_python_task` - (Optional) (List) This field is a block and is documented below. - -* `max_retries` - (Optional) (Integer) An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with a FAILED result_state or INTERNAL_ERROR life_cycle_state. The value -1 means to retry indefinitely and the value 0 means to never retry. The default behavior is to never retry. - -* `jar_uri` - (Optional) (String) - -* `spark_submit_parameters` - (Optional) (List) - -* `timeout_seconds` - (Optional) (Integer) An optional timeout applied to each run of this job. The default behavior is to have no timeout. - -* `min_retry_interval_millis` - (Optional) (Integer) An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried. - -* `max_concurrent_runs` - (Optional) (Integer) An optional maximum allowed number of concurrent runs of the job. - -* `email_notifications` - (Optional) (List) An optional set of email addresses notified when runs of this job begin and complete and when this job is deleted. The default behavior is to not send any emails. This field is a block and is documented below. - -* `python_parameters` - (Optional) (List) Deprecated. Please use `spark_python_task`. - -* `library_pypi` - (Optional) (Set) This field is a block and is documented below. - -* `spark_submit_task` - (Optional) (List) This field is a block and is documented below. - -* `library_jar` - (Optional) (Set) This field is a block and is documented below. - -* `library_whl` - (Optional) (Set) This field is a block and is documented below. - -* `spark_jar_task` - (Optional) (List) This field is a block and is documented below. - -* `schedule` - (Optional) (List) An optional periodic schedule for this job. The default behavior is that the job runs when triggered by clicking Run Now in the Jobs UI or sending an API request to runNow. This field is a block and is documented below. - -* `jar_parameters` - (Optional) (List) Deprecated. Please use `spark_jar_task`. - -* `library_egg` - (Optional) (Set) This field is a block and is documented below. - - - -### library_egg Configuration Block - - -* `status` - (Computed) (String) - -* `path` - (Optional) (String) - -* `messages` - (Computed) (List) - - -### schedule Configuration Block - - -* `quartz_cron_expression` - (Required) (String) - -* `timezone_id` - (Required) (String) - -* `pause_status` - (Optional) (String) - - -### spark_jar_task Configuration Block - - -* `parameters` - (Optional) (List) - -* `jar_uri` - (Optional) (String) - -* `main_class_name` - (Optional) (String) - - -### library_whl Configuration Block - - -* `path` - (Optional) (String) - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - - -### library_jar Configuration Block - - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `path` - (Optional) (String) - - -### spark_submit_task Configuration Block - - -* `parameters` - (Optional) (List) - - -### library_pypi Configuration Block - - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `package` - (Optional) (String) - -* `repo` - (Optional) (String) - - -### email_notifications Configuration Block - - -* `on_failure` - (Optional) (List) - -* `no_alert_for_skipped_runs` - (Optional) (Bool) - -* `on_start` - (Optional) (List) - -* `on_success` - (Optional) (List) - - -### spark_python_task Configuration Block - - -* `python_file` - (Required) (String) - -* `parameters` - (Optional) (List) - - -### notebook_task Configuration Block - - -* `base_parameters` - (Optional) (Map) - -* `notebook_path` - (Required) (String) - - -### libraries Configuration Block - - -* `egg` - (Optional) (String) - -* `whl` - (Optional) (String) - -* `pypi` - (Optional) (List) This field is a block and is documented below. - -* `maven` - (Optional) (List) This field is a block and is documented below. - -* `cran` - (Optional) (List) This field is a block and is documented below. - -* `jar` - (Optional) (String) - - -### cran for libraries Configuration Block - - -* `package` - (Required) (String) - -* `repo` - (Optional) (String) - - -### maven for libraries Configuration Block - - -* `coordinates` - (Required) (String) - -* `repo` - (Optional) (String) - -* `exclusions` - (Optional) (List) - - -### pypi for libraries Configuration Block - - -* `package` - (Required) (String) - -* `repo` - (Optional) (String) - - -### library_maven Configuration Block - - -* `exclusions` - (Optional) (List) - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `coordinates` - (Optional) (String) - -* `repo` - (Optional) (String) - - -### library_cran Configuration Block - - -* `messages` - (Computed) (List) - -* `status` - (Computed) (String) - -* `package` - (Optional) (String) - -* `repo` - (Optional) (String) - - -### new_cluster Configuration Block - - -* `cluster_name` - (Optional) (String) - -* `spark_version` - (Required) (String) - -* `node_type_id` - (Computed) (String) - -* `autotermination_minutes` - (Optional) (Integer) - -* `ssh_public_keys` - (Optional) (List) - -* `enable_elastic_disk` - (Computed) (Bool) - -* `driver_node_type_id` - (Computed) (String) - -* `spark_env_vars` - (Optional) (Map) - -* `idempotency_token` - (Optional) (String) - -* `num_workers` - (Optional) (Integer) - -* `autoscale` - (Optional) (List) This field is a block and is documented below. - -* `instance_pool_id` - (Optional) (String) - -* `docker_image` - (Optional) (List) This field is a block and is documented below. - -* `cluster_log_conf` - (Optional) (List) This field is a block and is documented below. - -* `single_user_name` - (Optional) (String) - -* `cluster_id` - (Optional) (String) - -* `policy_id` - (Optional) (String) - -* `aws_attributes` - (Optional) (List) This field is a block and is documented below. - -* `spark_conf` - (Optional) (Map) - -* `custom_tags` - (Optional) (Map) - -* `init_scripts` - (Optional) (List) This field is a block and is documented below. - - -### init_scripts for new_cluster Configuration Block - - -* `s3` - (Optional) (List) This field is a block and is documented below. - -* `dbfs` - (Optional) (List) This field is a block and is documented below. - - -### dbfs for init_scripts for new_cluster Configuration Block - - -* `destination` - (Required) (String) - - -### s3 for init_scripts for new_cluster Configuration Block - - -* `canned_acl` - (Optional) (String) - -* `destination` - (Required) (String) - -* `region` - (Optional) (String) - -* `endpoint` - (Optional) (String) - -* `enable_encryption` - (Optional) (Bool) - -* `encryption_type` - (Optional) (String) - -* `kms_key` - (Optional) (String) - - -### aws_attributes for new_cluster Configuration Block - - -* `availability` - (Computed) (String) - -* `zone_id` - (Computed) (String) - -* `instance_profile_arn` - (Optional) (String) - -* `spot_bid_price_percent` - (Computed) (Integer) - -* `ebs_volume_type` - (Computed) (String) - -* `ebs_volume_count` - (Computed) (Integer) - -* `ebs_volume_size` - (Computed) (Integer) - -* `first_on_demand` - (Computed) (Integer) - - -### cluster_log_conf for new_cluster Configuration Block - - -* `dbfs` - (Optional) (List) This field is a block and is documented below. - -* `s3` - (Optional) (List) This field is a block and is documented below. - - -### s3 for cluster_log_conf for new_cluster Configuration Block - - -* `destination` - (Required) (String) - -* `region` - (Optional) (String) - -* `endpoint` - (Optional) (String) - -* `enable_encryption` - (Optional) (Bool) - -* `encryption_type` - (Optional) (String) - -* `kms_key` - (Optional) (String) - -* `canned_acl` - (Optional) (String) - - -### dbfs for cluster_log_conf for new_cluster Configuration Block - - -* `destination` - (Required) (String) - - -### docker_image for new_cluster Configuration Block - - -* `url` - (Required) (String) - -* `basic_auth` - (Optional) (List) This field is a block and is documented below. - - -### basic_auth for docker_image for new_cluster Configuration Block - - -* `username` - (Required) (String) - -* `password` - (Required) (String) - - -### autoscale for new_cluster Configuration Block - - -* `min_workers` - (Optional) (Integer) - -* `max_workers` - (Optional) (Integer) - - - - - - -## Import - -The resource job can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_job.object -``` \ No newline at end of file diff --git a/docs/stubs/mws_credentials.md b/docs/stubs/mws_credentials.md deleted file mode 100644 index 529af3db3a..0000000000 --- a/docs/stubs/mws_credentials.md +++ /dev/null @@ -1,39 +0,0 @@ -# databricks_mws_credentials Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `account_id` - (Required) (String) - -* `credentials_name` - (Required) (String) - -* `role_arn` - (Required) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the mws credentials. - -* `creation_time` - (Integer) - -* `external_id` - (String) - -* `credentials_id` - (String) - - -## Import - -The resource mws credentials can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_mws_credentials.object -``` \ No newline at end of file diff --git a/docs/stubs/mws_networks.md b/docs/stubs/mws_networks.md deleted file mode 100644 index 49515d1b4d..0000000000 --- a/docs/stubs/mws_networks.md +++ /dev/null @@ -1,56 +0,0 @@ -# databricks_mws_networks Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `account_id` - (Required) (String) - -* `error_messages` - (Computed) (List) This field is a block and is documented below. - -* `network_name` - (Required) (String) - -* `vpc_id` - (Required) (String) - -* `subnet_ids` - (Required) (Set) - -* `security_group_ids` - (Required) (Set) - - - -### error_messages Configuration Block - - -* `error_type` - (Computed) (String) - -* `error_message` - (Computed) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the mws networks. - -* `creation_time` - (Integer) - -* `network_id` - (String) - -* `vpc_status` - (String) - -* `workspace_id` - (Integer) - - -## Import - -The resource mws networks can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_mws_networks.object -``` \ No newline at end of file diff --git a/docs/stubs/mws_storage_configurations.md b/docs/stubs/mws_storage_configurations.md deleted file mode 100644 index c6bd58d4a5..0000000000 --- a/docs/stubs/mws_storage_configurations.md +++ /dev/null @@ -1,37 +0,0 @@ -# databricks_mws_storage_configurations Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `bucket_name` - (Required) (String) - -* `account_id` - (Required) (String) - -* `storage_configuration_name` - (Required) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the mws storage configurations. - -* `creation_time` - (Integer) - -* `storage_configuration_id` - (String) - - -## Import - -The resource mws storage configurations can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_mws_storage_configurations.object -``` \ No newline at end of file diff --git a/docs/stubs/mws_workspaces.md b/docs/stubs/mws_workspaces.md deleted file mode 100644 index 40d5d3172b..0000000000 --- a/docs/stubs/mws_workspaces.md +++ /dev/null @@ -1,68 +0,0 @@ -# databricks_mws_workspaces Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `customer_managed_key_id` - (Optional) (String) - -* `network_id` - (Optional) (String) - -* `is_no_public_ip_enabled` - (Optional) (Bool) - -* `account_id` - (Required) (String) - -* `credentials_id` - (Required) (String) - -* `network_error_messages` - (Computed) (List) This field is a block and is documented below. - -* `deployment_name` - (Required) (String) - -* `workspace_name` - (Required) (String) - -* `aws_region` - (Required) (String) - -* `storage_configuration_id` - (Required) (String) - -* `verify_workspace_runnning` - (Required) (Bool) - - - -### network_error_messages Configuration Block - - -* `error_message` - (Computed) (String) - -* `error_type` - (Computed) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the mws workspaces. - -* `workspace_status_message` - (String) - -* `workspace_status` - (String) - -* `creation_time` - (Integer) - -* `workspace_url` - (String) - -* `workspace_id` - (Integer) - - -## Import - -The resource mws workspaces can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_mws_workspaces.object -``` \ No newline at end of file diff --git a/docs/stubs/notebook.md b/docs/stubs/notebook.md deleted file mode 100644 index 74f1f32f48..0000000000 --- a/docs/stubs/notebook.md +++ /dev/null @@ -1,43 +0,0 @@ -# databricks_notebook Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `mkdirs` - (Optional) (Bool) - -* `format` - (Optional) (String) - -* `content` - (Required) (String) - -* `path` - (Required) (String) - -* `language` - (Optional) (String) - -* `overwrite` - (Optional) (Bool) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the notebook. - -* `object_type` - (String) - -* `object_id` - (Integer) - - -## Import - -The resource notebook can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_notebook.object -``` \ No newline at end of file diff --git a/docs/stubs/permissions.md b/docs/stubs/permissions.md deleted file mode 100644 index 163a544689..0000000000 --- a/docs/stubs/permissions.md +++ /dev/null @@ -1,58 +0,0 @@ -# databricks_permissions Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `notebook_path` - (Optional) (String) - -* `cluster_id` - (Optional) (String) - -* `job_id` - (Optional) (String) - -* `notebook_id` - (Optional) (String) - -* `directory_id` - (Optional) (String) - -* `directory_path` - (Optional) (String) - -* `access_control` - (Required) (List) This field is a block and is documented below. - -* `cluster_policy_id` - (Optional) (String) - -* `instance_pool_id` - (Optional) (String) - - - -### access_control Configuration Block - - -* `permission_level` - (Required) (String) - -* `user_name` - (Optional) (String) - -* `group_name` - (Optional) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the permissions. - -* `object_type` - (String) - - -## Import - -The resource permissions can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_permissions.object -``` \ No newline at end of file diff --git a/docs/stubs/scim_group.md b/docs/stubs/scim_group.md deleted file mode 100644 index 501db3feb1..0000000000 --- a/docs/stubs/scim_group.md +++ /dev/null @@ -1,37 +0,0 @@ -# databricks_scim_group Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `display_name` - (Required) (String) - -* `members` - (Optional) (Set) - -* `roles` - (Optional) (Set) - -* `entitlements` - (Optional) (Set) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the scim group. - -* `inherited_roles` - (Set) - - -## Import - -The resource scim group can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_scim_group.object -``` \ No newline at end of file diff --git a/docs/stubs/scim_user.md b/docs/stubs/scim_user.md deleted file mode 100644 index fac8f26ca6..0000000000 --- a/docs/stubs/scim_user.md +++ /dev/null @@ -1,41 +0,0 @@ -# databricks_scim_user Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `roles` - (Optional) (Set) - -* `entitlements` - (Optional) (Set) - -* `default_roles` - (Required) (Set) - -* `set_admin` - (Optional) (Bool) - -* `user_name` - (Required) (String) - -* `display_name` - (Optional) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the scim user. - -* `inherited_roles` - (Set) - - -## Import - -The resource scim user can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_scim_user.object -``` \ No newline at end of file diff --git a/docs/stubs/secret.md b/docs/stubs/secret.md deleted file mode 100644 index 191d495d28..0000000000 --- a/docs/stubs/secret.md +++ /dev/null @@ -1,35 +0,0 @@ -# databricks_secret Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `string_value` - (Required) (String) - -* `scope` - (Required) (String) - -* `key` - (Required) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the secret. - -* `last_updated_timestamp` - (Integer) - - -## Import - -The resource secret can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_secret.object -``` \ No newline at end of file diff --git a/docs/stubs/secret_acl.md b/docs/stubs/secret_acl.md deleted file mode 100644 index 7c96649247..0000000000 --- a/docs/stubs/secret_acl.md +++ /dev/null @@ -1,28 +0,0 @@ -# databricks_secret_acl Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `scope` - (Required) (String) - -* `principal` - (Required) (String) - -* `permission` - (Required) (String) - - - - - - -## Import - -The resource secret acl can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_secret_acl.object -``` \ No newline at end of file diff --git a/docs/stubs/secret_scope.md b/docs/stubs/secret_scope.md deleted file mode 100644 index 2a6a1ff0f7..0000000000 --- a/docs/stubs/secret_scope.md +++ /dev/null @@ -1,33 +0,0 @@ -# databricks_secret_scope Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `name` - (Required) (String) - -* `initial_manage_principal` - (Optional) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the secret scope. - -* `backend_type` - (String) - - -## Import - -The resource secret scope can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_secret_scope.object -``` \ No newline at end of file diff --git a/docs/stubs/token.md b/docs/stubs/token.md deleted file mode 100644 index 4d2cc69810..0000000000 --- a/docs/stubs/token.md +++ /dev/null @@ -1,37 +0,0 @@ -# databricks_token Resource - - -## Example Usage -my_usage - -## Argument Reference - -The following arguments are required: - -* `creation_time` - (Computed) (Integer) - -* `expiry_time` - (Computed) (Integer) - -* `lifetime_seconds` - (Optional) (Integer) - -* `comment` - (Optional) (String) - - - - -## Attribute Reference - -In addition to all arguments above, the following attributes are exported: - -* `id` - Canonical unique identifier for the token. - -* `token_value` - (String) - - -## Import - -The resource token can be imported using the `object`, e.g. - -```bash -$ terraform import databricks_token.object -``` \ No newline at end of file diff --git a/identity/resource_scim_user.go b/identity/resource_scim_user.go index 772a948d7a..a0c977aad0 100644 --- a/identity/resource_scim_user.go +++ b/identity/resource_scim_user.go @@ -10,55 +10,29 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/helper/schema" ) +type scimUser struct { + UserName string `json:"user_name"` + DisplayName string `json:"display_name,omitempty"` + Roles []string `json:"roles,omitempty" tf:"slice_set"` + Entitlements []string `json:"entitlements,omitempty" tf:"slice_set"` + DefaultRoles []string `json:"default_roles" tf:"slice_set"` + InheritedRoles []string `json:"inherited_roles,omitempty" tf:"slice_set,computed"` + SetAdmin bool `json:"set_admin,omitempty"` +} + +// ResourceScimUser .. func ResourceScimUser() *schema.Resource { return &schema.Resource{ Create: resourceScimUserCreate, Update: resourceScimUserUpdate, Read: resourceScimUserRead, Delete: resourceScimUserDelete, - - Schema: map[string]*schema.Schema{ - "user_name": { - Type: schema.TypeString, - Required: true, - ForceNew: true, - }, - "display_name": { - Type: schema.TypeString, - Optional: true, - }, - "roles": { - Type: schema.TypeSet, - Optional: true, - ConfigMode: schema.SchemaConfigModeAttr, - Elem: &schema.Schema{Type: schema.TypeString}, - Set: schema.HashString, - }, - "entitlements": { - Type: schema.TypeSet, - Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, - Set: schema.HashString, - }, - "inherited_roles": { - Type: schema.TypeSet, - Computed: true, - Elem: &schema.Schema{Type: schema.TypeString}, - Set: schema.HashString, - }, - "default_roles": { - Type: schema.TypeSet, - Required: true, - Elem: &schema.Schema{Type: schema.TypeString}, - Set: schema.HashString, - }, - "set_admin": { - Type: schema.TypeBool, - Optional: true, - Default: false, - Set: schema.HashString, - }, - }, + Schema: internal.StructToSchema(scimUser{}, func( + s map[string]*schema.Schema) map[string]*schema.Schema { + s["user_name"].ForceNew = true + s["set_admin"].Default = false + return s + }), } } diff --git a/provider/generate_test.go b/provider/generate_test.go index 8477043282..72f220479f 100644 --- a/provider/generate_test.go +++ b/provider/generate_test.go @@ -40,9 +40,13 @@ func (stub *resourceTestStub) Reads(t *testing.T) { stub.stoobyDo(t, "Read", ` func TestResource{{.Name}}Read(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - // read log output of test util for further stubs... - }, resource{{.Name}}, nil, internal.ActionWithID("abc", resource{{.Name}}Read)) + Fixtures: []qa.HTTPFixture{ + // read log output of test util for further stubs... + }, + Resource: Resource{{.Name}}(), + Read: true, + ID: "abc", + }.Apply(t) assert.NoError(t, err, err) assert.Equal(t, "abc", d.Id(), "Id should not be empty") {{range $index, $element := .Resource.Schema}}assert.Equal(t, "...{{$index}}", d.Get("{{$index}}")) @@ -51,34 +55,42 @@ func (stub *resourceTestStub) Reads(t *testing.T) { stub.stoobyDo(t, "Read_NotFound", ` func TestResource{{.Name}}Read_NotFound(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - { // read log output for correct url... - Method: "GET", - Resource: "/api/2.0/...", - Response: common.APIErrorBody{ - ErrorCode: "NOT_FOUND", - Message: "Item not found", + Fixtures: []qa.HTTPFixture{ + { // read log output for correct url... + Method: "GET", + Resource: "/api/2.0/...", + Response: common.APIErrorBody{ + ErrorCode: "NOT_FOUND", + Message: "Item not found", + }, + Status: 404, }, - Status: 404, }, - }, resource{{.Name}}, nil, internal.ActionWithID("abc", resource{{.Name}}Read)) + Resource: Resource{{.Name}}(), + Read: true, + ID: "abc", + }.Apply(t) assert.NoError(t, err, err) assert.Equal(t, "", d.Id(), "Id should be empty for missing resources") }`) stub.stoobyDo(t, "Read_Error", ` func TestResource{{.Name}}Read_Error(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - { // read log output for correct url... - Method: "GET", - Resource: "/api/2.0/...", - Response: common.APIErrorBody{ - ErrorCode: "INVALID_REQUEST", - Message: "Internal error happened", + Fixtures: []qa.HTTPFixture{ + { // read log output for correct url... + Method: "GET", + Resource: "/api/2.0/...", + Response: common.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, }, - Status: 400, }, - }, resource{{.Name}}, nil, internal.ActionWithID("abc", resource{{.Name}}Read)) + Resource: Resource{{.Name}}(), + Read: true, + ID: "abc", + }.Apply(t) qa.AssertErrorStartsWith(t, err, "Internal error happened") assert.Equal(t, "abc", d.Id(), "Id should not be empty for error reads") }`) @@ -88,33 +100,41 @@ func (stub *resourceTestStub) Creates(t *testing.T) { stub.stoobyDo(t, "Create", ` func TestResource{{.Name}}Create(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - // request #1 - most likely POST - // request #2 - same as in TestResource{{.Name}}Read - }, resource{{.Name}}, map[string]interface{}{ - {{range $key, $element := .Resource.Schema}}"{{$key}}": "...", + Fixtures: []qa.HTTPFixture{ + // request #1 - most likely POST + // request #2 - same as in TestResource{{.Name}}Read + }, + Resource: Resource{{.Name}}(), + Create: true, + HCL: `+"`"+` + {{range $key, $element := .Resource.Schema}}{{$key}} = "..." {{end}} - }, resource{{.Name}}Create) + `+"`"+`, + }.Apply(t) assert.NoError(t, err, err) assert.Equal(t, "...", d.Id()) }`) stub.stoobyDo(t, "Create_Error", ` func TestResource{{.Name}}Create_Error(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - { // read log output for better stub url... - Method: "POST", - Resource: "/api/2.0/...", - Response: common.APIErrorBody{ - ErrorCode: "INVALID_REQUEST", - Message: "Internal error happened", + Fixtures: []qa.HTTPFixture{ + { // read log output for better stub url... + Method: "POST", + Resource: "/api/2.0/...", + Response: common.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, }, - Status: 400, }, - }, resource{{.Name}}, map[string]interface{}{ - {{range $key, $element := .Resource.Schema}}"{{$key}}": "...", + Resource: Resource{{.Name}}(), + Create: true, + HCL: `+"`"+` + {{range $key, $element := .Resource.Schema}}{{$key}} = "..." {{end}} - }, resource{{.Name}}Create) + `+"`"+`, + }.Apply(t) qa.AssertErrorStartsWith(t, err, "Internal error happened") assert.Equal(t, "", d.Id(), "Id should be empty for error creates") }`) @@ -124,33 +144,43 @@ func (stub *resourceTestStub) Updates(t *testing.T) { stub.stoobyDo(t, "Update", ` func TestResource{{.Name}}Update(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - // request #1 - most likely POST - // request #2 - same as in TestResource{{.Name}}Read - }, resource{{.Name}}, map[string]interface{}{ - {{range $key, $element := .Resource.Schema}}"{{$key}}": "...", + Fixtures: []qa.HTTPFixture{ + // request #1 - most likely POST + // request #2 - same as in TestResource{{.Name}}Read + }, + Resource: Resource{{.Name}}(), + Update: true, + ID: "abc", + HCL: `+"`"+` + {{range $key, $element := .Resource.Schema}}{{$key}} = "..." {{end}} - }, internal.ActionWithID("abc", resource{{.Name}}Update)) + `+"`"+`, + }.Apply(t) assert.NoError(t, err, err) assert.Equal(t, "abc", d.Id(), "Id should be the same as in reading") }`) stub.stoobyDo(t, "Update_Error", ` func TestResource{{.Name}}Update_Error(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - { // read log output for better stub url... - Method: "POST", - Resource: "/api/2.0/.../edit", - Response: common.APIErrorBody{ - ErrorCode: "INVALID_REQUEST", - Message: "Internal error happened", + Fixtures: []qa.HTTPFixture{ + { // read log output for better stub url... + Method: "POST", + Resource: "/api/2.0/.../edit", + Response: common.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, }, - Status: 400, - }, - }, resource{{.Name}}, map[string]interface{}{ - {{range $key, $element := .Resource.Schema}}"{{$key}}": "...", + }, + Resource: Resource{{.Name}}(), + Update: true, + ID: "abc", + HCL: `+"`"+` + {{range $key, $element := .Resource.Schema}}{{$key}} = "..." {{end}} - }, internal.ActionWithID("abc", resource{{.Name}}Update)) + `+"`"+`, + }.Apply(t) qa.AssertErrorStartsWith(t, err, "Internal error happened") assert.Equal(t, "abc", d.Id()) }`) @@ -160,32 +190,40 @@ func (stub *resourceTestStub) Deletes(t *testing.T) { stub.stoobyDo(t, "Delete", ` func TestResource{{.Name}}Delete(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - { // read log output for better stub url... - Method: "POST", - Resource: "/api/2.0/.../delete", - ExpectedRequest: map[string]string{ - "...id": "abc", + Fixtures: []qa.HTTPFixture{ + { // read log output for better stub url... + Method: "POST", + Resource: "/api/2.0/.../delete", + ExpectedRequest: map[string]string{ + "...id": "abc", + }, }, }, - }, resource{{.Name}}, nil, internal.ActionWithID("abc", resource{{.Name}}Delete)) + Resource: Resource{{.Name}}(), + Delete: true, + ID: "abc", + }.Apply(t) assert.NoError(t, err, err) assert.Equal(t, "abc", d.Id()) }`) stub.stoobyDo(t, "Delete_Error", ` func TestResource{{.Name}}Delete_Error(t *testing.T) { d, err := qa.ResourceFixture{ - Fixtures: []qa.HTTPFixture{ - { - Method: "POST", - Resource: "/api/2.0/.../delete", - Response: common.APIErrorBody{ - ErrorCode: "INVALID_REQUEST", - Message: "Internal error happened", + Fixtures: []qa.HTTPFixture{ + { + Method: "POST", + Resource: "/api/2.0/.../delete", + Response: common.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, }, - Status: 400, }, - }, resource{{.Name}}, nil, internal.ActionWithID("abc", resource{{.Name}}Delete)) + Resource: Resource{{.Name}}(), + Delete: true, + ID: "abc", + }.Apply(t) qa.AssertErrorStartsWith(t, err, "Internal error happened") assert.Equal(t, "abc", d.Id()) }`) @@ -196,7 +234,7 @@ func TestGenerateTestCodeStubs(t *testing.T) { funcs := getExistingUnitTests() p := DatabricksProvider().(*schema.Provider) for name, resource := range p.ResourcesMap { - if name != "databricks_group" { + if name != "databricks_scim_user" { continue } stub := resourceTestStub{Resource: resource, others: &funcs} diff --git a/storage/acceptance/adls_gen1_test.go b/storage/acceptance/adls_gen1_test.go new file mode 100644 index 0000000000..c0966d12d6 --- /dev/null +++ b/storage/acceptance/adls_gen1_test.go @@ -0,0 +1,42 @@ +package acceptance + +import ( + "os" + "testing" + + "github.com/databrickslabs/databricks-terraform/internal/acceptance" + "github.com/databrickslabs/databricks-terraform/internal/qa" + "github.com/hashicorp/terraform-plugin-sdk/helper/resource" +) + +func TestAzureAccAdlsGen1Mount_correctly_mounts(t *testing.T) { + if _, ok := os.LookupEnv("CLOUD_ENV"); !ok { + t.Skip("Acceptance tests skipped unless env 'CLOUD_ENV' is set") + } + acceptance.AccTest(t, resource.TestCase{ + Steps: []resource.TestStep{ + { + Config: qa.EnvironmentTemplate(t, ` + resource "databricks_secret_scope" "terraform" { + name = "terraform-{var.RANDOM}" + initial_manage_principal = "users" + } + resource "databricks_secret" "client_secret" { + key = "datalake_sp_secret" + string_value = "{env.ARM_CLIENT_SECRET}" + scope = databricks_secret_scope.terraform.name + } + resource "databricks_azure_adls_gen1_mount" "mount" { + container_name = "dev" + storage_account_name = "{env.TEST_GEN2_ADAL_NAME}" + mount_name = "localdir{var.RANDOM}" + tenant_id = "{env.ARM_TENANT_ID}" + client_id = "{env.ARM_CLIENT_ID}" + client_secret_scope = databricks_secret_scope.terraform.name + client_secret_key = databricks_secret.client_secret.key + initialize_file_system = true + }`), + }, + }, + }) +} diff --git a/website/themes/hugo-theme-learn/layouts/partials/header.html b/website/themes/hugo-theme-learn/layouts/partials/header.html index 114f3620ed..0ace5c3c3e 100644 --- a/website/themes/hugo-theme-learn/layouts/partials/header.html +++ b/website/themes/hugo-theme-learn/layouts/partials/header.html @@ -91,6 +91,15 @@
{{ end }}
+ +
+

+ Documentaiton has migrated to + Terraform Registry page. This website is no longer maintained and holding any up-to-date information and will + be deleted before October 2020. Please update any bookmarks to new location. +

+
+ {{if and (not .IsHome) (not .Params.chapter) }}

{{ if eq .Kind "taxonomy" }} @@ -109,3 +118,5 @@

{{.value|safeHTML}} {{end}} {{end}} + +