From 6f504373aa61b327448b9f73ed2d2c01f3093609 Mon Sep 17 00:00:00 2001 From: Serge Smertin <259697+nfx@users.noreply.github.com> Date: Thu, 25 Jun 2020 07:21:48 +0200 Subject: [PATCH] Add more readability for errors (#129) * Add resource wait retry for workspace create * Made vscode integration testing simpler * added read support for username/password for config files * Made errors concise and explainable * Fix formatting issue * cleaned up token request structs * make fmt * Added some documentation * keying composite literals * Add resource wait retry for workspace create * Made vscode integration testing simpler * added read support for username/password for config files * Made errors concise and explainable * Fix formatting issue * cleaned up token request structs * make fmt * Added some documentation * keying composite literals * Add missing resource check in resourceClusterPolicyRead * Apply review comments * More correct implementation of 404-check * Make README more user-friendly * More links * added integration test to verify that all apis can either handle 404s or check the error message to verify resource is missing; added travis buddy integration; added unit test for handling errors; deprecated dbfs file sync; doc fix for scim user; * added skip for testMissingWorkspaceResources to run only if TF_ACC is set so it runs with both aws and azure integration tests * cleaned up makefile * refactored tokenexpirytime to the api client config so the client is aware of expiry time of token as metadata; added getAndAssetEnv for generating the dbapi client for testing missing errors * added a int test missing cluster policy * adjusted the headers on the index and added the id attribute for cluster policy * corrected typos * fix another credentials typo Co-authored-by: Serge Smertin Co-authored-by: Sriharsha Tikkireddy --- .vscode/launch.json | 56 ++-- .vscode/settings.json | 1 + CONTRIBUTING.md | 209 +++++++++++++- Makefile | 7 - README.md | 241 ++-------------- client/model/token.go | 6 + client/service/client.go | 89 ++++-- client/service/client_test.go | 91 ++++++ client/service/instance_profiles.go | 9 +- client/service/secret_scopes.go | 7 +- client/service/secrets.go | 7 +- client/service/tokens.go | 16 +- databricks/azure_auth.go | 47 ++- databricks/azure_auth_test.go | 9 - databricks/provider.go | 33 ++- databricks/provider_test.go | 20 ++ ...source_databricks_azure_adls_gen1_mount.go | 2 +- ...source_databricks_azure_adls_gen2_mount.go | 2 +- .../resource_databricks_azure_blob_mount.go | 2 +- databricks/resource_databricks_cluster.go | 9 +- .../resource_databricks_cluster_policy.go | 7 + ...resource_databricks_cluster_policy_test.go | 21 +- databricks/resource_databricks_dbfs_file.go | 11 +- .../resource_databricks_dbfs_file_sync.go | 3 +- databricks/resource_databricks_group.go | 4 +- .../resource_databricks_group_aws_test.go | 4 +- ...ource_databricks_group_instance_profile.go | 4 +- ...abricks_group_instance_profile_aws_test.go | 2 +- .../resource_databricks_group_member.go | 4 +- ...source_databricks_group_member_aws_test.go | 2 +- .../resource_databricks_instance_pool.go | 11 +- .../resource_databricks_instance_profile.go | 11 +- ...source_databricks_instance_profile_test.go | 135 +++++++++ databricks/resource_databricks_job.go | 10 +- .../resource_databricks_mws_credentials.go | 9 +- .../resource_databricks_mws_networks.go | 9 +- ...e_databricks_mws_storage_configurations.go | 9 +- .../resource_databricks_mws_workspaces.go | 34 ++- databricks/resource_databricks_notebook.go | 11 +- databricks/resource_databricks_scim_group.go | 12 +- databricks/resource_databricks_scim_user.go | 12 +- databricks/resource_databricks_secret.go | 25 +- databricks/resource_databricks_secret_acl.go | 10 +- .../resource_databricks_secret_scope.go | 10 +- databricks/resource_databricks_token.go | 10 +- databricks/utils.go | 5 - databricks/utils_test.go | 268 +++++++++++++++++- docs/index.md | 214 +++++++------- docs/resources/cluster_policy.md | 2 + website/content/Resources/scim_user.md | 1 + website/content/_index.md | 7 +- 51 files changed, 1131 insertions(+), 609 deletions(-) create mode 100644 client/service/client_test.go create mode 100644 databricks/resource_databricks_instance_profile_test.go diff --git a/.vscode/launch.json b/.vscode/launch.json index a4b018201f..9eb574f066 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,25 +1,33 @@ { -     // Use IntelliSense to learn about possible attributes. -     // Hover to view descriptions of existing attributes. -     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 -     "version": "0.2.0", -     "configurations": [ -         { -             "name": "Launch test function", -             "type": "go", -             "request": "launch", -             "mode": "test", -             "program": "${workspaceRoot}/databricks/resource_databricks_azure_adls_gen2_mount_test.go", -             "args": [ -                 "-test.v", -                 "-test.run", -                 "TestAccAzureAdlsGen2Mount_capture_error" -             ], - "env": { - "TF_ACC" : "1" - // "TEST_RESOURCE_GROUP" : "${env:TEST_RESOURCE_GROUP}" - }, -             "showLog": true -         } -     ] - } \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch test function", + "type": "go", + "request": "launch", + "mode": "test", + "program": "${file}", + "args": [ + "-test.v", + "-test.run", + "${selectedText}" + ], + "dlvLoadConfig": { + "followPointers": true, + "maxVariableRecurse": 1, + "maxStringLen": 64, + "maxArrayValues": 64, + "maxStructFields": -1 + }, + "env": { + "TF_ACC": "1", + "DATABRICKS_CONFIG_PROFILE": "sandbox" + // "TEST_RESOURCE_GROUP" : "${env:TEST_RESOURCE_GROUP}" + }, + "showLog": true + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 9bfe033df9..751438f96c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,5 @@ { + "go.testFlags": ["-v"], "go.delveConfig": { "dlvLoadConfig": { "followPointers": true, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7426f2eafe..3cf74cfcc8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1,208 @@ -We happily welcome contributions to databricks-terraform. We use GitHub Issues to track community reported issues and GitHub Pull Requests for accepting changes. \ No newline at end of file +Contributing to Databricks Terraform Provider +--- + +- [Contributing to Databricks Terraform Provider](#contributing-to-databricks-terraform-provider) +- [Install using go](#install-using-go) +- [Downloading the source code and installing the artifact](#downloading-the-source-code-and-installing-the-artifact) +- [Developing with Visual Studio Code Devcontainers](#developing-with-visual-studio-code-devcontainers) +- [Building and Installing with Docker](#building-and-installing-with-docker) +- [Testing](#testing) +- [Linting](#linting) +- [Integration Testing](#integration-testing) +- [Project Components](#project-components) + - [Databricks Terraform Provider Resources State](#databricks-terraform-provider-resources-state) + - [Databricks Terraform Data Sources State](#databricks-terraform-data-sources-state) + +We happily welcome contributions to databricks-terraform. We use GitHub Issues to track community reported issues and GitHub Pull Requests for accepting changes. + +## Install using go + +Please note that there is a Makefile which contains all the commands you would need to run this project. + +This code base to contribute to requires the following software (this is also all configured for the [Visual Studio Code Devcontainer](#developing-with-visual-studio-code-devcontainers)): + +* [golang 1.13.X](https://golang.org/dl/) +* [terraform v0.12.x](https://www.terraform.io/downloads.html) +* make command + +To make sure everything is installed correctly please run the following commands: + +Testing go installation: +```bash +$ go version +go version go1.13.3 darwin/amd64 +``` + +Testing terraform installation: +```bash +$ terraform --version +Terraform v0.12.19 + +Your version of Terraform is out of date! The latest version +is 0.12.24. You can update by downloading from https://www.terraform.io/downloads.html + +``` + +Testing make installation: +```bash +$ make --version +GNU Make 3.81 +Copyright (C) 2006 Free Software Foundation, Inc. +This is free software; see the source for copying conditions. +There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. + +This program built for i386-apple-darwin11.3.0 +``` + +## Downloading the source code and installing the artifact + +* After installing `golang`, `terraform`, and `make` you will now build the artifact. + +```bash +$ go get -v -u github.com/databrickslabs/databricks-terraform && cd $GOPATH/src/github.com/databrickslabs/databricks-terraform +``` + +:warning: If you are fetching from a private repository please use the following command: + +```bash +$ GOSUMDB=off GOPROXY=direct go get -v -u github.com/databrickslabs/databricks-terraform && cd $GOPATH/src/github.com/databrickslabs/databricks-terraform +``` + +* When you are in the root directory of the repository please run: + +```bash +$ make build +``` + +* Locate your [terraform plugins directory](https://www.terraform.io/docs/extend/how-terraform-works.html#plugin-locations) + or the root folder of your terraform code + +* Copy the `terraform-provider-databricks` artifact to that terraform plugins locations + +```bash +$ mkdir -p ~/.terraform.d/plugins/ && cp terraform-provider-databricks ~/.terraform.d/plugins/terraform-provider-databricks +``` + +Now your plugin for the Databricks Terraform provider is installed correctly. You can actually use the provider. + +## Developing with Visual Studio Code Devcontainers + +This project has configuration for working with [Visual Studio Code Devcontainers](https://code.visualstudio.com/docs/remote/containers) - this allows you to containerise your development prerequisites (e.g. golang, terraform). To use this you will need [Visual Studio Code](https://code.visualstudio.com/) and [Docker](https://www.docker.com/products/docker-desktop). + +To get started, clone this repo and open the folder with Visual Studio Code. If you don't have the [Remote Development extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) then you should be prompted to install it. + +Once the folder is loaded and the extension is installed you should be prompted to re-open the folder in a devcontainer. This will built and run the container image with the correct tools (and versions) ready to start working on and building the code. The in-built terminal will launch a shell inside the container for running `make` commands etc. + +See the docs for more details on working with [devcontainers](https://code.visualstudio.com/docs/remote/containers). + +## Building and Installing with Docker + +To install and build the code if you dont want to install golang, terraform, etc. All you need is docker and git. + +First make sure you clone the repository and you are in the directory. + +Then build the docker image with this command: + +```bash +$ docker build -t databricks-terraform . +``` + +Then run the execute the terraform binary via the following command and volume mount. Make sure that you are in the directory + with the terraform code. The following command you can execute the following commands and additional ones as part of + the terraform binary. + +```bash +$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform init +$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform plan +$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform apply +``` + +## Testing + +* [ ] Integration tests should be run at a client level against both azure and aws to maintain sdk parity against both apis **(currently only on one cloud)** +* [x] Terraform acceptance tests should be run against both aws and azure to maintain parity of provider between both cloud services **(currently only on one cloud)** + +## Linting + +Please use makefile for linting. If you run `golangci-lint` by itself it will fail due to different tags containing same functions. +So please run `make lint` instead. + +## Integration Testing + +Currently Databricks supports two cloud providers `azure` and `aws` thus integration testing with the correct cloud service provider is +crucial for making sure that the provider behaves as expected on all supported clouds. This type of testing separation is being managed via build tags +to allow for duplicate method names and environment variables to configure clients. + +The current integration test implementation uses `CLOUD_ENV` environment variable and can use the value of `azure` or `aws`. +You can execute the acceptance with the following make commands `make terraform-acc-azure`, and `make terraform-acc-aws` for +azure and aws respectively. + +This involves bootstrapping the provider via a .env configuration file. Without these files in the root directory the tests +will fail as the provider will not have a authorized token and host. + +The configuration file for `aws` should be like the following and be named `.aws.env`: +```.env +DATABRICKS_HOST= +DATABRICKS_TOKEN= +``` + +The configuration file for `azure` should be like the following and be named `.azure.env`: +```.env +DATABRICKS_AZURE_CLIENT_ID= +DATABRICKS_AZURE_CLIENT_SECRET= +DATABRICKS_AZURE_TENANT_ID= +DATABRICKS_AZURE_SUBSCRIPTION_ID= +DATABRICKS_AZURE_RESOURCE_GROUP= +AZURE_REGION= +DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP= +DATABRICKS_AZURE_WORKSPACE_NAME= +``` + +Note that azure integration tests will use service principal based auth. Even though it is using a service principal, +it will still be generating a personal access token to perform creation of resources. + + +## Project Components + +### Databricks Terraform Provider Resources State + +| Resource | Implemented | Import Support | Acceptance Tests | Documentation | Reviewed | Finalize Schema | +|----------------------------------|--------------------|----------------------|----------------------|----------------------|----------------------|----------------------| +| databricks_token | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_secret_scope | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_secret | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_secret_acl | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_instance_pool | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_scim_user | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_scim_group | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_notebook | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_cluster | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_job | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_dbfs_file | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_dbfs_file_sync | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_instance_profile | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_aws_s3_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_azure_blob_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_azure_adls_gen1_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | +| databricks_azure_adls_gen2_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | + +### Databricks Terraform Data Sources State + +| Data Source | Implemented | Acceptance Tests | Documentation | Reviewed | +|-----------------------------|----------------------|----------------------|----------------------|----------------------| +| databricks_notebook | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_notebook_paths | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_dbfs_file | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_dbfs_file_paths | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_zones | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_runtimes | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_instance_pool | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_scim_user | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_scim_group | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_cluster | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_job | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_mount | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_instance_profile | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_database | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | +| databricks_table | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | diff --git a/Makefile b/Makefile index 1965ba6d0d..f603c83153 100644 --- a/Makefile +++ b/Makefile @@ -48,9 +48,6 @@ vendor: @echo "==> Filling vendor folder with library code..." @go mod vendor -local-install: build - mv terraform-provider-databricks $(HOME)/.terraform.d/plugins/terraform-provider-databricks_v0.2.0 - # INTEGRATION TESTING WITH AZURE terraform-acc-azure: lint @echo "==> Running Terraform Acceptance Tests for Azure..." @@ -82,8 +79,4 @@ hugo: @echo "==> Making Docs..." @cd website && hugo -d ../docs/ -internal-docs-sync: - @echo "==> Uploading Website..." - @azcopy login --service-principal --application-id $(AZCOPY_SPA_CLIENT_ID) --tenant-id=$(AZCOPY_SPA_TENANT_ID) && azcopy sync './website/public' '$(AZCOPY_STORAGE_ACCT)' --recursive - .PHONY: build fmt python-setup docs vendor terraform-local build fmt coverage test lint \ No newline at end of file diff --git a/README.md b/README.md index dbf8c3e91c..b772e5dbe0 100644 --- a/README.md +++ b/README.md @@ -2,245 +2,38 @@ [![Build Status](https://travis-ci.org/databrickslabs/terraform-provider-databricks.svg?branch=master)](https://travis-ci.org/databrickslabs/terraform-provider-databricks) -If you are looking for the docs for this provider see: https://databrickslabs.github.io/terraform-provider-databricks/overview/ - - -## Quickstart: Building and Using the Provider - -### Quick install +[Documentation](https://databrickslabs.github.io/terraform-provider-databricks/provider/) | [Contributing and Development Guidelines](CONTRIBUTING.md) To quickly install the binary please execute the following curl command in your shell. ```bash -$ curl https://raw.githubusercontent.com/databrickslabs/databricks-terraform/master/godownloader-databricks-provider.sh | bash -s -- -b $HOME/.terraform.d/plugins +curl https://raw.githubusercontent.com/databrickslabs/databricks-terraform/master/godownloader-databricks-provider.sh | bash -s -- -b $HOME/.terraform.d/plugins ``` -The command should have moved the binary into your `~/.terraform.d/plugins` folder. - -You can `ls` the previous directory to verify. - -### Install using go - -Please note that there is a Makefile which contains all the commands you would need to run this project. - -This code base to contribute to requires the following software (this is also all configured for the [Visual Studio Code Devcontainer](#developing-with-visual-studio-code-devcontainers)): - -* [golang 1.13.X](https://golang.org/dl/) -* [terraform v0.12.x](https://www.terraform.io/downloads.html) -* make command - -To make sure everything is installed correctly please run the following commands: - -Testing go installation: -```bash -$ go version -go version go1.13.3 darwin/amd64 -``` - -Testing terraform installation: -```bash -$ terraform --version -Terraform v0.12.19 - -Your version of Terraform is out of date! The latest version -is 0.12.24. You can update by downloading from https://www.terraform.io/downloads.html - -``` - -Testing make installation: -```bash -$ make --version -GNU Make 3.81 -Copyright (C) 2006 Free Software Foundation, Inc. -This is free software; see the source for copying conditions. -There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. - -This program built for i386-apple-darwin11.3.0 -``` - -### Downloading the source code and installing the artifact - -* After installing `golang`, `terraform`, and `make` you will now build the artifact. - -```bash -$ go get -v -u github.com/databrickslabs/databricks-terraform && cd $GOPATH/src/github.com/databrickslabs/databricks-terraform -``` - -:warning: If you are fetching from a private repository please use the following command: - -```bash -$ GOSUMDB=off GOPROXY=direct go get -v -u github.com/databrickslabs/databricks-terraform && cd $GOPATH/src/github.com/databrickslabs/databricks-terraform -``` - -* When you are in the root directory of the repository please run: - -```bash -$ make build -``` - -* Locate your [terraform plugins directory](https://www.terraform.io/docs/extend/how-terraform-works.html#plugin-locations) - or the root folder of your terraform code - -* Copy the `terraform-provider-databricks` artifact to that terraform plugins locations - -```bash -$ mkdir -p ~/.terraform.d/plugins/ && cp terraform-provider-databricks ~/.terraform.d/plugins/terraform-provider-databricks -``` - -Now your plugin for the Databricks Terraform provider is installed correctly. You can actually use the provider. - -### Basic Terraform example - -Sample terraform code +Then create a small sample file, named `main.tf` with approximately following contents. Replace `` with newly created [PAT Token](https://docs.databricks.com/dev-tools/api/latest/authentication.html). It will create [a simple cluster](https://databrickslabs.github.io/terraform-provider-databricks/resources/cluster/). ```terraform provider "databricks" { - host = "http://databrickshost.com" - token = "dapitokenhere" + host = "https://abc-defg-024.cloud.databricks.com/" + token = "" } -// Creating a basic user -resource "databricks_scim_user" "my-user" { - user_name = join("", ["test-user", "+",count.index,"@databricks.com"]) - display_name = "Test User" -} -``` - -Then run `terraform init` then `terraform apply` to apply the hcl code to your databricks workspace. - -Please refer to the detailed documentation provided in the html documentation for detailed use of the providers. - -Also refer to these [examples](examples/) for more scenarios. - -### Provider Documentation - -Provider documentation can be located in the releases tab and documentation is packaged up along with -the binary of choice. - -### Developing with Visual Studio Code Devcontainers - -This project has configuration for working with [Visual Studio Code Devcontainers](https://code.visualstudio.com/docs/remote/containers) - this allows you to containerise your development prerequisites (e.g. golang, terraform). To use this you will need [Visual Studio Code](https://code.visualstudio.com/) and [Docker](https://www.docker.com/products/docker-desktop). - -To get started, clone this repo and open the folder with Visual Studio Code. If you don't have the [Remote Development extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) then you should be prompted to install it. - -Once the folder is loaded and the extension is installed you should be prompted to re-open the folder in a devcontainer. This will built and run the container image with the correct tools (and versions) ready to start working on and building the code. The in-built terminal will launch a shell inside the container for running `make` commands etc. - -See the docs for more details on working with [devcontainers](https://code.visualstudio.com/docs/remote/containers). - -### Building and Installing with Docker - -To install and build the code if you dont want to install golang, terraform, etc. All you need is docker and git. - -First make sure you clone the repository and you are in the directory. - -Then build the docker image with this command: - -```bash -$ docker build -t databricks-terraform . -``` - -Then run the execute the terraform binary via the following command and volume mount. Make sure that you are in the directory - with the terraform code. The following command you can execute the following commands and additional ones as part of - the terraform binary. - -```bash -$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform init -$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform plan -$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform apply -``` - -## Testing - -* [ ] Integration tests should be run at a client level against both azure and aws to maintain sdk parity against both apis **(currently only on one cloud)** -* [x] Terraform acceptance tests should be run against both aws and azure to maintain parity of provider between both cloud services **(currently only on one cloud)** - -### Linting - -Please use makefile for linting. If you run `golangci-lint` by itself it will fail due to different tags containing same functions. -So please run `make lint` instead. - -### Integration Testing - -Currently Databricks supports two cloud providers `azure` and `aws` thus integration testing with the correct cloud service provider is -crucial for making sure that the provider behaves as expected on all supported clouds. This type of testing separation is being managed via build tags -to allow for duplicate method names and environment variables to configure clients. - -The current integration test implementation uses `CLOUD_ENV` environment variable and can use the value of `azure` or `aws`. -You can execute the acceptance with the following make commands `make terraform-acc-azure`, and `make terraform-acc-aws` for -azure and aws respectively. - -This involves bootstrapping the provider via a .env configuration file. Without these files in the root directory the tests -will fail as the provider will not have a authorized token and host. +resource "databricks_cluster" "shared_autoscaling" { + cluster_name = "Shared Autoscaling" + spark_version = "6.6.x-scala2.11" + node_type_id = "i3.xlarge" + autotermination_minutes = 20 -The configuration file for `aws` should be like the following and be named `.aws.env`: -```.env -DATABRICKS_HOST= -DATABRICKS_TOKEN= -``` - -The configuration file for `azure` should be like the following and be named `.azure.env`: -```.env -DATABRICKS_AZURE_CLIENT_ID= -DATABRICKS_AZURE_CLIENT_SECRET= -DATABRICKS_AZURE_TENANT_ID= -DATABRICKS_AZURE_SUBSCRIPTION_ID= -DATABRICKS_AZURE_RESOURCE_GROUP= -AZURE_REGION= -DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP= -DATABRICKS_AZURE_WORKSPACE_NAME= + autoscale { + min_workers = 1 + max_workers = 50 + } +} ``` -Note that azure integration tests will use service principal based auth. Even though it is using a service principal, -it will still be generating a personal access token to perform creation of resources. - - -## Project Components - -### Databricks Terraform Provider Resources State - -| Resource | Implemented | Import Support | Acceptance Tests | Documentation | Reviewed | Finalize Schema | -|----------------------------------|--------------------|----------------------|----------------------|----------------------|----------------------|----------------------| -| databricks_token | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_secret_scope | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_secret | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_secret_acl | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_instance_pool | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_scim_user | :white_check_mark: | :white_large_square: | :white_check_mark: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_scim_group | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_notebook | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_cluster | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_job | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file_sync | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_instance_profile | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_aws_s3_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_azure_blob_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_azure_adls_gen1_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | -| databricks_azure_adls_gen2_mount | :white_check_mark: | :white_large_square: | :white_large_square: | :white_check_mark: | :white_large_square: | :white_large_square: | - -### Databricks Terraform Data Sources State - -| Data Source | Implemented | Acceptance Tests | Documentation | Reviewed | -|-----------------------------|----------------------|----------------------|----------------------|----------------------| -| databricks_notebook | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_notebook_paths | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_dbfs_file_paths | :white_check_mark: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_zones | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_runtimes | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_instance_pool | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_scim_user | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_scim_group | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_cluster | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_job | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_mount | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_instance_profile | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_database | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | -| databricks_table | :white_large_square: | :white_large_square: | :white_large_square: | :white_large_square: | - +Then run `terraform init` then `terraform apply` to apply the hcl code to your Databricks workspace. Please refer to the [end-user documentation](https://databrickslabs.github.io/terraform-provider-databricks/provider/) for detailed use of the provider. Also refer to these [examples](examples/) for more scenarios. ## Project Support Please note that all projects in the /databrickslabs github account are provided for your exploration only, and are not formally supported by Databricks with Service Level Agreements (SLAs). They are provided AS-IS and we do not make any guarantees of any kind. Please do not submit a support ticket relating to any issues arising from the use of these projects. -Any issues discovered through the use of this project should be filed as GitHub Issues on the Repo. They will be reviewed as time permits, but there are no formal SLAs for support. +Any issues discovered through the use of this project should be filed as GitHub Issues on the Repo. They will be reviewed as time permits, but there are no formal SLAs for support. \ No newline at end of file diff --git a/client/model/token.go b/client/model/token.go index 72e7ade35c..6efaa51a79 100644 --- a/client/model/token.go +++ b/client/model/token.go @@ -1,5 +1,11 @@ package model +// TokenRequest asks for a token +type TokenRequest struct { + LifetimeSeconds int32 `json:"lifetime_seconds,omitempty"` + Comment string `json:"comment,omitempty"` +} + // TokenResponse is a struct that contains information about token that is created from the create tokens api type TokenResponse struct { TokenValue string `json:"token_value,omitempty"` diff --git a/client/service/client.go b/client/service/client.go index 72979ea81e..f80c32bb0f 100644 --- a/client/service/client.go +++ b/client/service/client.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "reflect" + "regexp" "strings" "sync" "time" @@ -28,8 +29,7 @@ const ( Azure CloudServiceProvider = "Azure" ) -// DBApiErrorBody is a struct for a custom api error for all the services on databrickss. -type DBApiErrorBody struct { +type apiErrorBody struct { ErrorCode string `json:"error_code,omitempty"` Message string `json:"message,omitempty"` // The following two are for scim api only for RFC 7644 Section 3.7.3 https://tools.ietf.org/html/rfc7644#section-3.7.3 @@ -37,16 +37,38 @@ type DBApiErrorBody struct { ScimStatus string `json:"status,omitempty"` } -// DBApiError is a generic struct for an api error on databricks -type DBApiError struct { - ErrorBody *DBApiErrorBody +// APIError is a generic struct for an api error on databricks +type APIError struct { + ErrorCode string + Message string + Resource string StatusCode int - Err error } -// Error is a interface implementation of the error interface. -func (r DBApiError) Error() string { - return fmt.Sprintf("status %d: err %v", r.StatusCode, r.Err) +// Error returns error message string instead of +func (apiError APIError) Error() string { + docs := apiError.DocumentationURL() + if docs == "" { + return fmt.Sprintf("%s\n(%d on %s)", apiError.Message, apiError.StatusCode, apiError.Resource) + } + return fmt.Sprintf("%s\nPlease consult API docs at %s for details.", + apiError.Message, + docs) +} + +func (apiError APIError) IsMissing() bool { + return apiError.StatusCode == http.StatusNotFound +} + +// DocumentationURL guesses doc link +func (apiError APIError) DocumentationURL() string { + endpointRE := regexp.MustCompile(`/api/2.0/([^/]+)/([^/]+)$`) + endpointMatches := endpointRE.FindStringSubmatch(apiError.Resource) + if len(endpointMatches) < 3 { + return "" + } + return fmt.Sprintf("https://docs.databricks.com/dev-tools/api/latest/%s.html#%s", + endpointMatches[1], endpointMatches[2]) } // AuthType is a custom type for a type of authentication allowed on Databricks @@ -61,8 +83,12 @@ var clientAuthorizerMutex sync.Mutex // DBApiClientConfig is used to configure the DataBricks Client type DBApiClientConfig struct { - Host string - Token string + Host string + Token string + // new token should be requested from + // the workspace before this time comes + // not yet used in the client but can be set + TokenExpiryTime int64 AuthType AuthType UserAgent string DefaultHeaders map[string]string @@ -128,16 +154,47 @@ func checkHTTPRetry(ctx context.Context, resp *http.Response, err error) (bool, if err != nil { return false, err } - var errorBody DBApiErrorBody + var errorBody apiErrorBody err = json.Unmarshal(body, &errorBody) + // this is most likely HTML... since un-marshalling JSON failed if err != nil { - return false, fmt.Errorf("Response from server (%d) %s: %v", resp.StatusCode, string(body), err) + // Status parts first in case html message is not as expected + statusParts := strings.SplitN(resp.Status, " ", 2) + if len(statusParts) < 2 { + errorBody.ErrorCode = "UNKNOWN" + } else { + errorBody.ErrorCode = strings.ReplaceAll(strings.ToUpper(strings.Trim(statusParts[1], " .")), " ", "_") + } + stringBody := string(body) + messageRE := regexp.MustCompile(`
(.*)
`) + messageMatches := messageRE.FindStringSubmatch(stringBody) + // No messages with
format found so return a default APIError + if len(messageMatches) < 2 { + return false, APIError{ + Message: fmt.Sprintf("Response from server (%d) %s: %v", resp.StatusCode, stringBody, err), + ErrorCode: errorBody.ErrorCode, + StatusCode: resp.StatusCode, + Resource: resp.Request.URL.Path, + } + } + errorBody.Message = strings.Trim(messageMatches[1], " .") } - dbAPIError := DBApiError{ - ErrorBody: &errorBody, + dbAPIError := APIError{ + Message: errorBody.Message, + ErrorCode: errorBody.ErrorCode, StatusCode: resp.StatusCode, - Err: fmt.Errorf("Response from server %s", string(body)), + Resource: resp.Request.URL.Path, + } + // Handle scim error message details + if dbAPIError.Message == "" && errorBody.ScimDetail != "" { + if errorBody.ScimDetail == "null" { + dbAPIError.Message = "SCIM API Internal Error" + } else { + dbAPIError.Message = errorBody.ScimDetail + } + dbAPIError.ErrorCode = fmt.Sprintf("SCIM_%s", errorBody.ScimStatus) } + // Handle transient errors for retries for _, substring := range transientErrorStringMatches { if strings.Contains(errorBody.Message, substring) { log.Println("Failed request detected: Retryable type found. Attempting retry...") diff --git a/client/service/client_test.go b/client/service/client_test.go new file mode 100644 index 0000000000..5da5d7dad8 --- /dev/null +++ b/client/service/client_test.go @@ -0,0 +1,91 @@ +package service + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func ForceErrorServer(t *testing.T, response string, responseStatus int, apiCall func(client DBApiClient) (interface{}, error)) (interface{}, error) { + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rw.WriteHeader(responseStatus) + _, err := rw.Write([]byte(response)) + assert.NoError(t, err, err) + })) + // Close the server when test finishes + defer server.Close() + var config DBApiClientConfig + config.Host = server.URL + config.Setup() + + var dbClient DBApiClient + dbClient.SetConfig(&config) + + return apiCall(dbClient) +} + +func TestClient_HandleErrors(t *testing.T) { + tests := []struct { + name string + response string + responseStatus int + expectedErrorCode string + expectedMessage string + expectedResource string + expectedStatusCode int + apiCall func(client DBApiClient) (interface{}, error) + }{ + { + name: "Status 404", + response: `{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": "Token ... does not exist!" + }`, + responseStatus: http.StatusNotFound, + expectedErrorCode: "RESOURCE_DOES_NOT_EXIST", + expectedMessage: "Token ... does not exist!", + expectedResource: "/api/2.0/token/create", + expectedStatusCode: 404, + apiCall: func(client DBApiClient) (interface{}, error) { + return client.Tokens().Create(10, "USERS") + }, + }, + { + name: "HTML Status 404", + response: `
 Hello world
`, + responseStatus: http.StatusNotFound, + expectedErrorCode: "NOT_FOUND", + expectedMessage: "Hello world", + expectedResource: "/api/2.0/token/create", + expectedStatusCode: 404, + apiCall: func(client DBApiClient) (interface{}, error) { + return client.Tokens().Create(10, "USERS") + }, + }, + { + name: "Invalid HTML Status 404", + response: ` Hello world `, + responseStatus: http.StatusNotFound, + expectedErrorCode: "NOT_FOUND", + expectedMessage: "Response from server (404) Hello world : invalid character '<' looking for beginning of value", + expectedResource: "/api/2.0/token/create", + expectedStatusCode: 404, + apiCall: func(client DBApiClient) (interface{}, error) { + return client.Tokens().Create(10, "USERS") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ForceErrorServer(t, tt.response, tt.responseStatus, tt.apiCall) + t.Log(err) + assert.IsType(t, APIError{}, err) + assert.Equal(t, tt.expectedErrorCode, err.(APIError).ErrorCode, "error code is not the same") + assert.Equal(t, tt.expectedMessage, err.(APIError).Message, "message is not the same") + assert.Equal(t, tt.expectedResource, err.(APIError).Resource, "resource is not the same") + assert.Equal(t, tt.expectedStatusCode, err.(APIError).StatusCode, "status code is not the same") + }) + } +} diff --git a/client/service/instance_profiles.go b/client/service/instance_profiles.go index daa96d33cb..5a39fc8dae 100644 --- a/client/service/instance_profiles.go +++ b/client/service/instance_profiles.go @@ -40,8 +40,13 @@ func (a InstanceProfilesAPI) Read(instanceProfileARN string) (string, error) { } } - return response, fmt.Errorf("Instance profile with name: %s not found in "+ - "list of instance profiles in the workspace!", instanceProfileARN) + return response, APIError{ + ErrorCode: "NOT_FOUND", + Message: fmt.Sprintf("Instance profile with name: %s not found in "+ + "list of instance profiles in the workspace!", instanceProfileARN), + Resource: "/api/2.0/instance-profiles/list", + StatusCode: http.StatusNotFound, + } } // List lists all the instance profiles in the workspace diff --git a/client/service/secret_scopes.go b/client/service/secret_scopes.go index 58031ecc96..641fc516c3 100644 --- a/client/service/secret_scopes.go +++ b/client/service/secret_scopes.go @@ -64,5 +64,10 @@ func (a SecretScopesAPI) Read(scopeName string) (model.SecretScope, error) { return scope, nil } } - return secretScope, fmt.Errorf("no Secret Scope found with scope name %s", scopeName) + return secretScope, APIError{ + ErrorCode: "NOT_FOUND", + Message: fmt.Sprintf("no Secret Scope found with scope name %s", scopeName), + Resource: "/api/2.0/secrets/scopes/list", + StatusCode: http.StatusNotFound, + } } diff --git a/client/service/secrets.go b/client/service/secrets.go index 4fa918d50c..74331d6407 100644 --- a/client/service/secrets.go +++ b/client/service/secrets.go @@ -74,5 +74,10 @@ func (a SecretsAPI) Read(scope string, key string) (model.SecretMetadata, error) return secret, nil } } - return secretMeta, fmt.Errorf("no Secret Scope found with secret metadata scope name: %s and key: %s", scope, key) + return secretMeta, APIError{ + ErrorCode: "NOT_FOUND", + Message: fmt.Sprintf("no secret Scope found with secret metadata scope name: %s and key: %s", scope, key), + Resource: "/api/2.0/secrets/scopes/list", + StatusCode: http.StatusNotFound, + } } diff --git a/client/service/tokens.go b/client/service/tokens.go index beea0c37b8..d2678b2d4f 100644 --- a/client/service/tokens.go +++ b/client/service/tokens.go @@ -17,12 +17,9 @@ type TokensAPI struct { func (a TokensAPI) Create(lifeTimeSeconds int32, comment string) (model.TokenResponse, error) { var tokenData model.TokenResponse - tokenCreateRequest := struct { - LifetimeSeconds int32 `json:"lifetime_seconds,omitempty"` - Comment string `json:"comment,omitempty"` - }{ - lifeTimeSeconds, - comment, + tokenCreateRequest := model.TokenRequest{ + LifetimeSeconds: lifeTimeSeconds, + Comment: comment, } tokenCreateResponse, err := a.Client.performQuery(http.MethodPost, "/token/create", "2.0", nil, tokenCreateRequest, nil) @@ -59,7 +56,12 @@ func (a TokensAPI) Read(tokenID string) (model.TokenInfo, error) { return tokenInfoRecord, nil } } - return tokenInfo, fmt.Errorf("Unable to locate token: %s", tokenID) + return tokenInfo, APIError{ + ErrorCode: "NOT_FOUND", + Message: fmt.Sprintf("Unable to locate token: %s", tokenID), + Resource: "/api/2.0/token/list", + StatusCode: http.StatusNotFound, + } } // Delete will delete the token given a token id diff --git a/databricks/azure_auth.go b/databricks/azure_auth.go index d1c1af1742..ba2bae5718 100644 --- a/databricks/azure_auth.go +++ b/databricks/azure_auth.go @@ -5,10 +5,12 @@ import ( "fmt" "log" "net/http" + urlParse "net/url" "time" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" + "github.com/databrickslabs/databricks-terraform/client/model" "github.com/databrickslabs/databricks-terraform/client/service" ) @@ -100,11 +102,12 @@ func (t *TokenPayload) getManagementToken() (string, error) { func (t *TokenPayload) getWorkspace(config *service.DBApiClientConfig, managementToken string) (*Workspace, error) { log.Println("[DEBUG] Getting Workspace ID via management token.") + // Escape all the ids url := fmt.Sprintf("https://management.azure.com/subscriptions/%s/resourceGroups/%s"+ "/providers/Microsoft.Databricks/workspaces/%s", - t.SubscriptionID, - t.ResourceGroup, - t.WorkspaceName) + urlParse.PathEscape(t.SubscriptionID), + urlParse.PathEscape(t.ResourceGroup), + urlParse.PathEscape(t.WorkspaceName)) headers := map[string]string{ "Content-Type": "application/json", "cache-control": "no-cache", @@ -154,17 +157,9 @@ func (t *TokenPayload) getADBPlatformToken() (string, error) { return platformToken.OAuthToken(), nil } -func getWorkspaceAccessToken(config *service.DBApiClientConfig, managementToken, adbWorkspaceUrl, adbWorkspaceResourceID, adbPlatformToken string) (string, error) { +func getWorkspaceAccessToken(config *service.DBApiClientConfig, managementToken, adbWorkspaceUrl, adbWorkspaceResourceID, adbPlatformToken string) (*model.TokenResponse, error) { log.Println("[DEBUG] Creating workspace token") - apiLifeTimeInSeconds := int32(3600) - comment := "Secret made via SP" - payload := struct { - LifetimeSeconds int32 `json:"lifetime_seconds,omitempty"` - Comment string `json:"comment,omitempty"` - }{ - apiLifeTimeInSeconds, - comment, - } + url := adbWorkspaceUrl + "/api/2.0/token/create" headers := map[string]string{ "Content-Type": "application/x-www-form-urlencoded", "X-Databricks-Azure-Workspace-Resource-Id": adbWorkspaceResourceID, @@ -173,19 +168,20 @@ func getWorkspaceAccessToken(config *service.DBApiClientConfig, managementToken, "Authorization": "Bearer " + adbPlatformToken, } - url := adbWorkspaceUrl + "/api/2.0/token/create" - resp, err := service.PerformQuery(config, http.MethodPost, url, "2.0", headers, true, true, payload, nil) + var tokenResponse model.TokenResponse + resp, err := service.PerformQuery(config, http.MethodPost, url, "2.0", + headers, true, true, model.TokenRequest{ + LifetimeSeconds: int32(3600), + Comment: "Secret made via SP", + }, nil) if err != nil { - return "", err + return nil, err } - - var responseMap map[string]interface{} - err = json.Unmarshal(resp, &responseMap) + err = json.Unmarshal(resp, &tokenResponse) if err != nil { - return "", err + return nil, err } - - return responseMap["token_value"].(string), nil + return &tokenResponse, nil } // Main function call that gets made and it follows 4 steps at the moment: @@ -214,13 +210,16 @@ func (t *TokenPayload) initWorkspaceAndGetClient(config *service.DBApiClientConf } // Get workspace personal access token - workspaceAccessToken, err := getWorkspaceAccessToken(config, managementToken, adbWorkspaceUrl, adbWorkspace.ID, adbPlatformToken) + workspaceAccessTokenResp, err := getWorkspaceAccessToken(config, managementToken, adbWorkspaceUrl, adbWorkspace.ID, adbPlatformToken) if err != nil { return err } config.Host = adbWorkspaceUrl - config.Token = workspaceAccessToken + config.Token = workspaceAccessTokenResp.TokenValue + if workspaceAccessTokenResp.TokenInfo != nil { + config.TokenExpiryTime = workspaceAccessTokenResp.TokenInfo.ExpiryTime + } return nil } diff --git a/databricks/azure_auth_test.go b/databricks/azure_auth_test.go index 7253bad34b..e7558af0b1 100644 --- a/databricks/azure_auth_test.go +++ b/databricks/azure_auth_test.go @@ -1,8 +1,6 @@ package databricks import ( - "fmt" - "os" "testing" "github.com/databrickslabs/databricks-terraform/client/model" @@ -54,10 +52,3 @@ func TestAzureAuthCreateApiToken(t *testing.T) { assert.NoError(t, instancePoolErr, instancePoolErr) } - -// getAndAssertEnv fetches the env for testing and also asserts that the env value is not Zero i.e "" -func getAndAssertEnv(t *testing.T, key string) string { - value, present := os.LookupEnv(key) - assert.True(t, present, fmt.Sprintf("Env variable %s is not set", key)) - return value -} diff --git a/databricks/provider.go b/databricks/provider.go index 7835741c97..eeef5a3c35 100644 --- a/databricks/provider.go +++ b/databricks/provider.go @@ -95,12 +95,12 @@ func Provider(version string) terraform.ResourceProvider { Description: "Location of the Databricks CLI credentials file, that is created\n" + "by `databricks configure --token` command. By default, it is located\n" + "in ~/.databrickscfg. Check https://docs.databricks.com/dev-tools/cli/index.html#set-up-authentication for docs. Config\n" + - "file credetials will only be used when host/token are not provided.", + "file credentials will only be used when host/token are not provided.", }, "profile": { - Type: schema.TypeString, - Optional: true, - Default: "DEFAULT", + Type: schema.TypeString, + Optional: true, + DefaultFunc: schema.EnvDefaultFunc("DATABRICKS_CONFIG_PROFILE", "DEFAULT"), Description: "Connection profile specified within ~/.databrickscfg. Please check\n" + "https://docs.databricks.com/dev-tools/cli/index.html#connection-profiles for documentation.", }, @@ -271,20 +271,25 @@ func tryDatabricksCliConfigFile(d *schema.ResourceData, config *service.DBApiCli "Please check https://docs.databricks.com/dev-tools/cli/index.html#set-up-authentication for details", configFile) } if profile, ok := d.GetOk("profile"); ok { - dbcliConfig := cfg.Section(profile.(string)) - token := dbcliConfig.Key("token").String() - if "" == token { - return fmt.Errorf("config file %s is corrupt: cannot find token in %s profile", + dbcli := cfg.Section(profile.(string)) + config.Host = dbcli.Key("host").String() + if config.Host == "" { + return fmt.Errorf("config file %s is corrupt: cannot find host in %s profile", configFile, profile) } - config.Token = token - - host := dbcliConfig.Key("host").String() - if "" == host { - return fmt.Errorf("config file %s is corrupt: cannot find host in %s profile", + if dbcli.HasKey("username") && dbcli.HasKey("password") { + username := dbcli.Key("username").String() + password := dbcli.Key("password").String() + tokenUnB64 := fmt.Sprintf("%s:%s", username, password) + config.Token = base64.StdEncoding.EncodeToString([]byte(tokenUnB64)) + config.AuthType = service.BasicAuth + } else { + config.Token = dbcli.Key("token").String() + } + if config.Token == "" { + return fmt.Errorf("config file %s is corrupt: cannot find token in %s profile", configFile, profile) } - config.Host = host } return nil diff --git a/databricks/provider_test.go b/databricks/provider_test.go index 4840607d58..6461107516 100644 --- a/databricks/provider_test.go +++ b/databricks/provider_test.go @@ -40,6 +40,18 @@ func init() { } } +// getIntegrationDBAPIClient gets the client given CLOUD_ENV as those env variables get loaded +func getIntegrationDBAPIClient(t *testing.T) *service.DBApiClient { + var config service.DBApiClientConfig + config.Token = getAndAssertEnv(t, "DATABRICKS_TOKEN") + config.Host = getAndAssertEnv(t, "DATABRICKS_HOST") + config.Setup() + + var c service.DBApiClient + c.SetConfig(&config) + return &c +} + func getMWSClient() *service.DBApiClient { // Configure MWS Provider mwsHost := os.Getenv("DATABRICKS_MWS_HOST") @@ -59,6 +71,7 @@ func getMWSClient() *service.DBApiClient { } func TestMain(m *testing.M) { + // This should not be asserted as it may not always be set for all tests cloudEnv := os.Getenv("CLOUD_ENV") envFileName := fmt.Sprintf("../.%s.env", cloudEnv) err := godotenv.Load(envFileName) @@ -198,3 +211,10 @@ func TestAccDatabricksCliConfigWorks(t *testing.T) { }, ) } + +// getAndAssertEnv fetches the env for testing and also asserts that the env value is not Zero i.e "" +func getAndAssertEnv(t *testing.T, key string) string { + value, present := os.LookupEnv(key) + assert.True(t, present, fmt.Sprintf("Env variable %s is not set", key)) + return value +} diff --git a/databricks/resource_databricks_azure_adls_gen1_mount.go b/databricks/resource_databricks_azure_adls_gen1_mount.go index 385e95e4f0..d2df96d050 100644 --- a/databricks/resource_databricks_azure_adls_gen1_mount.go +++ b/databricks/resource_databricks_azure_adls_gen1_mount.go @@ -140,7 +140,7 @@ func resourceAzureAdlsGen1Read(d *schema.ResourceData, m interface{}) error { clientSecretKey := d.Get("client_secret_key").(string) err := changeClusterIntoRunningState(clusterID, client) if err != nil { - if isClusterMissing(err.Error(), clusterID) { + if isClusterMissing(err, clusterID) { log.Printf("Unable to refresh mount '%s' as cluster '%s' is missing", mountName, clusterID) d.SetId("") return nil diff --git a/databricks/resource_databricks_azure_adls_gen2_mount.go b/databricks/resource_databricks_azure_adls_gen2_mount.go index 3f63c4ed26..c2d3cd5388 100644 --- a/databricks/resource_databricks_azure_adls_gen2_mount.go +++ b/databricks/resource_databricks_azure_adls_gen2_mount.go @@ -144,7 +144,7 @@ func resourceAzureAdlsGen2Read(d *schema.ResourceData, m interface{}) error { err := changeClusterIntoRunningState(clusterID, client) if err != nil { - if isClusterMissing(err.Error(), clusterID) { + if isClusterMissing(err, clusterID) { log.Printf("Unable to refresh mount '%s' as cluster '%s' is missing", mountName, clusterID) d.SetId("") return nil diff --git a/databricks/resource_databricks_azure_blob_mount.go b/databricks/resource_databricks_azure_blob_mount.go index a23a5c7489..50a35c3953 100644 --- a/databricks/resource_databricks_azure_blob_mount.go +++ b/databricks/resource_databricks_azure_blob_mount.go @@ -126,7 +126,7 @@ func resourceAzureBlobMountRead(d *schema.ResourceData, m interface{}) error { err := changeClusterIntoRunningState(clusterID, client) if err != nil { - if isClusterMissing(err.Error(), clusterID) { + if isClusterMissing(err, clusterID) { log.Printf("Unable to refresh mount '%s' as cluster '%s' is missing", mountName, clusterID) d.SetId("") return nil diff --git a/databricks/resource_databricks_cluster.go b/databricks/resource_databricks_cluster.go index c7cdd13de1..b68dca6d0b 100644 --- a/databricks/resource_databricks_cluster.go +++ b/databricks/resource_databricks_cluster.go @@ -2,6 +2,7 @@ package databricks import ( "errors" + "fmt" "log" "strings" "time" @@ -542,7 +543,7 @@ func resourceClusterRead(d *schema.ResourceData, m interface{}) error { clusterInfo, err := client.Clusters().Get(id) if err != nil { - if isClusterMissing(err.Error(), id) { + if isClusterMissing(err, id) { log.Printf("Missing cluster with id: %s.", id) d.SetId("") return nil @@ -1452,3 +1453,9 @@ func getMapFromOneItemSet(input interface{}) map[string]interface{} { } return nil } + +func isClusterMissing(err error, resourceID string) bool { + apiErr, ok := err.(service.APIError) + return (ok && apiErr.IsMissing()) || + strings.Contains(err.Error(), fmt.Sprintf("Cluster %s does not exist", resourceID)) +} diff --git a/databricks/resource_databricks_cluster_policy.go b/databricks/resource_databricks_cluster_policy.go index e8a4bb7d61..ebd4aebea2 100644 --- a/databricks/resource_databricks_cluster_policy.go +++ b/databricks/resource_databricks_cluster_policy.go @@ -1,6 +1,8 @@ package databricks import ( + "log" + "github.com/hashicorp/terraform-plugin-sdk/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/helper/validation" @@ -37,6 +39,11 @@ func resourceClusterPolicyCreate(d *schema.ResourceData, m interface{}) error { func resourceClusterPolicyRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) clusterPolicy, err := client.ClusterPolicies().Get(d.Id()) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) + d.SetId("") + return nil + } if err != nil { return err } diff --git a/databricks/resource_databricks_cluster_policy_test.go b/databricks/resource_databricks_cluster_policy_test.go index 7973d5e621..e636ece44d 100644 --- a/databricks/resource_databricks_cluster_policy_test.go +++ b/databricks/resource_databricks_cluster_policy_test.go @@ -29,14 +29,11 @@ func TestAccClusterPolicyResourceFullLifecycle(t *testing.T) { return err } policy = *resp + if policy.Definition == "" { + return fmt.Errorf("Empty policy definition found") + } return nil }), - func(s *terraform.State) error { - if policy.Definition == "" { - return fmt.Errorf("Empty policy definition found") - } - return nil - }, resource.TestCheckResourceAttr("databricks_cluster_policy.external_metastore", "name", fmt.Sprintf("Terraform policy %s", randomName)), ), @@ -59,6 +56,18 @@ func TestAccClusterPolicyResourceFullLifecycle(t *testing.T) { return nil }), }, + { + // and create it again + Config: testExternalMetastore(randomName + ": UPDATED"), + Check: testAccIDCallback(t, "databricks_cluster_policy.external_metastore", + func(client *service.DBApiClient, id string) error { + _, err := client.ClusterPolicies().Get(id) + if err != nil { + return err + } + return nil + }), + }, }, }) } diff --git a/databricks/resource_databricks_dbfs_file.go b/databricks/resource_databricks_dbfs_file.go index 01417c2a9c..2e451205b4 100644 --- a/databricks/resource_databricks_dbfs_file.go +++ b/databricks/resource_databricks_dbfs_file.go @@ -1,10 +1,8 @@ package databricks import ( - "fmt" "log" "path/filepath" - "strings" "github.com/databrickslabs/databricks-terraform/client/service" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" @@ -93,8 +91,8 @@ func resourceDBFSFileRead(d *schema.ResourceData, m interface{}) error { fileInfo, err := client.DBFS().Status(id) if err != nil { - if isDBFSFileMissing(err.Error(), id) { - log.Printf("Missing dbfs file with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -144,8 +142,3 @@ func resourceDBFSFileDelete(d *schema.ResourceData, m interface{}) error { err := client.DBFS().Delete(id, false) return err } - -func isDBFSFileMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") && - strings.Contains(errorMsg, fmt.Sprintf("No file or directory exists on path %s.", resourceID)) -} diff --git a/databricks/resource_databricks_dbfs_file_sync.go b/databricks/resource_databricks_dbfs_file_sync.go index c3b3a66e40..b35224b678 100644 --- a/databricks/resource_databricks_dbfs_file_sync.go +++ b/databricks/resource_databricks_dbfs_file_sync.go @@ -14,7 +14,8 @@ func resourceDBFSFileSync() *schema.Resource { Read: resourceDBFSFileSyncRead, Delete: resourceDBFSFileSyncDelete, Update: resourceDBFSFileSyncUpdate, - + DeprecationMessage: "this is not really a valid resource and can cause issues in maintenance as it is not " + + "valid resource supportable by databricks rest apis", Schema: map[string]*schema.Schema{ "src_path": { Type: schema.TypeString, diff --git a/databricks/resource_databricks_group.go b/databricks/resource_databricks_group.go index 8cb20355f2..99b12e8025 100644 --- a/databricks/resource_databricks_group.go +++ b/databricks/resource_databricks_group.go @@ -64,8 +64,8 @@ func resourceGroupRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) group, err := client.Groups().Read(id) if err != nil { - if isScimGroupMissing(err.Error(), id) { - log.Printf("Missing scim group with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } diff --git a/databricks/resource_databricks_group_aws_test.go b/databricks/resource_databricks_group_aws_test.go index fceffef7af..523dafc543 100644 --- a/databricks/resource_databricks_group_aws_test.go +++ b/databricks/resource_databricks_group_aws_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAccAWSGroupResource(t *testing.T) { +func TestAccAwsGroupResource(t *testing.T) { var Group model.Group // generate a random name for each tokenInfo test run, to avoid // collisions from multiple concurrent tests. @@ -58,7 +58,7 @@ func TestAccAWSGroupResource(t *testing.T) { }) } -func TestAccAWSGroupResource_verify_entitlements(t *testing.T) { +func TestAccAwsGroupResource_verify_entitlements(t *testing.T) { var Group model.Group // generate a random name for each tokenInfo test run, to avoid // collisions from multiple concurrent tests. diff --git a/databricks/resource_databricks_group_instance_profile.go b/databricks/resource_databricks_group_instance_profile.go index aed1372943..424804cdec 100644 --- a/databricks/resource_databricks_group_instance_profile.go +++ b/databricks/resource_databricks_group_instance_profile.go @@ -59,8 +59,8 @@ func resourceGroupInstanceProfileRead(d *schema.ResourceData, m interface{}) err // First verify if the group exists if err != nil { - if isScimGroupMissing(err.Error(), groupInstanceProfileID.GroupID) { - log.Printf("Missing group with id: %s.", groupInstanceProfileID.GroupID) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } diff --git a/databricks/resource_databricks_group_instance_profile_aws_test.go b/databricks/resource_databricks_group_instance_profile_aws_test.go index 30cf732e52..6f56a2d54a 100644 --- a/databricks/resource_databricks_group_instance_profile_aws_test.go +++ b/databricks/resource_databricks_group_instance_profile_aws_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAccAWSGroupInstanceProfileResource(t *testing.T) { +func TestAccAwsGroupInstanceProfileResource(t *testing.T) { var group model.Group // generate a random name for each tokenInfo test run, to avoid // collisions from multiple concurrent tests. diff --git a/databricks/resource_databricks_group_member.go b/databricks/resource_databricks_group_member.go index 3e59ded94e..61d965af3f 100644 --- a/databricks/resource_databricks_group_member.go +++ b/databricks/resource_databricks_group_member.go @@ -60,8 +60,8 @@ func resourceGroupMemberRead(d *schema.ResourceData, m interface{}) error { // First verify if the group exists if err != nil { - if isScimGroupMissing(err.Error(), id) { - log.Printf("Missing group with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } diff --git a/databricks/resource_databricks_group_member_aws_test.go b/databricks/resource_databricks_group_member_aws_test.go index 7fdcb260c7..6fd7d2d837 100644 --- a/databricks/resource_databricks_group_member_aws_test.go +++ b/databricks/resource_databricks_group_member_aws_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAccAWSGroupMemberResource(t *testing.T) { +func TestAccAwsGroupMemberResource(t *testing.T) { var group model.Group // generate a random name for each tokenInfo test run, to avoid // collisions from multiple concurrent tests. diff --git a/databricks/resource_databricks_instance_pool.go b/databricks/resource_databricks_instance_pool.go index 16c035b0ea..a335f5abf6 100644 --- a/databricks/resource_databricks_instance_pool.go +++ b/databricks/resource_databricks_instance_pool.go @@ -1,9 +1,7 @@ package databricks import ( - "fmt" "log" - "strings" "time" "github.com/databrickslabs/databricks-terraform/client/model" @@ -239,8 +237,8 @@ func resourceInstancePoolRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) instancePoolInfo, err := client.InstancePools().Read(id) if err != nil { - if isInstancePoolMissing(err.Error(), id) { - log.Printf("Missing instance pool with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -356,8 +354,3 @@ func resourceInstancePoolDelete(d *schema.ResourceData, m interface{}) error { err := client.InstancePools().Delete(id) return err } - -func isInstancePoolMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") && - strings.Contains(errorMsg, fmt.Sprintf("Can't find an instance pool with id: %s", resourceID)) -} diff --git a/databricks/resource_databricks_instance_profile.go b/databricks/resource_databricks_instance_profile.go index 54917d300a..0f64c57f65 100644 --- a/databricks/resource_databricks_instance_profile.go +++ b/databricks/resource_databricks_instance_profile.go @@ -1,9 +1,7 @@ package databricks import ( - "fmt" "log" - "strings" "github.com/databrickslabs/databricks-terraform/client/service" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" @@ -52,8 +50,8 @@ func resourceInstanceProfileRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) profile, err := client.InstanceProfiles().Read(id) if err != nil { - if isInstanceProfileMissing(err.Error(), id) { - log.Printf("Missing instance profile with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -69,8 +67,3 @@ func resourceInstanceProfileDelete(d *schema.ResourceData, m interface{}) error err := client.InstanceProfiles().Delete(id) return err } - -func isInstanceProfileMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, fmt.Sprintf("Instance profile with name: %s not found in "+ - "list of instance profiles in the workspace!", resourceID)) -} diff --git a/databricks/resource_databricks_instance_profile_test.go b/databricks/resource_databricks_instance_profile_test.go new file mode 100644 index 0000000000..9f6b7aea56 --- /dev/null +++ b/databricks/resource_databricks_instance_profile_test.go @@ -0,0 +1,135 @@ +package databricks + +import ( + "errors" + "fmt" + "testing" + + "github.com/databrickslabs/databricks-terraform/client/model" + "github.com/databrickslabs/databricks-terraform/client/service" + "github.com/hashicorp/terraform-plugin-sdk/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/terraform" + "github.com/stretchr/testify/assert" +) + +func TestAccAwsInstanceProfileResource(t *testing.T) { + var InstanceProfile model.InstanceProfileInfo + // generate a random name for each tokenInfo test run, to avoid + // collisions from multiple concurrent tests. + // the acctest package includes many helpers such as RandStringFromCharSet + // See https://godoc.org/github.com/hashicorp/terraform-plugin-sdk/helper/acctest + randomStr := acctest.RandStringFromCharSet(5, acctest.CharSetAlphaNum) + instanceProfile := fmt.Sprintf("arn:aws:iam::999999999999:instance-profile/%s", randomStr) + resource.Test(t, resource.TestCase{ + Providers: testAccProviders, + CheckDestroy: testAWSInstanceProfileResourceDestroy, + Steps: []resource.TestStep{ + { + // use a dynamic configuration with the random name from above + Config: testAWSDatabricksInstanceProfile(instanceProfile), + // compose a basic test, checking both remote and local values + Check: resource.ComposeTestCheckFunc( + // query the API to retrieve the tokenInfo object + testAWSInstanceProfileResourceExists("databricks_instance_profile.my_instance_profile", &InstanceProfile, t), + // verify remote values + testAWSInstanceProfileValues(t, &InstanceProfile, instanceProfile), + // verify local values + resource.TestCheckResourceAttr("databricks_instance_profile.my_instance_profile", + "instance_profile_arn", instanceProfile), + ), + Destroy: false, + }, + { + PreConfig: func() { + client := testAccProvider.Meta().(*service.DBApiClient) + err := client.InstanceProfiles().Delete(instanceProfile) + assert.NoError(t, err, err) + }, + // use a dynamic configuration with the random name from above + Config: testAWSDatabricksInstanceProfile(instanceProfile), + // compose a basic test, checking both remote and local values + Check: resource.ComposeTestCheckFunc( + // query the API to retrieve the tokenInfo object + testAWSInstanceProfileResourceExists("databricks_instance_profile.my_instance_profile", &InstanceProfile, t), + // verify remote values + testAWSInstanceProfileValues(t, &InstanceProfile, instanceProfile), + // verify local values + resource.TestCheckResourceAttr("databricks_instance_profile.my_instance_profile", + "instance_profile_arn", instanceProfile), + ), + PlanOnly: true, + ExpectNonEmptyPlan: true, + Destroy: false, + }, + { + // use a dynamic configuration with the random name from above + Config: testAWSDatabricksInstanceProfile(instanceProfile), + // compose a basic test, checking both remote and local values + Check: resource.ComposeTestCheckFunc( + // query the API to retrieve the tokenInfo object + testAWSInstanceProfileResourceExists("databricks_instance_profile.my_instance_profile", &InstanceProfile, t), + // verify remote values + testAWSInstanceProfileValues(t, &InstanceProfile, instanceProfile), + // verify local values + resource.TestCheckResourceAttr("databricks_instance_profile.my_instance_profile", + "instance_profile_arn", instanceProfile), + ), + Destroy: false, + }, + }, + }) +} + +func testAWSInstanceProfileResourceDestroy(s *terraform.State) error { + client := testAccProvider.Meta().(*service.DBApiClient) + for _, rs := range s.RootModule().Resources { + if rs.Type != "databricks_instance_profile" { + continue + } + _, err := client.InstanceProfiles().Read(rs.Primary.ID) + if err != nil { + return nil + } + return errors.New("resource InstanceProfile is not cleaned up") + } + return nil +} + +func testAWSInstanceProfileValues(t *testing.T, instanceProfileInfo *model.InstanceProfileInfo, instanceProfile string) resource.TestCheckFunc { + return func(s *terraform.State) error { + assert.True(t, instanceProfileInfo.InstanceProfileArn == instanceProfile) + return nil + } +} + +// testAccCheckTokenResourceExists queries the API and retrieves the matching Widget. +func testAWSInstanceProfileResourceExists(n string, instanceProfileInfo *model.InstanceProfileInfo, t *testing.T) resource.TestCheckFunc { + return func(s *terraform.State) error { + // find the corresponding state object + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + // retrieve the configured client from the test setup + conn := testAccProvider.Meta().(*service.DBApiClient) + resp, err := conn.InstanceProfiles().Read(rs.Primary.ID) + if err != nil { + return err + } + + // If no error, assign the response Widget attribute to the widget pointer + *instanceProfileInfo = model.InstanceProfileInfo{InstanceProfileArn: resp} + return nil + } +} + +func testAWSDatabricksInstanceProfile(instanceProfile string) string { + return fmt.Sprintf(` + resource "databricks_instance_profile" "my_instance_profile" { + instance_profile_arn = "%s" + skip_validation = true + } + `, instanceProfile) +} diff --git a/databricks/resource_databricks_job.go b/databricks/resource_databricks_job.go index eb250b35a5..c48193cd15 100644 --- a/databricks/resource_databricks_job.go +++ b/databricks/resource_databricks_job.go @@ -534,7 +534,7 @@ func resourceJobRead(d *schema.ResourceData, m interface{}) error { job, err := client.Jobs().Read(idInt) if err != nil { - if isJobMissing(err.Error(), id) { + if isJobMissing(err, id) { log.Printf("Missing job with id: %s.", id) d.SetId("") return nil @@ -1192,7 +1192,9 @@ func parseSchemaToLibraries(d *schema.ResourceData) []model.Library { return libraryList } -func isJobMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "INVALID_PARAMETER_VALUE") && - strings.Contains(errorMsg, fmt.Sprintf("Job %s does not exist.", resourceID)) +// Required as jobs do not return 404 not found +func isJobMissing(err error, resourceID string) bool { + apiErr, ok := err.(service.APIError) + return (ok && apiErr.IsMissing()) || + strings.Contains(err.Error(), fmt.Sprintf("Job %s does not exist.", resourceID)) } diff --git a/databricks/resource_databricks_mws_credentials.go b/databricks/resource_databricks_mws_credentials.go index 648fe37cb5..ea0675e95b 100644 --- a/databricks/resource_databricks_mws_credentials.go +++ b/databricks/resource_databricks_mws_credentials.go @@ -5,7 +5,6 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/helper/schema" "log" - "strings" ) func resourceMWSCredentials() *schema.Resource { @@ -73,8 +72,8 @@ func resourceMWSCredentialsRead(d *schema.ResourceData, m interface{}) error { } credentials, err := client.MWSCredentials().Read(packagedMwsID.MwsAcctID, packagedMwsID.ResourceID) if err != nil { - if isMWSCredentialsMissing(err.Error()) { - log.Printf("Missing e2 credentials with id: %s.", packagedMwsID.ResourceID) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -113,7 +112,3 @@ func resourceMWSCredentialsDelete(d *schema.ResourceData, m interface{}) error { err = client.MWSCredentials().Delete(packagedMwsID.MwsAcctID, packagedMwsID.ResourceID) return err } - -func isMWSCredentialsMissing(errorMsg string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") -} diff --git a/databricks/resource_databricks_mws_networks.go b/databricks/resource_databricks_mws_networks.go index b3c0963b73..18eb417f8c 100644 --- a/databricks/resource_databricks_mws_networks.go +++ b/databricks/resource_databricks_mws_networks.go @@ -3,7 +3,6 @@ package databricks import ( "log" "reflect" - "strings" "github.com/databrickslabs/databricks-terraform/client/model" "github.com/databrickslabs/databricks-terraform/client/service" @@ -116,8 +115,8 @@ func resourceMWSNetworkRead(d *schema.ResourceData, m interface{}) error { } network, err := client.MWSNetworks().Read(packagedMwsID.MwsAcctID, packagedMwsID.ResourceID) if err != nil { - if isMWSNetworkMissing(err.Error()) { - log.Printf("Missing e2 network with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -192,7 +191,3 @@ func convertErrorMessagesToListOfMaps(errorMsgs []model.NetworkHealth) []map[str } return resp } - -func isMWSNetworkMissing(errorMsg string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") -} diff --git a/databricks/resource_databricks_mws_storage_configurations.go b/databricks/resource_databricks_mws_storage_configurations.go index 19ff4c3626..2142f68c34 100644 --- a/databricks/resource_databricks_mws_storage_configurations.go +++ b/databricks/resource_databricks_mws_storage_configurations.go @@ -2,7 +2,6 @@ package databricks import ( "log" - "strings" "github.com/databrickslabs/databricks-terraform/client/service" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" @@ -69,8 +68,8 @@ func resourceMWSStorageConfigurationsRead(d *schema.ResourceData, m interface{}) } storageConifiguration, err := client.MWSStorageConfigurations().Read(packagedMwsID.MwsAcctID, packagedMwsID.ResourceID) if err != nil { - if isE2StorageConfigurationsMissing(err.Error()) { - log.Printf("Missing mws storage configurations with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -110,7 +109,3 @@ func resourceMWSStorageConfigurationsDelete(d *schema.ResourceData, m interface{ err = client.MWSStorageConfigurations().Delete(packagedMwsID.MwsAcctID, packagedMwsID.ResourceID) return err } - -func isE2StorageConfigurationsMissing(errorMsg string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") -} diff --git a/databricks/resource_databricks_mws_workspaces.go b/databricks/resource_databricks_mws_workspaces.go index 2f22cea10c..b5ac7301d3 100644 --- a/databricks/resource_databricks_mws_workspaces.go +++ b/databricks/resource_databricks_mws_workspaces.go @@ -7,12 +7,13 @@ import ( "github.com/databrickslabs/databricks-terraform/client/model" "github.com/databrickslabs/databricks-terraform/client/service" + "github.com/hashicorp/terraform-plugin-sdk/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/helper/validation" "log" + "net" "strconv" - "strings" "time" ) @@ -124,6 +125,21 @@ func resourceMWSWorkspaces() *schema.Resource { } } +func waitForWorkspaceURLResolution(workspace model.MWSWorkspace, timeoutDurationMinutes time.Duration) error { + hostAndPort := fmt.Sprintf("%s.cloud.databricks.com:443", workspace.DeploymentName) + url := fmt.Sprintf("https://%s.cloud.databricks.com", workspace.DeploymentName) + return resource.Retry(timeoutDurationMinutes, func() *resource.RetryError { + conn, err := net.DialTimeout("tcp", hostAndPort, 1*time.Minute) + if err != nil { + log.Printf("Cannot yet reach %s", url) + return resource.RetryableError(err) + } + log.Printf("Workspace %s is ready to use", url) + defer conn.Close() + return nil + }) +} + func resourceMWSWorkspacesCreate(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) mwsAcctID := d.Get("account_id").(string) @@ -162,6 +178,13 @@ func resourceMWSWorkspacesCreate(d *schema.ResourceData, m interface{}) error { } return err } + // wait maximum 5 minute for DNS caches to refresh, as + // sometimes we cannot make API calls to new workspaces + // immediately after it's created + err = waitForWorkspaceURLResolution(workspace, 5*time.Minute) + if err != nil { + return err + } return resourceMWSWorkspacesRead(d, m) } @@ -179,8 +202,8 @@ func resourceMWSWorkspacesRead(d *schema.ResourceData, m interface{}) error { workspace, err := client.MWSWorkspaces().Read(packagedMwsID.MwsAcctID, idInt64) if err != nil { - if isMWSWorkspaceMissing(err.Error(), id) { - log.Printf("Missing e2 workspace with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -319,8 +342,3 @@ func getNetworkErrors(networkRespList []model.NetworkHealth) string { } return strBuffer.String() } - -func isMWSWorkspaceMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") && - strings.Contains(errorMsg, fmt.Sprintf("workspace %s does not exist", resourceID)) -} diff --git a/databricks/resource_databricks_notebook.go b/databricks/resource_databricks_notebook.go index 75cca301fa..b3c8a61bf1 100644 --- a/databricks/resource_databricks_notebook.go +++ b/databricks/resource_databricks_notebook.go @@ -6,14 +6,12 @@ import ( "bytes" "encoding/base64" "encoding/json" - "fmt" "hash/crc32" "io" "log" "path/filepath" "sort" "strconv" - "strings" "github.com/databrickslabs/databricks-terraform/client/model" "github.com/databrickslabs/databricks-terraform/client/service" @@ -141,8 +139,8 @@ func resourceNotebookRead(d *schema.ResourceData, m interface{}) error { format := d.Get("format").(string) notebookData, err := client.Notebooks().Export(id, model.ExportFormat(format)) if err != nil { - if isNotebookMissing(err.Error(), id) { - log.Printf("Missing notebook with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -250,8 +248,3 @@ func getDBCCheckSumForCommands(fileIO io.Reader) (int, error) { } return int(crc32.ChecksumIEEE(commandsBuffer.Bytes())), nil } - -func isNotebookMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") && - strings.Contains(errorMsg, fmt.Sprintf("Path (%s) doesn't exist.", resourceID)) -} diff --git a/databricks/resource_databricks_scim_group.go b/databricks/resource_databricks_scim_group.go index 9a2e4dd62d..5b1e644d4b 100644 --- a/databricks/resource_databricks_scim_group.go +++ b/databricks/resource_databricks_scim_group.go @@ -1,9 +1,7 @@ package databricks import ( - "fmt" "log" - "strings" "github.com/databrickslabs/databricks-terraform/client/model" "github.com/databrickslabs/databricks-terraform/client/service" @@ -125,8 +123,8 @@ func resourceScimGroupRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) group, err := client.Groups().Read(id) if err != nil { - if isScimGroupMissing(err.Error(), id) { - log.Printf("Missing scim group with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -248,9 +246,3 @@ func resourceScimGroupDelete(d *schema.ResourceData, m interface{}) error { err := client.Groups().Delete(id) return err } - -func isScimGroupMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "urn:ietf:params:scim:api:messages:2.0:Error") && - strings.Contains(errorMsg, fmt.Sprintf("Group with id %s not found.", resourceID)) && - strings.Contains(errorMsg, "404") -} diff --git a/databricks/resource_databricks_scim_user.go b/databricks/resource_databricks_scim_user.go index a55d7b90e2..4cd4678dde 100644 --- a/databricks/resource_databricks_scim_user.go +++ b/databricks/resource_databricks_scim_user.go @@ -1,11 +1,9 @@ package databricks import ( - "fmt" "log" "reflect" "sort" - "strings" "github.com/databrickslabs/databricks-terraform/client/model" "github.com/databrickslabs/databricks-terraform/client/service" @@ -131,8 +129,8 @@ func resourceScimUserRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) user, err := client.Users().Read(id) if err != nil { - if isScimUserMissing(err.Error(), id) { - log.Printf("Missing scim user with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -251,12 +249,6 @@ func resourceScimUserDelete(d *schema.ResourceData, m interface{}) error { return err } -func isScimUserMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "urn:ietf:params:scim:api:messages:2.0:Error") && - strings.Contains(errorMsg, fmt.Sprintf("User with id %s not found.", resourceID)) && - strings.Contains(errorMsg, "404") -} - func sliceContains(value string, list []string) bool { for _, v := range list { if reflect.DeepEqual(v, value) { diff --git a/databricks/resource_databricks_secret.go b/databricks/resource_databricks_secret.go index 44d2fbb2b7..1925701d28 100644 --- a/databricks/resource_databricks_secret.go +++ b/databricks/resource_databricks_secret.go @@ -1,7 +1,6 @@ package databricks import ( - "fmt" "log" "strings" @@ -74,7 +73,8 @@ func resourceSecretRead(d *schema.ResourceData, m interface{}) error { } secretMetaData, err := client.Secrets().Read(scope, key) if err != nil { - if isErrorRecoverable(err, scope, key) { + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -108,24 +108,3 @@ func resourceSecretDelete(d *schema.ResourceData, m interface{}) error { err = client.Secrets().Delete(scope, key) return err } - -func isErrorRecoverable(err error, scope string, key string) bool { - if isSecretMissing(err.Error(), scope, key) { - log.Printf("Missing secret with id: %s in scope with id: %s.", scope, key) - return true - } - if isScopeMissing(err.Error(), scope) { - log.Printf("Missing scope with id: %s; secret %s cannot exist without scope", scope, key) - return true - } - - return false -} - -func isSecretMissing(errorMsg, scope string, key string) bool { - return strings.Contains(errorMsg, fmt.Sprintf("no Secret Scope found with secret metadata scope name: %s and key: %s", scope, key)) -} - -func isScopeMissing(errorMsg, scope string) bool { - return strings.Contains(errorMsg, fmt.Sprintf("Scope %s does not exist!", scope)) -} diff --git a/databricks/resource_databricks_secret_acl.go b/databricks/resource_databricks_secret_acl.go index 474298f18d..43e122b483 100644 --- a/databricks/resource_databricks_secret_acl.go +++ b/databricks/resource_databricks_secret_acl.go @@ -1,7 +1,6 @@ package databricks import ( - "fmt" "log" "strings" @@ -70,8 +69,8 @@ func resourceSecretACLRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) secretACL, err := client.SecretAcls().Read(scope, principal) if err != nil { - if isSecretACLMissing(err.Error(), scope, principal) { - log.Printf("Missing secret acl in scope with id: %s and principal: %s.", scope, principal) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -99,8 +98,3 @@ func resourceSecretACLDelete(d *schema.ResourceData, m interface{}) error { err = client.SecretAcls().Delete(scope, key) return err } - -func isSecretACLMissing(errorMsg, scope string, principal string) bool { - return strings.Contains(errorMsg, "RESOURCE_DOES_NOT_EXIST") && - strings.Contains(errorMsg, fmt.Sprintf("Failed to get secret acl for principal %s for scope %s.", principal, scope)) -} diff --git a/databricks/resource_databricks_secret_scope.go b/databricks/resource_databricks_secret_scope.go index 22e4201f8e..3c22aac89d 100644 --- a/databricks/resource_databricks_secret_scope.go +++ b/databricks/resource_databricks_secret_scope.go @@ -1,9 +1,7 @@ package databricks import ( - "fmt" "log" - "strings" "github.com/databrickslabs/databricks-terraform/client/service" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" @@ -52,8 +50,8 @@ func resourceSecretScopeRead(d *schema.ResourceData, m interface{}) error { id := d.Id() scope, err := client.SecretScopes().Read(id) if err != nil { - if isSecretScopeMissing(err.Error(), id) { - log.Printf("Missing secret scope with name: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -74,7 +72,3 @@ func resourceSecretScopeDelete(d *schema.ResourceData, m interface{}) error { err := client.SecretScopes().Delete(id) return err } - -func isSecretScopeMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, fmt.Sprintf("no Secret Scope found with scope name %s", resourceID)) -} diff --git a/databricks/resource_databricks_token.go b/databricks/resource_databricks_token.go index 3147d4fa48..c14e577046 100644 --- a/databricks/resource_databricks_token.go +++ b/databricks/resource_databricks_token.go @@ -1,9 +1,7 @@ package databricks import ( - "fmt" "log" - "strings" "github.com/databrickslabs/databricks-terraform/client/service" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" @@ -68,8 +66,8 @@ func resourceTokenRead(d *schema.ResourceData, m interface{}) error { client := m.(*service.DBApiClient) token, err := client.Tokens().Read(id) if err != nil { - if isTokenMissing(err.Error(), id) { - log.Printf("Missing databricks api token with id: %s.", id) + if e, ok := err.(service.APIError); ok && e.IsMissing() { + log.Printf("missing resource due to error: %v\n", e) d.SetId("") return nil } @@ -89,7 +87,3 @@ func resourceTokenDelete(d *schema.ResourceData, m interface{}) error { err := client.Tokens().Delete(tokenID) return err } - -func isTokenMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, fmt.Sprintf("Unable to locate token: %s", resourceID)) -} diff --git a/databricks/utils.go b/databricks/utils.go index e28916e193..d4e0596aed 100644 --- a/databricks/utils.go +++ b/databricks/utils.go @@ -71,11 +71,6 @@ func changeClusterIntoRunningState(clusterID string, client *service.DBApiClient return fmt.Errorf("cluster is in a non recoverable state: %s", currentState) } -func isClusterMissing(errorMsg, resourceID string) bool { - return strings.Contains(errorMsg, "INVALID_PARAMETER_VALUE") && - strings.Contains(errorMsg, fmt.Sprintf("Cluster %s does not exist", resourceID)) -} - // PackagedMWSIds is a struct that contains both the MWS acct id and the ResourceId (resources are networks, creds, etc.) type PackagedMWSIds struct { MwsAcctID string diff --git a/databricks/utils_test.go b/databricks/utils_test.go index 5ef49ca30b..eba2ffea2d 100644 --- a/databricks/utils_test.go +++ b/databricks/utils_test.go @@ -1,31 +1,287 @@ package databricks import ( + "errors" + "fmt" + "os" + "strconv" "testing" + "github.com/databrickslabs/databricks-terraform/client/service" + "github.com/hashicorp/terraform-plugin-sdk/helper/acctest" + "github.com/stretchr/testify/assert" ) +func TestMissingMWSResources(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode.") + } + + mwsAcctId := os.Getenv("DATABRICKS_MWS_ACCT_ID") + randStringId := acctest.RandString(10) + randIntId := 2000000 + acctest.RandIntRange(100000, 20000000) + + client := getMWSClient() + tests := []struct { + name string + readFunc func() error + isCustomCheck bool + resourceID string + customCheckFunc func(err error, rId string) bool + }{ + { + name: "CheckIfMWSCredentialsAreMissing", + readFunc: func() error { + _, err := client.MWSCredentials().Read(mwsAcctId, randStringId) + return err + }, + }, + { + name: "CheckIfMWSNetworksAreMissing", + readFunc: func() error { + _, err := client.MWSNetworks().Read(mwsAcctId, randStringId) + return err + }, + }, + { + name: "CheckIfMWSStorageConfigurationsAreMissing", + readFunc: func() error { + _, err := client.MWSStorageConfigurations().Read(mwsAcctId, randStringId) + return err + }, + }, + { + name: "CheckIfMWSWorkspacesAreMissing", + readFunc: func() error { + _, err := client.MWSWorkspaces().Read(mwsAcctId, int64(randIntId)) + return err + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.isCustomCheck { + // Test custom check because api call does not return 404 not found if the resource does not exist + testVerifyResourceIsMissingCustomVerification(t, tt.resourceID, tt.readFunc, tt.customCheckFunc) + } else { + testVerifyResourceIsMissing(t, tt.readFunc) + } + }) + } +} + +// Capture this test for aws +func TestAccAwsMissingWorkspaceResources(t *testing.T) { + testMissingWorkspaceResources(t, service.AWS) +} + +// Capture this test for azure +func TestAccAzureMissingWorkspaceResources(t *testing.T) { + testMissingWorkspaceResources(t, service.Azure) +} + +func testMissingWorkspaceResources(t *testing.T, cloud service.CloudServiceProvider) { + if _, ok := os.LookupEnv("TF_ACC"); !ok { + t.Skip("Acceptance tests skipped unless env 'TF_ACC' set") + } + + randIntId := 2000000 + acctest.RandIntRange(100000, 20000000) + randStringId := acctest.RandString(10) + // example 405E7E8E4A000024 + randomClusterPolicyId := fmt.Sprintf("400E9E9E9A%d", + acctest.RandIntRange(100000, 999999), + ) + // example 0101-120000-brick1-pool-ABCD1234 + randomInstancePoolId := fmt.Sprintf( + "%v-%v-%s-pool-%s", + acctest.RandIntRange(1000, 9999), + acctest.RandIntRange(100000, 999999), + acctest.RandString(6), + acctest.RandString(8), + ) + client := getIntegrationDBAPIClient(t) + + type testTable struct { + name string + readFunc func() error + isCustomCheck bool + resourceID string + customCheckFunc func(err error, rId string) bool + } + tests := []testTable{ + { + name: "CheckIfTokensAreMissing", + readFunc: func() error { + _, err := client.Tokens().Read(randStringId) + return err + }, + }, + { + name: "CheckIfSecretScopesAreMissing", + readFunc: func() error { + _, err := client.SecretScopes().Read(randStringId) + return err + }, + }, + { + name: "CheckIfSecretsAreMissing", + readFunc: func() error { + _, err := client.Secrets().Read(randStringId, randStringId) + return err + }, + }, + { + name: "CheckIfSecretsACLsAreMissing", + readFunc: func() error { + _, err := client.SecretAcls().Read(randStringId, randStringId) + return err + }, + }, + { + name: "CheckIfSecretsACLsAreMissing", + readFunc: func() error { + _, err := client.SecretAcls().Read(randStringId, randStringId) + return err + }, + }, + { + name: "CheckIfNotebooksAreMissing", + readFunc: func() error { + // ID must start with a / + _, err := client.Notebooks().Read("/" + randStringId) + return err + }, + }, + { + name: "CheckIfInstancePoolsAreMissing", + readFunc: func() error { + _, err := client.InstancePools().Read(randomInstancePoolId) + return err + }, + }, + { + name: "CheckIfClustersAreMissing", + readFunc: func() error { + _, err := client.Clusters().Get(randStringId) + return err + }, + isCustomCheck: true, + customCheckFunc: isClusterMissing, + resourceID: randStringId, + }, + { + name: "CheckIfDBFSFilesAreMissing", + readFunc: func() error { + _, err := client.DBFS().Read("/" + randStringId) + return err + }, + }, + { + name: "CheckIfGroupsAreMissing", + readFunc: func() error { + _, err := client.Groups().Read(randStringId) + t.Log(err) + return err + }, + }, + { + name: "CheckIfUsersAreMissing", + readFunc: func() error { + _, err := client.Users().Read(randStringId) + t.Log(err) + return err + }, + }, + { + name: "CheckIfClusterPoliciesAreMissing", + readFunc: func() error { + _, err := client.ClusterPolicies().Get(randomClusterPolicyId) + t.Log(err) + return err + }, + }, + { + name: "CheckIfJobsAreMissing", + readFunc: func() error { + _, err := client.Jobs().Read(int64(randIntId)) + return err + }, + isCustomCheck: true, + customCheckFunc: isJobMissing, + resourceID: strconv.Itoa(randIntId), + }, + } + // Handle aws only tests where instance profiles only exist on aws + if cloud == service.AWS { + awsOnlyTests := []testTable{ + { + name: "CheckIfInstanceProfilesAreMissing", + readFunc: func() error { + _, err := client.InstanceProfiles().Read(randStringId) + return err + }, + }, + } + tests = append(tests, awsOnlyTests...) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.isCustomCheck { + // Test custom check because api call does not return 404 not found if the resource does not exist + testVerifyResourceIsMissingCustomVerification(t, tt.resourceID, tt.readFunc, tt.customCheckFunc) + } else { + testVerifyResourceIsMissing(t, tt.readFunc) + } + }) + } +} + +func testVerifyResourceIsMissingCustomVerification(t *testing.T, resourceId string, readFunc func() error, + customCheck func(err error, rId string) bool) { + err := readFunc() + assert.NotNil(t, err, "err should not be nil") + assert.IsType(t, err, service.APIError{}, fmt.Sprintf("error: %s is not type api error", err.Error())) + if apiError, ok := err.(service.APIError); ok { + assert.True(t, customCheck(err, resourceId), fmt.Sprintf("error: %v is not missing;"+ + "\nstatus code: %v;"+ + "\nerror code: %s", + apiError, apiError.StatusCode, apiError.ErrorCode)) + } +} + +func testVerifyResourceIsMissing(t *testing.T, readFunc func() error) { + err := readFunc() + assert.NotNil(t, err, "err should not be nil") + assert.IsType(t, err, service.APIError{}, fmt.Sprintf("error: %s is not type api error", err.Error())) + if apiError, ok := err.(service.APIError); ok { + assert.True(t, apiError.IsMissing(), fmt.Sprintf("error: %v is not missing;"+ + "\nstatus code: %v;"+ + "\nerror code: %s", + apiError, apiError.StatusCode, apiError.ErrorCode)) + } +} + func TestIsClusterMissingTrueWhenClusterIdSpecifiedPresent(t *testing.T) { - errorMessage := "{\"error_code\":\"INVALID_PARAMETER_VALUE\",\"message\":\"Cluster 123 does not exist\"}" + err := errors.New("{\"error_code\":\"INVALID_PARAMETER_VALUE\",\"message\":\"Cluster 123 does not exist\"}") - result := isClusterMissing(errorMessage, "123") + result := isClusterMissing(err, "123") assert.True(t, result) } func TestIsClusterMissingFalseWhenClusterIdSpecifiedNotPresent(t *testing.T) { - errorMessage := "{\"error_code\":\"INVALID_PARAMETER_VALUE\",\"message\":\"Cluster 123 does not exist\"}" + err := errors.New("{\"error_code\":\"INVALID_PARAMETER_VALUE\",\"message\":\"Cluster 123 does not exist\"}") - result := isClusterMissing(errorMessage, "xyz") + result := isClusterMissing(err, "xyz") assert.False(t, result) } func TestIsClusterMissingFalseWhenErrorNotInCorrectFormat(t *testing.T) { - errorMessage := "{\"error_code\":\"INVALID_PARAMETER_VALUE\",\"message\":\"Something random went bang xyz\"}" + err := errors.New("{\"error_code\":\"INVALID_PARAMETER_VALUE\",\"message\":\"Something random went bang xyz\"}") - result := isClusterMissing(errorMessage, "xyz") + result := isClusterMissing(err, "xyz") assert.False(t, result) } diff --git a/docs/index.md b/docs/index.md index c6fe5dd747..4be1fbb53c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,56 +9,101 @@ description: |- # Databricks Provider The Databricks provider is what is used to interact with the Databricks resources. This needs to be configured so that -terraform can provision resources in your Databricks workspace on your behalf. +terraform can provision resources in your Databricks workspace on your behalf. -## Example Usage -### Token Based Auth +## Example Usage -``` hcl +```hcl provider "databricks" { - host = "http://databricks.domain.com" - token = "dapitokenhere" + host = "https://abc-defg-024.cloud.databricks.com/" + token = "" } -resource "databricks_scim_user" "my-user" { - user_name = "test-user@databricks.com" - display_name = "Test User" +resource "databricks_cluster" "shared_autoscaling" { + cluster_name = "Shared Autoscaling" + spark_version = "6.6.x-scala2.11" + node_type_id = "i3.xlarge" + autotermination_minutes = 20 + + autoscale { + min_workers = 1 + max_workers = 50 + } } ``` -### Basic Auth +## Authentication + +!> **Warning** Please be aware that hard coding any credentials is not something that is recommended. It may be best if +you store the credentials environment variables, `~/.databrickscfg` file or use tfvars file. + +There are currently three supported methods [to authenticate into](https://docs.databricks.com/dev-tools/api/latest/authentication.html) the Databricks platform to create resources: + +* [PAT Tokens](https://docs.databricks.com/dev-tools/api/latest/authentication.html) +* Username+Password pair +* Azure Active Directory Tokens via Azure Service Principal + +### Authenticating with Databricks CLI credentials + +No configuration options given to your provider will look up configured credentials in `~/.databrickscfg` file. +It is created by `databricks configure --token` command. Check https://docs.databricks.com/dev-tools/cli/index.html#set-up-authentication +for docs. Config file credentials will only be used when `host`/`token` or `azure_auth` options are not provided. +This is recommended way to use Databricks Terraform provider, in case you're using the same approach with +[AWS Shared Credentials File](https://www.terraform.io/docs/providers/aws/index.html#shared-credentials-file) +or [Azure CLI authentication](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/azure_cli). ``` hcl provider "databricks" { - host = "http://databricks.domain.com" - basic_auth { - username = var.user - password = var.password - } } +``` -resource "databricks_scim_user" "my-user" { - user_name = "test-user@databricks.com" - display_name = "Test User" +One can specify non-standard location of configuration file through `config_file` parameter or `DATABRICKS_CONFIG_FILE` environment variable: + +``` hcl +provider "databricks" { + config_file = "/opt/databricks/cli-config" } ``` -### Profile Based Auth +One can specify a [CLI connection profile](https://docs.databricks.com/dev-tools/cli/index.html#connection-profiles) through `profile` parameter or `DATABRICKS_CONFIG_PROFILE` environment variable: ``` hcl provider "databricks" { - config_file = "~/.databrickscfg" - profile = "DEFAULT" + profile = "ML_WORKSPACE" } +``` -resource "databricks_scim_user" "my-user" { - user_name = "test-user@databricks.com" - display_name = "Test User" +### Authenticating with hostname and token + +One can use `host` and `token` parameters to supply credentials to workspace. In case environment variables are preferred, `DATABRICKS_HOST` and `DATABRICKS_TOKEN` could be used instead. This is second most recommended way of configuring this provider. + +``` hcl +provider "databricks" { + host = "http://abc-cdef-ghi.cloud.databricks.com" + token = "dapitokenhere" } ``` -### Azure SP Auth +### Authenticating with hostname, username and password + +!> **Warning** This approach is currently recommended only for provisioning AWS workspaces and should be avoided for regular use. + +One can use `basic_auth` parameter to supply username and password credentials to workspace. `DATABRICKS_USERNAME` and `DATABRICKS_PASSWORD` environment variables could be used instead. + +``` hcl +provider "databricks" { + host = "http://abc-cdef-ghi.cloud.databricks.com" + basic_auth { + username = var.user + password = var.password + } +} +``` + +### Authenticating with Azure Service Principal + +-> **Note** **Azure Service Principal Authentication** will only work on Azure Databricks where as the API Token authentication will work on both **Azure** and **AWS**. Internally `azure_auth` will generate a session-based PAT token. ``` hcl provider "azurerm" { @@ -77,7 +122,7 @@ resource "azurerm_databricks_workspace" "demo_test_workspace" { } provider "databricks" { - azure_auth = { + azure_auth { managed_resource_group = azurerm_databricks_workspace.demo_test_workspace.managed_resource_group_name azure_region = azurerm_databricks_workspace.demo_test_workspace.location workspace_name = azurerm_databricks_workspace.demo_test_workspace.name @@ -95,87 +140,6 @@ resource "databricks_scim_user" "my-user" { } ``` - -!> **Warning** Please be aware that hard coding credentials is not something that is recommended. It may be best if -you store the credentials environment variables or use tfvars file. - -## Authentication - -There are currently two supported methods to authenticate into the Databricks platform to create resources. - -* **API Token** -* **Azure Service Principal Authentication** - --> **Note** **Azure Service Principal Authentication** will only work on Azure Databricks where as the API Token -authentication will work on both **Azure** and **AWS** - - -### API Token - -Databricks hostname for the workspace and api token can be provided here. This configuration is very similar to the -Databricks CLI - -``` hcl -provider "databricks" { - host = "http://databricks.domain.com" - token = "dapitokenhere" -} -``` - -!> **Warning** Please be aware that hard coding credentials is not something that is recommended. -It may be best if you store the credentials environment variables or use tfvars file. - - - -### Azure Service Principal Auth - -``` hcl -provider "databricks" { - azure_auth = { - managed_resource_group = "${azurerm_databricks_workspace.sri_test_workspace.managed_resource_group_name}" - azure_region = "${azurerm_databricks_workspace.sri_test_workspace.location}" - workspace_name = "${azurerm_databricks_workspace.sri_test_workspace.name}" - resource_group = "${azurerm_databricks_workspace.sri_test_workspace.resource_group_name}" - client_id = "${var.client_id}" - client_secret = "${var.client_secret}" - tenant_id = "${var.tenant_id}" - subscription_id = "${var.subscription_id}" - } -} -``` - -### Environment variables - -The following variables can be passed via environment variables: - -* `host` → `DATABRICKS_HOST` -* `token` → `DATABRICKS_TOKEN` -* `basic_auth.username` → `DATABRICKS_USERNAME` -* `basic_auth.password` → `DATABRICKS_PASSWORD` -* `config_file` → `DATABRICKS_CONFIG_FILE` -* `managed_resource_group` → `DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP` -* `azure_region` → `AZURE_REGION` -* `workspace_name` → `DATABRICKS_AZURE_WORKSPACE_NAME` -* `resource_group` → `DATABRICKS_AZURE_RESOURCE_GROUP` -* `subscription_id` → `DATABRICKS_AZURE_SUBSCRIPTION_ID` or `ARM_SUBSCRIPTION_ID` -* `client_secret` → `DATABRICKS_AZURE_CLIENT_SECRET` or `ARM_CLIENT_SECRET` -* `client_id` → `DATABRICKS_AZURE_CLIENT_ID` or `ARM_CLIENT_ID` -* `tenant_id` → `DATABRICKS_AZURE_TENANT_ID` or `ARM_TENANT_ID` - -For example you can have the following provider definition: - -``` hcl -provider "databricks" {} -``` - -Then run the following code and the following environment variables will be injected into the provider. - -``` bash -$ export HOST="http://databricks.domain.com" -$ export TOKEN="dapitokenhere" -$ terraform plan -``` - ## Argument Reference The following arguments are supported by the db provider block: @@ -202,8 +166,6 @@ https://docs.databricks.com/dev-tools/cli/index.html#connection-profiles for doc * `azure_auth` - (optional) This is a azure_auth block ([documented below]((#azure_auth-configuration-block))) required to authenticate to the Databricks via an azure service principal that has access to the workspace. This is optional as you can use the api token based auth. - - ### basic_auth Configuration Block Example: @@ -245,7 +207,7 @@ This is the authentication required to authenticate to the Databricks via an azu principal that has access to the workspace. This is optional as you can use the api token based auth. The azure_auth block contains the following arguments: -* `managed_resource_group` - (required) This is the managed workgroup id when the Databricks workspace is provisioned. +* `managed_resource_group` - (required) This is the managed resource group id when the Databricks workspace is provisioned. Alternatively you can provide this value as an environment variable `DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP`. * `azure_region` - (required) This is the azure region in which your workspace is deployed. @@ -274,3 +236,35 @@ resides in. Alternatively you can provide this value as an environment variable Where there are multiple environment variable options, the `DATABRICKS_AZURE_*` environment variables takes precedence and the `ARM_*` environment variables provide a way to share authentication configuration when using the `databricks-terraform` provider alongside the `azurerm` provider. + +## Environment variables + +The following variables can be passed via environment variables: + +* `host` → `DATABRICKS_HOST` +* `token` → `DATABRICKS_TOKEN` +* `basic_auth.username` → `DATABRICKS_USERNAME` +* `basic_auth.password` → `DATABRICKS_PASSWORD` +* `config_file` → `DATABRICKS_CONFIG_FILE` +* `managed_resource_group` → `DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP` +* `azure_region` → `AZURE_REGION` +* `workspace_name` → `DATABRICKS_AZURE_WORKSPACE_NAME` +* `resource_group` → `DATABRICKS_AZURE_RESOURCE_GROUP` +* `subscription_id` → `DATABRICKS_AZURE_SUBSCRIPTION_ID` or `ARM_SUBSCRIPTION_ID` +* `client_secret` → `DATABRICKS_AZURE_CLIENT_SECRET` or `ARM_CLIENT_SECRET` +* `client_id` → `DATABRICKS_AZURE_CLIENT_ID` or `ARM_CLIENT_ID` +* `tenant_id` → `DATABRICKS_AZURE_TENANT_ID` or `ARM_TENANT_ID` + +For example you can have the following provider definition: + +``` hcl +provider "databricks" {} +``` + +Then run the following code and the following environment variables will be injected into the provider. + +``` bash +$ export DATABRICKS_HOST="http://databricks.domain.com" +$ export DATABRICKS_TOKEN="dapitokenhere" +$ terraform plan +``` \ No newline at end of file diff --git a/docs/resources/cluster_policy.md b/docs/resources/cluster_policy.md index 8e13857be2..b41b079087 100644 --- a/docs/resources/cluster_policy.md +++ b/docs/resources/cluster_policy.md @@ -59,6 +59,8 @@ The following arguments are required: In addition to all arguments above, the following attributes are exported: +* `id` - Canonical unique identifier for the cluster policy. This equal to policy_id. + * `policy_id` - Canonical unique identifier for the cluster policy. ## Import diff --git a/website/content/Resources/scim_user.md b/website/content/Resources/scim_user.md index 6cc482a18b..a73a0abc60 100644 --- a/website/content/Resources/scim_user.md +++ b/website/content/Resources/scim_user.md @@ -28,6 +28,7 @@ inherited roles or default roles. resource "databricks_scim_user" "my-user" { user_name = "testuser@databricks.com" display_name = "Test User" + default_roles = [] entitlements = [ "allow-cluster-create", ] diff --git a/website/content/_index.md b/website/content/_index.md index 67e5813009..566468300a 100644 --- a/website/content/_index.md +++ b/website/content/_index.md @@ -6,7 +6,7 @@ chapter = false pre = "" +++ -# Lets lay some bricks! +# Databricks Terraform Provider ## Quick install @@ -25,3 +25,8 @@ You can `ls` the previous directory to verify. Please provide feedback in github issues. There is a template for this: {{% button href="https://github.com/databrickslabs/terraform-provider-databricks/issues/new?assignees=stikkireddy&labels=question&template=feedback.md&title=%5BFEEDBACK%2FQUESTION%5D+Short+Description+of+feedback" icon="fas fa-comment-dots" %}}Please provide feedback!{{% /button %}} + +## Project Support +Please note that all projects in the /databrickslabs github account are provided for your exploration only, and are not formally supported by Databricks with Service Level Agreements (SLAs). They are provided AS-IS and we do not make any guarantees of any kind. Please do not submit a support ticket relating to any issues arising from the use of these projects. + +Any issues discovered through the use of this project should be filed as GitHub Issues on the Repo. They will be reviewed as time permits, but there are no formal SLAs for support. \ No newline at end of file