diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 84d8a374af..e3ef342463 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -37,7 +37,9 @@ RUN \ # --> Delve for debugging go get github.com/go-delve/delve/cmd/dlv@v1.3.2 \ # --> Go language server - && go get golang.org/x/tools/gopls@v0.2.1 \ + && go get golang.org/x/tools/gopls@v0.4.1 \ + # --> Goimports + && go get golang.org/x/tools/cmd/goimports \ # --> Gotestsum && go get gotest.tools/gotestsum@v0.4.2 \ # --> Go symbols and outline for go to symbol support and test support diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4d1b2ae293..f9aff16b68 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,6 +3,14 @@ { "name": "databricks-terraform", "dockerFile": "Dockerfile", + "mounts": [ + // Mount go mod cache + "source=databricks-terraform-gomodcache,target=/go/pkg,type=volume", + // Keep command history + "source=databricks-terraform-bashhistory,target=/commandhistory,type=volume", + // Mount docker socket for docker builds + "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind", + ], "runArgs": [ // Uncomment the next line to use a non-root user. On Linux, this will prevent // new files getting created as root, but you may need to update the USER_UID @@ -12,14 +20,8 @@ "--security-opt", "seccomp=unconfined", "--privileged", - - "--name", "databricks-terraform-devcontainer", - // Mount go mod cache - "-v", "databricks-terraform-gomodcache:/go/pkg", - // Keep command history - "-v", "databricks-terraform-bashhistory:/root/commandhistory", - // Mount docker socket for docker builds - "-v", "/var/run/docker.sock:/var/run/docker.sock", + "--name", + "databricks-terraform-devcontainer", // Use host network "--network=host", ], @@ -51,8 +53,9 @@ // "postCreateCommand": "go version", // Add the IDs of extensions you want installed when the container is created in the array below. "extensions": [ - "ms-vscode.go", + "golang.go", "mauve.terraform", + "hashicorp.terraform", "ms-vsliveshare.vsliveshare" ] } \ No newline at end of file diff --git a/Makefile b/Makefile index 42d0a209ff..1965ba6d0d 100644 --- a/Makefile +++ b/Makefile @@ -48,10 +48,13 @@ vendor: @echo "==> Filling vendor folder with library code..." @go mod vendor +local-install: build + mv terraform-provider-databricks $(HOME)/.terraform.d/plugins/terraform-provider-databricks_v0.2.0 + # INTEGRATION TESTING WITH AZURE terraform-acc-azure: lint @echo "==> Running Terraform Acceptance Tests for Azure..." - @CLOUD_ENV="azure" TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -tags=azure -short -coverprofile=coverage.out ./... + @/bin/bash integration-environment-azure/run.sh # INTEGRATION TESTING WITH AWS terraform-acc-aws: lint diff --git a/databricks/azure_auth.go b/databricks/azure_auth.go index 1ce0170b0d..d1c1af1742 100644 --- a/databricks/azure_auth.go +++ b/databricks/azure_auth.go @@ -5,7 +5,7 @@ import ( "fmt" "log" "net/http" - urlParse "net/url" + "time" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" @@ -17,15 +17,6 @@ const ( ADBResourceID string = "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" ) -// AzureAuth is a struct that contains information about the azure sp authentication -type AzureAuth struct { - TokenPayload *TokenPayload - ManagementToken string - AdbWorkspaceResourceID string - AdbAccessToken string - AdbPlatformToken string -} - // TokenPayload contains all the auth information for azure sp authentication type TokenPayload struct { ManagedResourceGroup string @@ -38,6 +29,39 @@ type TokenPayload struct { TenantID string } +type Workspace struct { + Name string `json:"name"` + ID string `json:"id"` + Type string `json:"type"` + Sku struct { + Name string `json:"name"` + } `json:"sku"` + Location string `json:"location"` + Properties struct { + ManagedResourceGroupID string `json:"managedResourceGroupId"` + Parameters interface{} `json:"parameters"` + ProvisioningState string `json:"provisioningState"` + UIDefinitionURI string `json:"uiDefinitionUri"` + Authorizations []struct { + PrincipalID string `json:"principalId"` + RoleDefinitionID string `json:"roleDefinitionId"` + } `json:"authorizations"` + CreatedBy struct { + Oid string `json:"oid"` + Puid string `json:"puid"` + ApplicationID string `json:"applicationId"` + } `json:"createdBy"` + UpdatedBy struct { + Oid string `json:"oid"` + Puid string `json:"puid"` + ApplicationID string `json:"applicationId"` + } `json:"updatedBy"` + CreatedDateTime time.Time `json:"createdDateTime"` + WorkspaceID string `json:"workspaceId"` + WorkspaceURL string `json:"workspaceUrl"` + } `json:"properties"` +} + // WsProps contains information about the workspace properties type WsProps struct { ManagedResourceGroupID string `json:"managedResourceGroupId"` @@ -50,42 +74,41 @@ type WorkspaceRequest struct { Location string `json:"location"` } -func (a *AzureAuth) getManagementToken() error { +func (t *TokenPayload) getManagementToken() (string, error) { log.Println("[DEBUG] Creating Azure Databricks management OAuth token.") mgmtTokenOAuthCfg, err := adal.NewOAuthConfigWithAPIVersion(azure.PublicCloud.ActiveDirectoryEndpoint, - a.TokenPayload.TenantID, + t.TenantID, nil) if err != nil { - return err + return "", err } mgmtToken, err := adal.NewServicePrincipalToken( *mgmtTokenOAuthCfg, - a.TokenPayload.ClientID, - a.TokenPayload.ClientSecret, + t.ClientID, + t.ClientSecret, azure.PublicCloud.ServiceManagementEndpoint) if err != nil { - return err + return "", err } err = mgmtToken.Refresh() if err != nil { - return err + return "", err } - a.ManagementToken = mgmtToken.OAuthToken() - return nil + + return mgmtToken.OAuthToken(), nil } -func (a *AzureAuth) getWorkspaceID(config *service.DBApiClientConfig) error { +func (t *TokenPayload) getWorkspace(config *service.DBApiClientConfig, managementToken string) (*Workspace, error) { log.Println("[DEBUG] Getting Workspace ID via management token.") - // Escape all the ids url := fmt.Sprintf("https://management.azure.com/subscriptions/%s/resourceGroups/%s"+ "/providers/Microsoft.Databricks/workspaces/%s", - urlParse.PathEscape(a.TokenPayload.SubscriptionID), - urlParse.PathEscape(a.TokenPayload.ResourceGroup), - urlParse.PathEscape(a.TokenPayload.WorkspaceName)) + t.SubscriptionID, + t.ResourceGroup, + t.WorkspaceName) headers := map[string]string{ "Content-Type": "application/json", "cache-control": "no-cache", - "Authorization": "Bearer " + a.ManagementToken, + "Authorization": "Bearer " + managementToken, } type apiVersion struct { APIVersion string `url:"api-version"` @@ -93,49 +116,48 @@ func (a *AzureAuth) getWorkspaceID(config *service.DBApiClientConfig) error { uriPayload := apiVersion{ APIVersion: "2018-04-01", } - var responseMap map[string]interface{} resp, err := service.PerformQuery(config, http.MethodGet, url, "2.0", headers, false, true, uriPayload, nil) if err != nil { - return err + return nil, err } - err = json.Unmarshal(resp, &responseMap) + + var workspace Workspace + err = json.Unmarshal(resp, &workspace) if err != nil { - return err + return nil, err } - a.AdbWorkspaceResourceID = responseMap["id"].(string) - return err + return &workspace, err } -func (a *AzureAuth) getADBPlatformToken() error { +func (t *TokenPayload) getADBPlatformToken() (string, error) { log.Println("[DEBUG] Creating Azure Databricks management OAuth token.") platformTokenOAuthCfg, err := adal.NewOAuthConfigWithAPIVersion(azure.PublicCloud.ActiveDirectoryEndpoint, - a.TokenPayload.TenantID, + t.TenantID, nil) if err != nil { - return err + return "", err } platformToken, err := adal.NewServicePrincipalToken( *platformTokenOAuthCfg, - a.TokenPayload.ClientID, - a.TokenPayload.ClientSecret, + t.ClientID, + t.ClientSecret, ADBResourceID) if err != nil { - return err + return "", err } err = platformToken.Refresh() if err != nil { - return err + return "", err } - a.AdbPlatformToken = platformToken.OAuthToken() - return nil + + return platformToken.OAuthToken(), nil } -func (a *AzureAuth) getWorkspaceAccessToken(config *service.DBApiClientConfig) error { +func getWorkspaceAccessToken(config *service.DBApiClientConfig, managementToken, adbWorkspaceUrl, adbWorkspaceResourceID, adbPlatformToken string) (string, error) { log.Println("[DEBUG] Creating workspace token") - apiLifeTimeInSeconds := int32(600) + apiLifeTimeInSeconds := int32(3600) comment := "Secret made via SP" - url := "https://" + a.TokenPayload.AzureRegion + ".azuredatabricks.net/api/2.0/token/create" payload := struct { LifetimeSeconds int32 `json:"lifetime_seconds,omitempty"` Comment string `json:"comment,omitempty"` @@ -145,60 +167,60 @@ func (a *AzureAuth) getWorkspaceAccessToken(config *service.DBApiClientConfig) e } headers := map[string]string{ "Content-Type": "application/x-www-form-urlencoded", - "X-Databricks-Azure-Workspace-Resource-Id": a.AdbWorkspaceResourceID, - "X-Databricks-Azure-SP-Management-Token": a.ManagementToken, + "X-Databricks-Azure-Workspace-Resource-Id": adbWorkspaceResourceID, + "X-Databricks-Azure-SP-Management-Token": managementToken, "cache-control": "no-cache", - "Authorization": "Bearer " + a.AdbPlatformToken, + "Authorization": "Bearer " + adbPlatformToken, } - var responseMap map[string]interface{} + url := adbWorkspaceUrl + "/api/2.0/token/create" resp, err := service.PerformQuery(config, http.MethodPost, url, "2.0", headers, true, true, payload, nil) if err != nil { - return err + return "", err } + + var responseMap map[string]interface{} err = json.Unmarshal(resp, &responseMap) if err != nil { - return err + return "", err } - a.AdbAccessToken = responseMap["token_value"].(string) - return nil + + return responseMap["token_value"].(string), nil } // Main function call that gets made and it follows 4 steps at the moment: // 1. Get Management OAuth Token using management endpoint -// 2. Get Workspace ID +// 2. Get Workspace ID and URL // 3. Get Azure Databricks Platform OAuth Token using Databricks resource id // 4. Get Azure Databricks Workspace Personal Access Token for the SP (60 min duration) -func (a *AzureAuth) initWorkspaceAndGetClient(config *service.DBApiClientConfig) error { - //var dbClient service.DBApiClient - +func (t *TokenPayload) initWorkspaceAndGetClient(config *service.DBApiClientConfig) error { // Get management token - err := a.getManagementToken() + managementToken, err := t.getManagementToken() if err != nil { return err } // Get workspace access token - err = a.getWorkspaceID(config) + adbWorkspace, err := t.getWorkspace(config, managementToken) if err != nil { return err } + adbWorkspaceUrl := "https://" + adbWorkspace.Properties.WorkspaceURL // Get platform token - err = a.getADBPlatformToken() + adbPlatformToken, err := t.getADBPlatformToken() if err != nil { return err } // Get workspace personal access token - err = a.getWorkspaceAccessToken(config) + workspaceAccessToken, err := getWorkspaceAccessToken(config, managementToken, adbWorkspaceUrl, adbWorkspace.ID, adbPlatformToken) if err != nil { return err } - //// TODO: Eventually change this to include new Databricks domain names. May have to add new vars and/or deprecate existing args. - config.Host = "https://" + a.TokenPayload.AzureRegion + ".azuredatabricks.net" - config.Token = a.AdbAccessToken + config.Host = adbWorkspaceUrl + config.Token = workspaceAccessToken return nil } diff --git a/databricks/azure_auth_test.go b/databricks/azure_auth_test.go index 11858b668a..7253bad34b 100644 --- a/databricks/azure_auth_test.go +++ b/databricks/azure_auth_test.go @@ -1,6 +1,7 @@ package databricks import ( + "fmt" "os" "testing" @@ -20,24 +21,19 @@ func TestAzureAuthCreateApiToken(t *testing.T) { t.Skip("skipping integration test in short mode.") } - azureAuth := AzureAuth{ - TokenPayload: &TokenPayload{ - ManagedResourceGroup: os.Getenv("DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP"), - AzureRegion: os.Getenv("AZURE_REGION"), - WorkspaceName: os.Getenv("DATABRICKS_AZURE_WORKSPACE_NAME"), - ResourceGroup: os.Getenv("DATABRICKS_AZURE_RESOURCE_GROUP"), - }, - ManagementToken: "", - AdbWorkspaceResourceID: "", - AdbAccessToken: "", - AdbPlatformToken: "", + tokenPayload := TokenPayload{ + ManagedResourceGroup: getAndAssertEnv(t, "DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP"), + AzureRegion: getAndAssertEnv(t, "AZURE_REGION"), + WorkspaceName: getAndAssertEnv(t, "DATABRICKS_AZURE_WORKSPACE_NAME"), + ResourceGroup: getAndAssertEnv(t, "DATABRICKS_AZURE_RESOURCE_GROUP"), + SubscriptionID: getAndAssertEnv(t, "DATABRICKS_AZURE_SUBSCRIPTION_ID"), + TenantID: getAndAssertEnv(t, "DATABRICKS_AZURE_TENANT_ID"), + ClientID: getAndAssertEnv(t, "DATABRICKS_AZURE_CLIENT_ID"), + ClientSecret: getAndAssertEnv(t, "DATABRICKS_AZURE_CLIENT_SECRET"), } - azureAuth.TokenPayload.SubscriptionID = os.Getenv("DATABRICKS_AZURE_SUBSCRIPTION_ID") - azureAuth.TokenPayload.TenantID = os.Getenv("DATABRICKS_AZURE_TENANT_ID") - azureAuth.TokenPayload.ClientID = os.Getenv("DATABRICKS_AZURE_CLIENT_ID") - azureAuth.TokenPayload.ClientSecret = os.Getenv("DATABRICKS_AZURE_CLIENT_SECRET") + config := GetIntegrationDBClientOptions() - err := azureAuth.initWorkspaceAndGetClient(config) + err := tokenPayload.initWorkspaceAndGetClient(config) assert.NoError(t, err, err) api := service.DBApiClient{} api.SetConfig(config) @@ -58,3 +54,10 @@ func TestAzureAuthCreateApiToken(t *testing.T) { assert.NoError(t, instancePoolErr, instancePoolErr) } + +// getAndAssertEnv fetches the env for testing and also asserts that the env value is not Zero i.e "" +func getAndAssertEnv(t *testing.T, key string) string { + value, present := os.LookupEnv(key) + assert.True(t, present, fmt.Sprintf("Env variable %s is not set", key)) + return value +} diff --git a/databricks/provider.go b/databricks/provider.go index 23b319871a..7835741c97 100644 --- a/databricks/provider.go +++ b/databricks/provider.go @@ -172,8 +172,8 @@ func providerConfigureAzureClient(d *schema.ResourceData, config *service.DBApiC log.Println("Creating db client via azure auth!") azureAuth, _ := d.GetOk("azure_auth") azureAuthMap := azureAuth.(map[string]interface{}) - //azureAuth AzureAuth{} tokenPayload := TokenPayload{} + // The if else is required for the reason that "azure_auth" schema object is not a block but a map // Maps do not inherently auto populate defaults from environment variables unless we explicitly assign values // This makes it very difficult to test @@ -182,16 +182,19 @@ func providerConfigureAzureClient(d *schema.ResourceData, config *service.DBApiC } else if os.Getenv("DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP") != "" { tokenPayload.ManagedResourceGroup = os.Getenv("DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP") } + if azureRegion, ok := azureAuthMap["azure_region"].(string); ok { tokenPayload.AzureRegion = azureRegion } else if os.Getenv("AZURE_REGION") != "" { tokenPayload.AzureRegion = os.Getenv("AZURE_REGION") } + if resourceGroup, ok := azureAuthMap["resource_group"].(string); ok { tokenPayload.ResourceGroup = resourceGroup } else if os.Getenv("DATABRICKS_AZURE_RESOURCE_GROUP") != "" { tokenPayload.ResourceGroup = os.Getenv("DATABRICKS_AZURE_RESOURCE_GROUP") } + if workspaceName, ok := azureAuthMap["workspace_name"].(string); ok { tokenPayload.WorkspaceName = workspaceName } else if os.Getenv("DATABRICKS_AZURE_WORKSPACE_NAME") != "" { @@ -245,16 +248,8 @@ func providerConfigureAzureClient(d *schema.ResourceData, config *service.DBApiC tokenPayload.TenantID = os.Getenv("ARM_TENANT_ID") } - azureAuthSetup := AzureAuth{ - TokenPayload: &tokenPayload, - ManagementToken: "", - AdbWorkspaceResourceID: "", - AdbAccessToken: "", - AdbPlatformToken: "", - } - // Setup the CustomAuthorizer Function to be called at API invoke rather than client invoke - config.CustomAuthorizer = azureAuthSetup.initWorkspaceAndGetClient + config.CustomAuthorizer = tokenPayload.initWorkspaceAndGetClient var dbClient service.DBApiClient dbClient.SetConfig(config) return &dbClient, nil @@ -303,7 +298,11 @@ func providerConfigure(d *schema.ResourceData, providerVersion string) (interfac //version information from go-releaser using -ldflags to tell the golang linker to send semver info config.UserAgent = fmt.Sprintf("databricks-tf-provider/%s", providerVersion) - if _, ok := d.GetOk("azure_auth"); !ok { + if _, ok := d.GetOk("azure_auth"); ok { + // Abstracted logic to another function that returns a interface{}, error to inject directly + // for the providers during cloud integration testing + return providerConfigureAzureClient(d, &config) + } else { if host, ok := d.GetOk("host"); ok { config.Host = host.(string) } @@ -328,10 +327,6 @@ func providerConfigure(d *schema.ResourceData, providerVersion string) (interfac return nil, fmt.Errorf("failed to get credentials from config file; error msg: %w", err) } } - } else { - // Abstracted logic to another function that returns a interface{}, error to inject directly - // for the providers during cloud integration testing - return providerConfigureAzureClient(d, &config) } var dbClient service.DBApiClient diff --git a/databricks/resource_databricks_azure_adls_gen2_mount_test.go b/databricks/resource_databricks_azure_adls_gen2_mount_test.go index 1096241582..d212863556 100644 --- a/databricks/resource_databricks_azure_adls_gen2_mount_test.go +++ b/databricks/resource_databricks_azure_adls_gen2_mount_test.go @@ -69,34 +69,18 @@ func testAccAzureAdlsGen2MountCorrectlyMounts() string { clientID := os.Getenv("ARM_CLIENT_ID") clientSecret := os.Getenv("ARM_CLIENT_SECRET") tenantID := os.Getenv("ARM_TENANT_ID") - subscriptionID := os.Getenv("ARM_SUBSCRIPTION_ID") - workspaceName := os.Getenv("TEST_WORKSPACE_NAME") - resourceGroupName := os.Getenv("TEST_RESOURCE_GROUP") - managedResourceGroupName := os.Getenv("TEST_MANAGED_RESOURCE_GROUP") - location := os.Getenv("TEST_LOCATION") gen2AdalName := os.Getenv("TEST_GEN2_ADAL_NAME") definition := fmt.Sprintf(` - provider "databricks" { - azure_auth = { - client_id = "%[1]s" - client_secret = "%[2]s" - tenant_id = "%[3]s" - subscription_id = "%[4]s" - - workspace_name = "%[5]s" - resource_group = "%[6]s" - managed_resource_group = "%[7]s" - azure_region = "%[8]s" - } - } - resource "databricks_cluster" "cluster" { num_workers = 1 spark_version = "6.4.x-scala2.11" node_type_id = "Standard_D3_v2" # Don't spend too much, turn off cluster after 15mins autotermination_minutes = 15 + spark_conf = { + "spark.databricks.delta.preview.enabled": "false" + } } resource "databricks_secret_scope" "terraform" { @@ -115,7 +99,7 @@ func testAccAzureAdlsGen2MountCorrectlyMounts() string { resource "databricks_azure_adls_gen2_mount" "mount" { cluster_id = databricks_cluster.cluster.id container_name = "dev" # Created by prereqs.tf - storage_account_name = "%[9]s" + storage_account_name = "%[4]s" directory = "" mount_name = "localdir${databricks_cluster.cluster.cluster_id}" tenant_id = "%[3]s" @@ -125,7 +109,7 @@ func testAccAzureAdlsGen2MountCorrectlyMounts() string { initialize_file_system = true } -`, clientID, clientSecret, tenantID, subscriptionID, workspaceName, resourceGroupName, managedResourceGroupName, location, gen2AdalName) +`, clientID, clientSecret, tenantID, gen2AdalName) return definition } @@ -141,19 +125,6 @@ func testAccAzureAdlsGen2MountCaptureError() string { gen2AdalName := os.Getenv("TEST_GEN2_ADAL_NAME") definition := fmt.Sprintf(` - provider "databricks" { - azure_auth = { - client_id = "%[1]s" - client_secret = "%[2]s" - tenant_id = "%[3]s" - subscription_id = "%[4]s" - - workspace_name = "%[5]s" - resource_group = "%[6]s" - managed_resource_group = "%[7]s" - azure_region = "%[8]s" - } - } resource "databricks_cluster" "cluster" { num_workers = 1 diff --git a/databricks/resource_databricks_azure_blob_mount_test.go b/databricks/resource_databricks_azure_blob_mount_test.go index 5cc1853b43..04ad7dc093 100644 --- a/databricks/resource_databricks_azure_blob_mount_test.go +++ b/databricks/resource_databricks_azure_blob_mount_test.go @@ -127,38 +127,19 @@ func testAccAzureBlobMountMountExists(n string, azureBlobMount *AzureBlobMount, } func testAccAzureBlobMountCorrectlyMounts() string { - clientID := os.Getenv("ARM_CLIENT_ID") - clientSecret := os.Getenv("ARM_CLIENT_SECRET") - tenantID := os.Getenv("ARM_TENANT_ID") - subscriptionID := os.Getenv("ARM_SUBSCRIPTION_ID") - workspaceName := os.Getenv("TEST_WORKSPACE_NAME") - resourceGroupName := os.Getenv("TEST_RESOURCE_GROUP") - managedResourceGroupName := os.Getenv("TEST_MANAGED_RESOURCE_GROUP") - location := os.Getenv("TEST_LOCATION") blobAccountKey := os.Getenv("TEST_STORAGE_ACCOUNT_KEY") blobAccountName := os.Getenv("TEST_STORAGE_ACCOUNT_NAME") definition := fmt.Sprintf(` - provider "databricks" { - azure_auth = { - client_id = "%[1]s" - client_secret = "%[2]s" - tenant_id = "%[3]s" - subscription_id = "%[4]s" - - workspace_name = "%[5]s" - resource_group = "%[6]s" - managed_resource_group = "%[7]s" - azure_region = "%[8]s" - } - } - resource "databricks_cluster" "cluster" { num_workers = 1 spark_version = "6.4.x-scala2.11" node_type_id = "Standard_D3_v2" # Don't spend too much, turn off cluster after 15mins autotermination_minutes = 15 + spark_conf = { + "spark.databricks.delta.preview.enabled": "false" + } } resource "databricks_secret_scope" "terraform" { @@ -170,20 +151,20 @@ func testAccAzureBlobMountCorrectlyMounts() string { resource "databricks_secret" "storage_key" { key = "blob_storage_key" - string_value = "%[10]s" + string_value = "%[1]s" scope = databricks_secret_scope.terraform.name } resource "databricks_azure_blob_mount" "mount" { cluster_id = databricks_cluster.cluster.id container_name = "dev" # Created by prereqs.tf - storage_account_name = "%[9]s" + storage_account_name = "%[2]s" mount_name = "dev" auth_type = "ACCESS_KEY" token_secret_scope = databricks_secret_scope.terraform.name token_secret_key = databricks_secret.storage_key.key } -`, clientID, clientSecret, tenantID, subscriptionID, workspaceName, resourceGroupName, managedResourceGroupName, location, blobAccountName, blobAccountKey) +`, blobAccountKey, blobAccountName) return definition } diff --git a/examples/simple-azure-tf-deployment/main.tf b/examples/simple-azure-tf-deployment/main.tf index b6b8ef4bbf..1a3d7537d5 100644 --- a/examples/simple-azure-tf-deployment/main.tf +++ b/examples/simple-azure-tf-deployment/main.tf @@ -24,18 +24,18 @@ variable "managed_resource_group_name" { } provider "azurerm" { - client_id = var.client_id - client_secret = var.client_secret - tenant_id = var.tenant_id + client_id = var.client_id + client_secret = var.client_secret + tenant_id = var.tenant_id subscription_id = var.subscription_id } resource "azurerm_databricks_workspace" "demo_test_workspace" { - location = "centralus" - name = "terraform-test-ws-6" - resource_group_name = var.resource_group + location = "centralus" + name = "terraform-test-ws-6" + resource_group_name = var.resource_group managed_resource_group_name = var.managed_resource_group_name - sku = "premium" + sku = "premium" } @@ -43,21 +43,21 @@ resource "azurerm_databricks_workspace" "demo_test_workspace" { provider "databricks" { azure_auth = { managed_resource_group = azurerm_databricks_workspace.demo_test_workspace.managed_resource_group_name - azure_region = azurerm_databricks_workspace.demo_test_workspace.location - workspace_name = azurerm_databricks_workspace.demo_test_workspace.name - resource_group = azurerm_databricks_workspace.demo_test_workspace.resource_group_name - client_id = var.client_id - client_secret = var.client_secret - tenant_id = var.tenant_id - subscription_id = var.subscription_id + azure_region = azurerm_databricks_workspace.demo_test_workspace.location + workspace_name = azurerm_databricks_workspace.demo_test_workspace.name + resource_group = azurerm_databricks_workspace.demo_test_workspace.resource_group_name + client_id = var.client_id + client_secret = var.client_secret + tenant_id = var.tenant_id + subscription_id = var.subscription_id } } // resource "databricks_scim_user" "my-user" { - count = 2 - user_name = join("", ["demo-test-user", "+",count.index,"@databricks.com"]) + count = 2 + user_name = join("", ["demo-test-user", "+", count.index, "@databricks.com"]) display_name = "demo Test User" entitlements = [ "allow-cluster-create", @@ -66,56 +66,56 @@ resource "databricks_scim_user" "my-user" { resource "databricks_scim_group" "my-group" { display_name = "demo Test Group" - members = [databricks_scim_user.my-user[0].id] + members = [databricks_scim_user.my-user[0].id] } resource "databricks_secret_scope" "my-scope" { name = "terraform-demo-scope2" } resource "databricks_secret" "test_secret" { - key = "demo-test-secret-1" + key = "demo-test-secret-1" string_value = "hello world 123" - scope = databricks_secret_scope.my-scope.name + scope = databricks_secret_scope.my-scope.name } resource "databricks_secret_acl" "my-acl" { - principal = "USERS" + principal = "USERS" permission = "READ" - scope = databricks_secret_scope.my-scope.name + scope = databricks_secret_scope.my-scope.name } resource "databricks_instance_pool" "my-pool" { - instance_pool_name = "demo-terraform-pool" - min_idle_instances = 0 - max_capacity = 5 - node_type_id = "Standard_DS3_v2" + instance_pool_name = "demo-terraform-pool" + min_idle_instances = 0 + max_capacity = 5 + node_type_id = "Standard_DS3_v2" idle_instance_autotermination_minutes = 10 disk_spec = { azure_disk_volume_type = "PREMIUM_LRS" - disk_size = 80 - disk_count = 1 + disk_size = 80 + disk_count = 1 } custom_tags = { - "creator": "demo user" - "testChange": "demo user" + "creator" : "demo user" + "testChange" : "demo user" } } resource "databricks_token" "my-token" { lifetime_seconds = 6000 - comment = "Testing terraform v2" + comment = "Testing terraform v2" } resource "databricks_notebook" "my-dbc-base" { - content = filebase64("${path.module}/demo-terraform.dbc") - path = "/terraform-test-folder/folder1/folder2/terraformsamplecode" + content = filebase64("${path.module}/demo-terraform.dbc") + path = "/terraform-test-folder/folder1/folder2/terraformsamplecode" overwrite = false - mkdirs = true - format = "DBC" + mkdirs = true + format = "DBC" } data "databricks_notebook" "my-notebook" { - path = "/terraform-test-folder/folder1/folder2/terraformsamplecode" + path = "/terraform-test-folder/folder1/folder2/terraformsamplecode" format = "DBC" } @@ -130,4 +130,4 @@ output "databricks_scope" { output "notebook-content" { value = data.databricks_notebook.my-notebook.content -} \ No newline at end of file +} diff --git a/go.sum b/go.sum index 9c2ba2e717..db15d0be11 100644 --- a/go.sum +++ b/go.sum @@ -328,6 +328,7 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0 h1:Dh6fw+p6FyRl5x/FvNswO1ji0lIGzm3KP8Y9VkS9PTE= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/integration-environment-azure/prereqs.tf b/integration-environment-azure/prereqs.tf index 18a9823007..8dc6049357 100644 --- a/integration-environment-azure/prereqs.tf +++ b/integration-environment-azure/prereqs.tf @@ -1,5 +1,5 @@ provider "azurerm" { - version = "~> 2.3" + version = "~> 2.14" features {} } @@ -22,7 +22,7 @@ resource "azurerm_databricks_workspace" "example" { name = "workspace${random_string.naming.result}" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - sku = "standard" + sku = "premium" managed_resource_group_name = "workspace${random_string.naming.result}" } diff --git a/integration-environment-azure/run.sh b/integration-environment-azure/run.sh index adc9d5e6ac..c6a223633f 100755 --- a/integration-environment-azure/run.sh +++ b/integration-environment-azure/run.sh @@ -29,19 +29,29 @@ then rm -f *.tfstate fi +if [ -n "$TEST_LOG_LEVEL" ] +then + export TF_LOG=$TEST_LOG_LEVEL + # Output debug log to file while tests run + export TF_LOG_PATH=$PWD/tf.log +fi + terraform init terraform apply -auto-approve -export TEST_RESOURCE_GROUP=$(terraform output rg_name) -export TEST_WORKSPACE_NAME=$(terraform output workspace_name) +export CLOUD_ENV="azure" export TEST_GEN2_ADAL_NAME=$(terraform output gen2_adal_name) -export TEST_MANAGED_RESOURCE_GROUP=$(terraform output workspace_managed_rg_name) -export TEST_LOCATION=$(terraform output location) export TEST_STORAGE_ACCOUNT_KEY=$(terraform output blob_storage_key) export TEST_STORAGE_ACCOUNT_NAME=$(terraform output blob_storage_name) +export DATABRICKS_AZURE_WORKSPACE_NAME=$(terraform output workspace_name) +export DATABRICKS_AZURE_RESOURCE_GROUP=$(terraform output rg_name) +export AZURE_REGION=$(terraform output location) +export DATABRICKS_AZURE_MANAGED_RESOURCE_GROUP=$(terraform output workspace_managed_rg_name) + echo -e "----> Running Azure Acceptance Tests \n\n" -# Output debug log to file while tests run -export TF_LOG_PATH=$PWD/tf.log + # Run all Azure integration tests -TF_LOG=debug TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -short -coverprofile=coverage.out -test.timeout 35m -run 'TestAccAzure' ./../... \ No newline at end of file +TF_ACC=1 gotestsum --format short-verbose --raw-command \ + go test -v -json -coverprofile=coverage.out \ + -test.timeout 35m -run 'TestAccAzure' ./../... \ No newline at end of file diff --git a/website/content/Resources/mws_workspaces.md b/website/content/Resources/mws_workspaces.md index 6c67b6b64e..e297ae02a6 100644 --- a/website/content/Resources/mws_workspaces.md +++ b/website/content/Resources/mws_workspaces.md @@ -134,7 +134,7 @@ In addition to all arguments above, the following attributes are exported: #### - `network_error_messages`: > Array of error messages about the network configuration. -> Contains the following objects: +> Contains the following objects: > error_type: The AWS resource associated with this error: credentials, VPC, subnet, security group, or network ACL. > error_message: Details of the error.