Merge pull request #68 from CloudFormations/develop

Develop
CloudFormations · Jun 4, 2024 · 2359e31 · 2359e31
2 parents c86c4e4 + b24dde2
commit 2359e31
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 2 deletions.
diff --git a/src/azure.databricks/python/notebooks/ingest/checkpayload/CheckPayloadFunctions.py b/src/azure.databricks/python/notebooks/ingest/checkpayload/CheckPayloadFunctions.py
@@ -27,7 +27,7 @@ def checkLoadType(loadType: str) -> None:
     return
 
 # For incremental loads we require primary keys to be used in the merge criteria.
-def checkMergeAndPKConditions(loadType:str, pkList: list[str]) -> None:
+def checkMergeAndPKConditions(loadType:str, pkList: list()) -> None:
     """
     Checks the combination of load type and primary key values providedprovided to prevent unsupported load types occurring.
     Currently supports Full ('F') and Incremental ('I') loads.

diff --git a/src/azure.datafactory/linkedService/Ingest_LS_Databricks_JobCluster_MIAuth.json b/src/azure.datafactory/linkedService/Ingest_LS_Databricks_JobCluster_MIAuth.json
@@ -0,0 +1,49 @@
+{
+	"name": "Ingest_LS_Databricks_JobCluster_MIAuth",
+	"properties": {
+		"parameters": {
+			"WorkspaceURL": {
+				"type": "string"
+			},
+			"ClusterVersion": {
+				"type": "string"
+			},
+			"NodeType": {
+				"type": "string"
+			},
+			"PythonVersion": {
+				"type": "string",
+				"defaultValue": "Unused"
+			},
+			"NumWorkers": {
+				"type": "string"
+			},
+			"SubscriptionId": {
+				"type": "string"
+			},
+			"ResourceName": {
+				"type": "string"
+			},
+			"ResourceGroupName": {
+				"type": "string"
+			}
+		},
+		"annotations": [
+			"JobCluster"
+		],
+		"type": "AzureDatabricks",
+		"typeProperties": {
+			"domain": "@linkedService().WorkspaceURL",
+			"authentication": "MSI",
+			"workspaceResourceId": "@concat('/subscriptions/',linkedService().SubscriptionId,'/resourceGroups/',linkedService().ResourceGroupName,'/providers/Microsoft.Databricks/workspaces/',linkedService().ResourceName)",
+			"newClusterNodeType": "@linkedService().NodeType",
+			"newClusterNumOfWorker": "@linkedService().NumWorkers",
+			"newClusterSparkEnvVars": {
+				"PYSPARK_PYTHON": "/databricks/python3/bin/python3"
+			},
+			"newClusterVersion": "@linkedService().ClusterVersion",
+			"clusterOption": "Autoscaling",
+			"newClusterInitScripts": []
+		}
+	}
+}
diff --git a/...re.datafactory/pipeline/Ingest_Merge.json → ...datafactory/pipeline/Ingest_PL_Merge.json b/...re.datafactory/pipeline/Ingest_Merge.json → ...datafactory/pipeline/Ingest_PL_Merge.json
@@ -1,5 +1,5 @@
 {
-	"name": "Ingest_Merge",
+	"name": "Ingest_PL_Merge",
 	"properties": {
 		"activities": [
 			{