Skip to content

Commit

Permalink
Merge pull request #70 from CloudFormations/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@MattPCollins
MattPCollins authored Jun 6, 2024
2 parents 2359e31 + 6fd1543 commit 9b8c3ee
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 12 deletions.
64 changes: 62 additions & 2 deletions src/azure.datafactory/pipeline/Ingest_PL_Files.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Fetch Binary Files Auth Username",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
Expand Down Expand Up @@ -80,7 +86,7 @@
"type": "Expression"
},
"DSUserName": {
"value": "@activity('Get Ingest Payload').output.firstRow.Username",
"value": "@activity('Fetch Binary Files Auth Username').output.value",
"type": "Expression"
},
"DSPassword": {
Expand Down Expand Up @@ -138,6 +144,30 @@
"resource": "https://vault.azure.net"
}
}
},
{
"name": "Fetch Binary Files Auth Username",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": true,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"method": "GET",
"url": {
"value": "@concat(activity('Get Ingest Payload').output.firstRow.Username,'?api-version=7.0')",
"type": "Expression"
},
"authentication": {
"type": "MSI",
"resource": "https://vault.azure.net"
}
}
}
]
},
Expand All @@ -153,6 +183,12 @@
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Fetch Parquet Files Auth Username",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
Expand Down Expand Up @@ -220,7 +256,7 @@
"type": "Expression"
},
"DSUserName": {
"value": "@activity('Get Ingest Payload').output.firstRow.Username",
"value": "@activity('Fetch Parquet Files Auth Username').output.value",
"type": "Expression"
},
"DSDirectory": {
Expand Down Expand Up @@ -286,6 +322,30 @@
"resource": "https://vault.azure.net"
}
}
},
{
"name": "Fetch Parquet Files Auth Username",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": true,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"method": "GET",
"url": {
"value": "@concat(activity('Get Ingest Payload').output.firstRow.Username,'?api-version=7.0')",
"type": "Expression"
},
"authentication": {
"type": "MSI",
"resource": "https://vault.azure.net"
}
}
}
]
}
Expand Down
96 changes: 93 additions & 3 deletions src/azure.datafactory/pipeline/Ingest_PL_MSSQL.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,12 @@
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Fetch SQL Auth Username",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
Expand Down Expand Up @@ -275,7 +281,7 @@
"type": "Expression"
},
"DSUsername": {
"value": "@activity('Get Ingest Payload').output.firstRow.Username",
"value": "@activity('Fetch SQL Auth Username').output.value",
"type": "Expression"
},
"DSPassword": {
Expand Down Expand Up @@ -333,6 +339,30 @@
"resource": "https://vault.azure.net"
}
}
},
{
"name": "Fetch SQL Auth Username",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": true,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"method": "GET",
"url": {
"value": "@concat(activity('Get Ingest Payload').output.firstRow.Username,'?api-version=7.0')",
"type": "Expression"
},
"authentication": {
"type": "MSI",
"resource": "https://vault.azure.net"
}
}
}
]
},
Expand All @@ -348,6 +378,12 @@
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Fetch Windows Auth Username",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
Expand Down Expand Up @@ -417,7 +453,7 @@
"type": "Expression"
},
"DSUsername": {
"value": "@activity('Get Ingest Payload').output.firstRow.Username",
"value": "@activity('Fetch Windows Auth Username').output.value",
"type": "Expression"
},
"DSPassword": {
Expand Down Expand Up @@ -475,6 +511,30 @@
"resource": "https://vault.azure.net"
}
}
},
{
"name": "Fetch Windows Auth Username",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": true,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"method": "GET",
"url": {
"value": "@concat(activity('Get Ingest Payload').output.firstRow.Username,'?api-version=7.0')",
"type": "Expression"
},
"authentication": {
"type": "MSI",
"resource": "https://vault.azure.net"
}
}
}
]
},
Expand All @@ -490,6 +550,12 @@
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Fetch SQL Login Auth Username",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
Expand Down Expand Up @@ -559,7 +625,7 @@
"type": "Expression"
},
"DSUsername": {
"value": "@activity('Get Ingest Payload').output.firstRow.Username",
"value": "@activity('Fetch SQL Login Auth Username').output.value",
"type": "Expression"
},
"DSPassword": {
Expand Down Expand Up @@ -617,6 +683,30 @@
"resource": "https://vault.azure.net"
}
}
},
{
"name": "Fetch SQL Login Auth Username",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": true,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"method": "GET",
"url": {
"value": "@concat(activity('Get Ingest Payload').output.firstRow.Username,'?api-version=7.0')",
"type": "Expression"
},
"authentication": {
"type": "MSI",
"resource": "https://vault.azure.net"
}
}
}
]
}
Expand Down
21 changes: 15 additions & 6 deletions src/azure.datafactory/pipeline/Ingest_PL_Merge.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,21 @@
}
]
}
],
"defaultActivities": [
{
"name": "Supported Linked Service Type",
"type": "Fail",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"message": {
"value": "@concat('The Linked Service type and authentication combination is not currently supported.')",
"type": "Expression"
},
"errorCode": "16"
}
}
]
}
},
Expand Down Expand Up @@ -243,7 +258,6 @@
},
{
"name": "Set Pipeline Run DateTime",

"type": "SetVariable",
"dependsOn": [],
"policy": {
Expand All @@ -270,11 +284,6 @@
"type": "String"
}
},
"variables": {
"PipelineRunDateTime": {
"type": "String"
}
},
"folder": {
"name": "Cumulus.Ingest"
},
Expand Down
32 changes: 31 additions & 1 deletion src/azure.datafactory/pipeline/Ingest_PL_Oracle.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Fetch SQL Auth Username",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
Expand Down Expand Up @@ -98,7 +104,7 @@
"type": "Expression"
},
"DSUsername": {
"value": "@activity('Get Ingest Payload').output.firstRow.Username",
"value": "@activity('Fetch SQL Auth Username').output.value",
"type": "Expression"
},
"DSOracleSid": {
Expand Down Expand Up @@ -164,6 +170,30 @@
"resource": "https://vault.azure.net"
}
}
},
{
"name": "Fetch SQL Auth Username",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": true,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"method": "GET",
"url": {
"value": "@concat(activity('Get Ingest Payload').output.firstRow.Username,'?api-version=7.0')",
"type": "Expression"
},
"authentication": {
"type": "MSI",
"resource": "https://vault.azure.net"
}
}
}
]
}
Expand Down

0 comments on commit 9b8c3ee

Please sign in to comment.