Skip to content

Commit

Permalink
Migrate DAG loading to Kubernetes Jobs and Enable Storage and KV wher…
Browse files Browse the repository at this point in the history 
…e possible. (#239)
  • Loading branch information
@danielscholl
danielscholl authored Nov 12, 2024
1 parent 6c64254 commit 4226ab0
Show file tree
Hide file tree
Showing 59 changed files with 2,643 additions and 2,194 deletions.
0 ...modules/script-blob-upload/Legal_COO.json → Legal_COO.json
View file
File renamed without changes.
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# OSDU Developer

[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub milestone details](https://img.shields.io/github/milestones/progress/azure/osdu-developer/1)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub milestone details](https://img.shields.io/github/milestones/progress/azure/osdu-developer/1)


<!-- ![Github Issues](https://img.shields.io/github/issues/azure/osdu-developer)
Expand Down Expand Up @@ -29,7 +29,7 @@ This project uses the following Azure Container Service preview features:
- [Node Auto Provisioning](https://learn.microsoft.com/en-us/azure/aks/node-autoprovision?tabs=azure-cli)
- [SSH Disable](https://learn.microsoft.com/en-us/azure/aks/manage-ssh-node-access?tabs=node-shell#disable-ssh-overview)

Review the [Getting Started](https://azure.github.io/osdu-developer/getting_started/) documentation prior to proceeding.
Review the [documentation](https://azure.github.io/osdu-developer/getting_started/) prior to proceeding.

### CLI Quickstart

Expand Down Expand Up @@ -58,7 +58,7 @@ azd hooks run settings
azd down --force --purge
```

### Portal Quickstart _(reduced capabilities)_
### Portal Quickstart

> **Tutorial:** [Deploy OSDU Personal Instance via Portal](https://azure.github.io/osdu-developer/tutorial_arm/)
Expand Down
127 changes: 70 additions & 57 deletions bicep/main.bicep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,11 @@ param customVMSize string = ''
@description('Specify the Ingress type for the cluster.')
param ingressType string = 'External'

@description('Feature Flag: Enable Storage accounts public access.')
param enableBlobPublicAccess bool = false

@description('(Optional) Software Load Override - {enable/osduCore/osduReference} --> true/false, {repository} --> https://github.com/azure/osdu-devloper {branch} --> branch:main')
param clusterSoftware object = {
enable: true
private: false
osduCore: true
osduReference: true
osduVersion: ''
Expand All @@ -53,6 +52,7 @@ param experimentalSoftware object = {
param clusterConfiguration object = {
enableNodeAutoProvisioning: true
enablePrivateCluster: false
enableLockDown: false
}

@description('Optional. Bring your own Virtual Network.')
Expand Down Expand Up @@ -367,7 +367,8 @@ module clusterBlade 'modules/blade_cluster.bicep' = {
enableTelemetry: enableTelemetry

enableNodeAutoProvisioning: clusterConfiguration.enableNodeAutoProvisioning == 'false' ? false : true
enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'false' ? false : true
enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'true' ? true : false
nodeResourceGroupLockDown: clusterConfiguration.enableLockDown == 'false' ? false : true

workspaceResourceId: logAnalytics.outputs.resourceId
identityId: enableVnetInjection ? networkBlade.outputs.networkConfiguration.identityId : stampIdentity.outputs.resourceId
Expand All @@ -383,6 +384,7 @@ module clusterBlade 'modules/blade_cluster.bicep' = {
]
}


/*
__________ ___ .___________. _______ .__ __. _______. __ ______ .__ __.
| ____\ \ / / | || ____|| \ | | / || | / __ \ | \ | |
Expand Down Expand Up @@ -418,6 +420,7 @@ module fluxExtension 'modules/flux-extension/main.bicep' = {
]
}


/*
_______. ______ .______ __ .______ .___________.
/ | / || _ \ | | | _ \ | |
Expand Down Expand Up @@ -478,7 +481,6 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' =
| |\ \----.| |____ | |__| | | | .----) | | | | |\ \----. | |
| _| `._____||_______| \______| |__| |_______/ |__| | _| `._____| |__|
*/

module registry 'br/public:avm/res/container-registry/registry:0.1.1' = {
name: '${configuration.name}-container-registry'
params: {
Expand Down Expand Up @@ -520,6 +522,7 @@ module registry 'br/public:avm/res/container-registry/registry:0.1.1' = {
}
}


/*
__ ___ ___________ ____ ____ ____ ___ __ __ __ .___________.
| |/ / | ____\ \ / / \ \ / / / \ | | | | | | | |
Expand Down Expand Up @@ -697,6 +700,7 @@ var commonLayerConfig = {
}
}


/* _______.___________. ______ .______ ___ _______ _______
/ | | / __ \ | _ \ / \ / _____|| ____|
| (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__
Expand Down Expand Up @@ -758,15 +762,20 @@ module storage 'modules/storage-account/main.bicep' = {
principalId: stampIdentity.outputs.principalId
principalType: 'ServicePrincipal'
}

{
roleDefinitionIdOrName: 'Storage File Data Privileged Contributor'
principalId: stampIdentity.outputs.principalId
principalType: 'ServicePrincipal'
}
]

// Apply Security
allowBlobPublicAccess: enableBlobPublicAccess
allowBlobPublicAccess: false
publicNetworkAccess: 'Enabled'

// TODO: Deployment Scripts don't support this yet.
// allowSharedKeyAccess: true
// TODO: This is required for Partition Service to access the storage account. Issue: https://github.com/Azure/osdu-developer/issues/230
allowSharedKeyAccess: true

// https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts
networkAcls: {
bypass: 'AzureServices'
Expand Down Expand Up @@ -832,6 +841,16 @@ module database 'modules/cosmos-db/main.bicep' = {
diagnosticWorkspaceId: logAnalytics.outputs.resourceId
diagnosticLogsRetentionInDays: 0

networkRestrictions: {
publicNetworkAccess: 'Enabled'
networkAclBypass: 'AzureServices'
ipRules: [
'${clusterBlade.outputs.natClusterIP}'
]
virtualNetworkRules: []
}


// Configure Service
capabilitiesToAdd: [
'EnableGremlin'
Expand Down Expand Up @@ -898,62 +917,51 @@ var directoryUploads = [
]

@batchSize(1)
module gitOpsUpload 'modules/software-upload/main.bicep' = [for item in directoryUploads: {
module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: if (clusterSoftware.private == 'true') {
name: '${configuration.name}-storage-${item.directory}-upload'
params: {
newStorageAccount: true
name: 'script-${storage.outputs.name}-${item.directory}'

location: location
storageAccountName: storage.outputs.name
identityName: stampIdentity.outputs.name
cleanupPreference: 'Always'
retentionInterval: 'PT1H'
timeout: 'PT30M'
runOnce: true

managedIdentities: {
userAssignedResourcesIds: [
stampIdentity.outputs.resourceId
]
}

directoryName: item.directory
kind: 'AzureCLI'
azCliVersion: '2.63.0'

environmentVariables: [
{ name: 'AZURE_STORAGE_ACCOUNT', value: storage.outputs.name }
{ name: 'FILE', value: 'main.zip' }
{ name: 'URL', value: 'https://github.com/azure/osdu-developer/archive/refs/heads/main.zip' }
{ name: 'CONTAINER', value: 'gitops' }
{ name: 'UPLOAD_DIR', value: string(item.directory) }
]
scriptContent: loadTextContent('./modules/deploy-scripts/software-upload.sh')
}
dependsOn: [
stampIdentity
storage
]
}]

module manifestDagShareUpload 'modules/script-share-upload/main.bicep' = {
name: '${configuration.name}-storage-dag-upload-manifest'
params: {
newStorageAccount: true
location: location
storageAccountName: storage.outputs.name
identityName: stampIdentity.outputs.name
//TODO: This can't be done yet.
// module storageAcl 'modules/network_acl_storage.bicep' = {
// name: '${configuration.name}-storage-acl'
// params: {
// storageName: storage.outputs.name
// location: location
// skuName: configuration.storage.sku
// natClusterIP: clusterBlade.outputs.natClusterIP
// }
// dependsOn: [
// gitOpsUpload
// ]
// }

shareName: 'airflow-dags'
filename: 'src/osdu_dags'
compress: true
fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz'
}
dependsOn: [
stampIdentity
storage
]
}

module csvDagShareUpload 'modules/script-share-csvdag/main.bicep' = {
name: '${configuration.name}-storage-dag-upload-csv'
params: {
newStorageAccount: true
location: location
storageAccountName: storage.outputs.name
identityName: stampIdentity.outputs.name

shareName: 'airflow-dags'
filename: 'airflowdags'
fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz'
keyVaultUrl: keyvault.outputs.uri
insightsKey: insights.outputs.instrumentationKey
clientId: applicationClientId
clientSecret: applicationClientSecret
}
dependsOn: [
stampIdentity
storage
]
}

/*
.______ ___ .______ .___________. __ .___________. __ ______ .__ __.
Expand Down Expand Up @@ -986,7 +994,7 @@ module partitionBlade 'modules/blade_partition.bicep' = {
kvName: keyvault.outputs.name
natClusterIP: clusterBlade.outputs.natClusterIP

enableBlobPublicAccess: enableBlobPublicAccess
enableBlobPublicAccess: false

partitions: configuration.partitions
managedIdentityName: stampIdentity.outputs.name
Expand Down Expand Up @@ -1039,13 +1047,16 @@ module configBlade 'modules/blade_configuration.bicep' = {
enableExperimental: experimentalSoftware.enable == 'true' ? true : false
enableAdminUI: experimentalSoftware.adminUI == 'true' ? true : false

sourceHost: clusterSoftware.private == 'true' ? 'azureBlob' : 'gitRepository'

emailAddress: emailAddress
applicationClientId: applicationClientId
applicationClientPrincipalOid: applicationClientPrincipalOid

managedIdentityName: stampIdentity.outputs.name
kvName: keyvault.outputs.name
kvUri: keyvault.outputs.uri
appInsightsKey: insights.outputs.instrumentationKey
partitionStorageNames: partitionBlade.outputs.partitionStorageNames
partitionServiceBusNames: partitionBlade.outputs.partitionServiceBusNames

Expand Down Expand Up @@ -1080,6 +1091,8 @@ module configBlade 'modules/blade_configuration.bicep' = {
}




// =============== //
// Outputs //
// =============== //
Expand Down
7 changes: 3 additions & 4 deletions bicep/main.parameters.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,11 @@
"enablePodSubnet": {
"value": "${ENABLE_POD_SUBNET}"
},
"enableBlobPublicAccess": {
"value": "${ENABLE_BLOB_PUBLIC_ACCESS}"
},
"clusterConfiguration": {
"value": {
"enableNodeAutoProvisioning": "${ENABLE_NODE_AUTO_PROVISIONING}",
"enablePrivateCluster": "${ENABLE_PRIVATE_CLUSTER}"
"enablePrivateCluster": "${ENABLE_PRIVATE_CLUSTER}",
"enableLockDown": "${ENABLE_LOCK_DOWN}"
}
},
"vnetConfiguration": {
Expand Down Expand Up @@ -62,6 +60,7 @@
"clusterSoftware": {
"value": {
"enable": "${ENABLE_SOFTWARE}",
"private": "${ENABLE_PRIVATE_SOFTWARE}",
"osduVersion": "${SOFTWARE_VERSION}",
"osduCore": "${ENABLE_OSDU_CORE}",
"osduReference": "${ENABLE_OSDU_REFERENCE}",
Expand Down
7 changes: 4 additions & 3 deletions bicep/modules/blade_cluster.bicep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ param enableNodeAutoProvisioning bool = true
@description('Feature Flag to Enable Private Cluster')
param enablePrivateCluster bool = true

@description('Feature Flag to Enable Node Resource Group Lock Down')
param nodeResourceGroupLockDown bool = true

/////////////////////////////////
// Configuration
/////////////////////////////////
Expand Down Expand Up @@ -148,7 +151,7 @@ module cluster './managed-cluster/main.bicep' = {
disableLocalAccounts: true
enableRBAC: true
aadProfileManaged: true
nodeResourceGroupLockDown: true
nodeResourceGroupLockDown: nodeResourceGroupLockDown

// Observability Settings
enableAzureDefender: true
Expand Down Expand Up @@ -378,8 +381,6 @@ module appConfigExtension './managed-cluster/aks_appconfig_extension.bicep' = {





// =============== //
// Outputs //
// =============== //
Expand Down
Loading

0 comments on commit 4226ab0

Please sign in to comment.