diff --git a/bicep/modules/script-blob-upload/Legal_COO.json b/Legal_COO.json similarity index 100% rename from bicep/modules/script-blob-upload/Legal_COO.json rename to Legal_COO.json diff --git a/README.md b/README.md index 80ae7382..c6e8d061 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OSDU Developer -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub milestone details](https://img.shields.io/github/milestones/progress/azure/osdu-developer/1) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub milestone details](https://img.shields.io/github/milestones/progress/azure/osdu-developer/1) true/false, {repository} --> https://github.com/azure/osdu-devloper {branch} --> branch:main') param clusterSoftware object = { enable: true + private: false osduCore: true osduReference: true osduVersion: '' @@ -53,6 +52,7 @@ param experimentalSoftware object = { param clusterConfiguration object = { enableNodeAutoProvisioning: true enablePrivateCluster: false + enableLockDown: false } @description('Optional. Bring your own Virtual Network.') @@ -367,7 +367,8 @@ module clusterBlade 'modules/blade_cluster.bicep' = { enableTelemetry: enableTelemetry enableNodeAutoProvisioning: clusterConfiguration.enableNodeAutoProvisioning == 'false' ? false : true - enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'false' ? false : true + enablePrivateCluster: clusterConfiguration.enablePrivateCluster == 'true' ? true : false + nodeResourceGroupLockDown: clusterConfiguration.enableLockDown == 'false' ? false : true workspaceResourceId: logAnalytics.outputs.resourceId identityId: enableVnetInjection ? networkBlade.outputs.networkConfiguration.identityId : stampIdentity.outputs.resourceId @@ -383,6 +384,7 @@ module clusterBlade 'modules/blade_cluster.bicep' = { ] } + /* __________ ___ .___________. _______ .__ __. _______. __ ______ .__ __. | ____\ \ / / | || ____|| \ | | / || | / __ \ | \ | | @@ -418,6 +420,7 @@ module fluxExtension 'modules/flux-extension/main.bicep' = { ] } + /* _______. ______ .______ __ .______ .___________. / | / || _ \ | | | _ \ | | @@ -478,7 +481,6 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = | |\ \----.| |____ | |__| | | | .----) | | | | |\ \----. | | | _| `._____||_______| \______| |__| |_______/ |__| | _| `._____| |__| */ - module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { name: '${configuration.name}-container-registry' params: { @@ -520,6 +522,7 @@ module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { } } + /* __ ___ ___________ ____ ____ ____ ___ __ __ __ .___________. | |/ / | ____\ \ / / \ \ / / / \ | | | | | | | | @@ -697,6 +700,7 @@ var commonLayerConfig = { } } + /* _______.___________. ______ .______ ___ _______ _______ / | | / __ \ | _ \ / \ / _____|| ____| | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ @@ -758,15 +762,20 @@ module storage 'modules/storage-account/main.bicep' = { principalId: stampIdentity.outputs.principalId principalType: 'ServicePrincipal' } - + { + roleDefinitionIdOrName: 'Storage File Data Privileged Contributor' + principalId: stampIdentity.outputs.principalId + principalType: 'ServicePrincipal' + } ] // Apply Security - allowBlobPublicAccess: enableBlobPublicAccess + allowBlobPublicAccess: false publicNetworkAccess: 'Enabled' - // TODO: Deployment Scripts don't support this yet. - // allowSharedKeyAccess: true + // TODO: This is required for Partition Service to access the storage account. Issue: https://github.com/Azure/osdu-developer/issues/230 + allowSharedKeyAccess: true + // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts networkAcls: { bypass: 'AzureServices' @@ -832,6 +841,16 @@ module database 'modules/cosmos-db/main.bicep' = { diagnosticWorkspaceId: logAnalytics.outputs.resourceId diagnosticLogsRetentionInDays: 0 + networkRestrictions: { + publicNetworkAccess: 'Enabled' + networkAclBypass: 'AzureServices' + ipRules: [ + '${clusterBlade.outputs.natClusterIP}' + ] + virtualNetworkRules: [] + } + + // Configure Service capabilitiesToAdd: [ 'EnableGremlin' @@ -898,62 +917,51 @@ var directoryUploads = [ ] @batchSize(1) -module gitOpsUpload 'modules/software-upload/main.bicep' = [for item in directoryUploads: { +module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: if (clusterSoftware.private == 'true') { name: '${configuration.name}-storage-${item.directory}-upload' params: { - newStorageAccount: true + name: 'script-${storage.outputs.name}-${item.directory}' + location: location - storageAccountName: storage.outputs.name - identityName: stampIdentity.outputs.name + cleanupPreference: 'Always' + retentionInterval: 'PT1H' + timeout: 'PT30M' + runOnce: true + + managedIdentities: { + userAssignedResourcesIds: [ + stampIdentity.outputs.resourceId + ] + } - directoryName: item.directory + kind: 'AzureCLI' + azCliVersion: '2.63.0' + + environmentVariables: [ + { name: 'AZURE_STORAGE_ACCOUNT', value: storage.outputs.name } + { name: 'FILE', value: 'main.zip' } + { name: 'URL', value: 'https://github.com/azure/osdu-developer/archive/refs/heads/main.zip' } + { name: 'CONTAINER', value: 'gitops' } + { name: 'UPLOAD_DIR', value: string(item.directory) } + ] + scriptContent: loadTextContent('./modules/deploy-scripts/software-upload.sh') } - dependsOn: [ - stampIdentity - storage - ] }] -module manifestDagShareUpload 'modules/script-share-upload/main.bicep' = { - name: '${configuration.name}-storage-dag-upload-manifest' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: stampIdentity.outputs.name +//TODO: This can't be done yet. +// module storageAcl 'modules/network_acl_storage.bicep' = { +// name: '${configuration.name}-storage-acl' +// params: { +// storageName: storage.outputs.name +// location: location +// skuName: configuration.storage.sku +// natClusterIP: clusterBlade.outputs.natClusterIP +// } +// dependsOn: [ +// gitOpsUpload +// ] +// } - shareName: 'airflow-dags' - filename: 'src/osdu_dags' - compress: true - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz' - } - dependsOn: [ - stampIdentity - storage - ] -} - -module csvDagShareUpload 'modules/script-share-csvdag/main.bicep' = { - name: '${configuration.name}-storage-dag-upload-csv' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: stampIdentity.outputs.name - - shareName: 'airflow-dags' - filename: 'airflowdags' - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz' - keyVaultUrl: keyvault.outputs.uri - insightsKey: insights.outputs.instrumentationKey - clientId: applicationClientId - clientSecret: applicationClientSecret - } - dependsOn: [ - stampIdentity - storage - ] -} /* .______ ___ .______ .___________. __ .___________. __ ______ .__ __. @@ -986,7 +994,7 @@ module partitionBlade 'modules/blade_partition.bicep' = { kvName: keyvault.outputs.name natClusterIP: clusterBlade.outputs.natClusterIP - enableBlobPublicAccess: enableBlobPublicAccess + enableBlobPublicAccess: false partitions: configuration.partitions managedIdentityName: stampIdentity.outputs.name @@ -1039,6 +1047,8 @@ module configBlade 'modules/blade_configuration.bicep' = { enableExperimental: experimentalSoftware.enable == 'true' ? true : false enableAdminUI: experimentalSoftware.adminUI == 'true' ? true : false + sourceHost: clusterSoftware.private == 'true' ? 'azureBlob' : 'gitRepository' + emailAddress: emailAddress applicationClientId: applicationClientId applicationClientPrincipalOid: applicationClientPrincipalOid @@ -1046,6 +1056,7 @@ module configBlade 'modules/blade_configuration.bicep' = { managedIdentityName: stampIdentity.outputs.name kvName: keyvault.outputs.name kvUri: keyvault.outputs.uri + appInsightsKey: insights.outputs.instrumentationKey partitionStorageNames: partitionBlade.outputs.partitionStorageNames partitionServiceBusNames: partitionBlade.outputs.partitionServiceBusNames @@ -1080,6 +1091,8 @@ module configBlade 'modules/blade_configuration.bicep' = { } + + // =============== // // Outputs // // =============== // diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 71468ed1..34aac859 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -26,13 +26,11 @@ "enablePodSubnet": { "value": "${ENABLE_POD_SUBNET}" }, - "enableBlobPublicAccess": { - "value": "${ENABLE_BLOB_PUBLIC_ACCESS}" - }, "clusterConfiguration": { "value": { "enableNodeAutoProvisioning": "${ENABLE_NODE_AUTO_PROVISIONING}", - "enablePrivateCluster": "${ENABLE_PRIVATE_CLUSTER}" + "enablePrivateCluster": "${ENABLE_PRIVATE_CLUSTER}", + "enableLockDown": "${ENABLE_LOCK_DOWN}" } }, "vnetConfiguration": { @@ -62,6 +60,7 @@ "clusterSoftware": { "value": { "enable": "${ENABLE_SOFTWARE}", + "private": "${ENABLE_PRIVATE_SOFTWARE}", "osduVersion": "${SOFTWARE_VERSION}", "osduCore": "${ENABLE_OSDU_CORE}", "osduReference": "${ENABLE_OSDU_REFERENCE}", diff --git a/bicep/modules/blade_cluster.bicep b/bicep/modules/blade_cluster.bicep index 9f94eb31..df41ae9e 100644 --- a/bicep/modules/blade_cluster.bicep +++ b/bicep/modules/blade_cluster.bicep @@ -48,6 +48,9 @@ param enableNodeAutoProvisioning bool = true @description('Feature Flag to Enable Private Cluster') param enablePrivateCluster bool = true +@description('Feature Flag to Enable Node Resource Group Lock Down') +param nodeResourceGroupLockDown bool = true + ///////////////////////////////// // Configuration ///////////////////////////////// @@ -148,7 +151,7 @@ module cluster './managed-cluster/main.bicep' = { disableLocalAccounts: true enableRBAC: true aadProfileManaged: true - nodeResourceGroupLockDown: true + nodeResourceGroupLockDown: nodeResourceGroupLockDown // Observability Settings enableAzureDefender: true @@ -378,8 +381,6 @@ module appConfigExtension './managed-cluster/aks_appconfig_extension.bicep' = { - - // =============== // // Outputs // // =============== // diff --git a/bicep/modules/blade_common.bicep b/bicep/modules/blade_common.bicep deleted file mode 100644 index 9ae28bb5..00000000 --- a/bicep/modules/blade_common.bicep +++ /dev/null @@ -1,478 +0,0 @@ -///////////////// -// Common Blade -///////////////// - -@description('Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false.') -param enableBlobPublicAccess bool - -@description('Optional. The tags to apply to the resources') -param tags object = {} - -@description('The location of resources to deploy') -param location string - -@description('Feature Flag to Enable Telemetry') -param enableTelemetry bool - -@description('The configuration for the blade section.') -param bladeConfig bladeSettings - -@description('The workspace resource Id for diagnostics') -param workspaceResourceId string - -@description('The Application Insights Instrumentation Key') -param insightsKey string - -@description('Conditional. The name of the parent user assigned identity. Required if the template is used in a standalone deployment.') -param identityName string - -@description('The IP address of the NAT cluster.') -param natClusterIP string - -@description('Optional. Customer Managed Encryption Key.') -param cmekConfiguration object = { - kvUrl: '' - keyName: '' - identityId: '' -} - -@description('Specify the AD Application Client Id.') -param applicationClientId string - -@description('Specify the AD Application Client Secret.') -@secure() -param applicationClientSecret string - -@description('Specify the AD Application Principal Id.') -param applicationClientPrincipalOid string = '' - - -var commonLayerConfig = { - insights: { - sku: 'web' - } - storage: { - sku: 'Standard_LRS' - containers: [ - 'system' - 'azure-webjobs-hosts' - 'azure-webjobs-eventhub' - 'gitops' - ] - tables: [ - 'partitionInfo' - ] - shares: [ - 'airflow-logs' - 'airflow-dags' - ] - } - database: { - name: 'osdu-graph' - throughput: 2000 - backup: 'Continuous' - graphs: [ - { - name: 'Entitlements' - automaticIndexing: true - partitionKeyPaths: [ - '/dataPartitionId' - ] - } - ] - } -} - -resource userAssignedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { - name: identityName -} - -/* - __ ___ ___________ ____ ____ ____ ___ __ __ __ .___________. -| |/ / | ____\ \ / / \ \ / / / \ | | | | | | | | -| ' / | |__ \ \/ / \ \/ / / ^ \ | | | | | | `---| |----` -| < | __| \_ _/ \ / / /_\ \ | | | | | | | | -| . \ | |____ | | \ / / _____ \ | `--' | | `----. | | -|__|\__\ |_______| |__| \__/ /__/ \__\ \______/ |_______| |__| -*/ - -// var name = '${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' - -// @description('The list of secrets to persist to the Key Vault') -// var vaultSecrets = [ -// { -// secretName: 'tenant-id' -// secretValue: subscription().tenantId -// } -// { -// secretName: 'app-dev-sp-tenant-id' -// secretValue: subscription().tenantId -// } -// { -// secretName: 'subscription-id' -// secretValue: subscription().subscriptionId -// } -// // Azure AD Secrets -// { -// secretName: 'app-dev-sp-password' -// secretValue: applicationClientSecret == '' ? 'dummy' : applicationClientSecret -// } -// { -// secretName: 'app-dev-sp-id' -// secretValue: applicationClientId -// } -// { -// secretName: 'cpng-user-name' -// secretValue: 'dbuser' -// } -// { -// secretName: 'cpng-user-password' -// secretValue: substring(uniqueString('dbuser', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'cpng-superuser-name' -// secretValue: 'dbadmin' -// } -// { -// secretName: 'cpng-superuser-password' -// secretValue: substring(uniqueString('dbadmin', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'airflow-db-connection' -// secretValue: 'postgresql://dbuser:${substring(uniqueString('dbuser', resourceGroup().id, bladeConfig.sectionName), 0, 8)}@airflow-cluster-rw.postgresql.svc.cluster.local:5432/airflow-db' -// } -// { -// secretName: 'airflow-admin-username' -// secretValue: 'admin' -// } -// { -// secretName: 'airflow-admin-password' -// secretValue: substring(uniqueString('airflow', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'airflow-fernet-key' -// secretValue: substring(uniqueString('airflow-fernet', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// { -// secretName: 'airflow-webserver-key' -// secretValue: substring(uniqueString('airflow-webserver', resourceGroup().id, bladeConfig.sectionName), 0, 8) -// } -// ] - -// module keyvault 'br/public:avm/res/key-vault/vault:0.5.1' = { -// name: '${bladeConfig.sectionName}-keyvault' -// params: { -// name: length(name) > 24 ? substring(name, 0, 24) : name -// location: location -// enableTelemetry: enableTelemetry - -// // Assign Tags -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// } -// ) - -// diagnosticSettings: [ -// { -// workspaceResourceId: workspaceResourceId -// } -// ] - -// enablePurgeProtection: false - -// // Configure RBAC -// enableRbacAuthorization: true -// roleAssignments: union( -// applicationClientPrincipalOid != '' ? [ -// { -// roleDefinitionIdOrName: 'Key Vault Secrets User' -// principalId: applicationClientPrincipalOid -// principalType: 'ServicePrincipal' -// } -// ] : [], -// [] -// ) - -// // Configure Network Access -// publicNetworkAccess: 'Enabled' -// networkAcls: { -// bypass: 'AzureServices' -// defaultAction: 'Deny' -// ipRules: [ -// { -// value: natClusterIP -// } -// ] -// } - -// // Configure Secrets -// secrets: { -// secureList: [for secret in vaultSecrets: { -// name: secret.secretName -// value: secret.secretValue -// }] -// } -// } -// } - - -/* _______.___________. ______ .______ ___ _______ _______ - / | | / __ \ | _ \ / \ / _____|| ____| - | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ - \ \ | | | | | | | / / /_\ \ | | |_ | | __| -.----) | | | | `--' | | |\ \----./ _____ \ | |__| | | |____ -|_______/ |__| \______/ | _| `._____/__/ \__\ \______| |_______| -*/ -// AVM Module Customized due to required Secrets. -// module storage './storage-account/main.bicep' = { -// name: '${bladeConfig.sectionName}-storage' -// params: { -// name: '${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' -// location: location -// skuName: commonLayerConfig.storage.sku - -// // Assign Tags -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// } -// ) - -// // Hook up Diagnostics -// diagnosticSettings: [ -// { -// workspaceResourceId: workspaceResourceId -// } -// ] - -// // Configure Service -// blobServices: { -// containers: map(commonLayerConfig.storage.containers, container => { -// name: container -// }) -// } -// tableServices: { -// tables: map(commonLayerConfig.storage.tables, table => { -// name: table -// }) -// } -// fileServices: { -// shares: map(commonLayerConfig.storage.shares, share => { -// name: share -// }) -// } - -// // Apply Security -// allowBlobPublicAccess: enableBlobPublicAccess - -// publicNetworkAccess: 'Enabled' - -// // TODO: Deployment Scripts don't support this yet. -// // allowSharedKeyAccess: true -// // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts -// networkAcls: { -// bypass: 'AzureServices' -// defaultAction: 'Allow' // <--- Allow all traffic. Should be changed to Deny. -// ipRules: [ -// { -// value: natClusterIP -// } -// ] -// } - -// // Persist Secrets to Vault -// secretsExportConfiguration: { -// keyVaultResourceId: keyvault.outputs.resourceId -// accountName: [ -// 'system-storage' -// 'tbl-storage' -// ] -// accessKey1: [ -// 'system-storage-key' -// 'tbl-storage-key' -// ] -// connectionString1: [ -// 'system-storage-connection' -// ] -// blobEndpoint: [ -// 'system-storage-blob-endpoint' -// ] -// tableEndpoint: [ -// 'tbl-storage-endpoint' -// ] -// } -// } -// } - - -/* - _______ .______ ___ .______ __ __ - / _____|| _ \ / \ | _ \ | | | | -| | __ | |_) | / ^ \ | |_) | | |__| | -| | |_ | | / / /_\ \ | ___/ | __ | -| |__| | | |\ \----./ _____ \ | | | | | | - \______| | _| `._____/__/ \__\ | _| |__| |__| -*/ -// AVM Module Customized due to required Secrets. -// module database './cosmos-db/main.bicep' = { -// name: '${bladeConfig.sectionName}-cosmos-db' -// params: { -// resourceName: bladeConfig.sectionName -// resourceLocation: location - -// // Assign Tags -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// } -// ) - -// // Hook up Diagnostics -// diagnosticWorkspaceId: workspaceResourceId -// diagnosticLogsRetentionInDays: 0 - -// // Configure Service -// capabilitiesToAdd: [ -// 'EnableGremlin' -// ] -// gremlinDatabases: [ -// { -// name: commonLayerConfig.database.name -// graphs: commonLayerConfig.database.graphs -// } -// ] -// throughput: commonLayerConfig.database.throughput -// backupPolicyType: commonLayerConfig.database.backup - -// // Hookup Customer Managed Encryption Key -// systemAssignedIdentity: false -// userAssignedIdentities: !empty(cmekConfiguration.identityId) ? { -// '${cmekConfiguration.identityId}': {} -// } : {} -// defaultIdentity: !empty(cmekConfiguration.identityId) ? cmekConfiguration.identityId : '' -// kvKeyUri: !empty(cmekConfiguration.kvUrl) && !empty(cmekConfiguration.keyName) ? '${cmekConfiguration.kvUrl}/keys/${cmekConfiguration.keyName}' : '' - -// // Persist Secrets to Vault -// keyVaultName: keyvault.outputs.name -// databaseEndpointSecretName: 'graph-db-endpoint' -// databasePrimaryKeySecretName: 'graph-db-primary-key' -// databaseConnectionStringSecretName: 'graph-db-connection' - - -// roleAssignments: [ -// { -// roleDefinitionIdOrName: 'Contributor' -// principals: [ -// { -// id: applicationClientPrincipalOid -// } -// ] -// principalType: 'ServicePrincipal' -// } -// ] -// } -// } - - -/* - _______. ______ .______ __ .______ .___________. _______. - / | / || _ \ | | | _ \ | | / | - | (----`| ,----'| |_) | | | | |_) | `---| |----` | (----` - \ \ | | | / | | | ___/ | | \ \ -.----) | | `----.| |\ \----.| | | | | | .----) | -|_______/ \______|| _| `._____||__| | _| |__| |_______/ -*/ - - -var directoryUploads = [ - { - directory: 'software' - } - { - directory: 'charts' - } - { - directory: 'stamp' - } -] - -@batchSize(1) -module gitOpsUpload './software-upload/main.bicep' = [for item in directoryUploads: { - name: '${bladeConfig.sectionName}-storage-${item.directory}-upload' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: userAssignedIdentity.name - - directoryName: item.directory - } - dependsOn: [ - storage - ] -}] - -module manifestDagShareUpload './script-share-upload/main.bicep' = { - name: '${bladeConfig.sectionName}-storage-dag-upload-manifest' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: userAssignedIdentity.name - - shareName: 'airflow-dags' - filename: 'src/osdu_dags' - compress: true - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz' - } - dependsOn: [ - storage - ] -} - -module csvDagShareUpload './script-share-csvdag/main.bicep' = { - name: '${bladeConfig.sectionName}-storage-dag-upload-csv' - params: { - newStorageAccount: true - location: location - storageAccountName: storage.outputs.name - identityName: userAssignedIdentity.name - - shareName: 'airflow-dags' - filename: 'airflowdags' - fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz' - keyVaultUrl: keyvault.outputs.uri - insightsKey: insightsKey - clientId: applicationClientId - clientSecret: applicationClientSecret - } - dependsOn: [ - storage - ] -} - - -// =============== // -// Outputs // -// =============== // - -output keyvaultName string = keyvault.outputs.name -output keyvaultUri string = keyvault.outputs.uri -output storageAccountName string = storage.outputs.name -output storageAccountResourceId string = storage.outputs.resourceId - - -// =============== // -// Definitions // -// =============== // - -type bladeSettings = { - @description('The name of the section name') - sectionName: string - @description('The display name of the section') - displayName: string -} diff --git a/bicep/modules/blade_configuration.bicep b/bicep/modules/blade_configuration.bicep index 3327ac74..3b0feba8 100644 --- a/bicep/modules/blade_configuration.bicep +++ b/bicep/modules/blade_configuration.bicep @@ -32,6 +32,9 @@ param applicationClientId string @description('Specify the AD Application Principal Id.') param applicationClientPrincipalOid string = '' +@description('Specify the Application Insights Key.') +param appInsightsKey string + @description('Software GIT Repository URL') param softwareRepository string @@ -98,14 +101,6 @@ param appSettings appConfigItem[] param dateStamp string = utcNow() - -///////////////////////////////// -// Configuration -///////////////////////////////// - - - - ///////////////////////////////// // Existing Resources ///////////////////////////////// @@ -253,6 +248,28 @@ var osdu_applications = [ } ] +var airflow_values = [ + // Insights Key and Client Secret come from secrets. + { + name: 'tenantId' + value: subscription().tenantId + contentType: 'text/plain' + label: 'configmap-airflow-values' + } + { + name: 'clientId' + value: applicationClientId + contentType: 'text/plain' + label: 'configmap-airflow-values' + } + { + name: 'keyvaultUri' + value: keyVault.properties.vaultUri + contentType: 'text/plain' + label: 'configmap-airflow-values' + } +] + var settings = [ { name: 'osdu_sentinel' @@ -345,7 +362,7 @@ module app_config './app-configuration/main.bicep' = { ] // Add Configuration - keyValues: concat(union(appSettings, settings, partitionStorageSettings, partitionBusSettings, osdu_applications, common_helm_values)) + keyValues: concat(union(appSettings, settings, partitionStorageSettings, partitionBusSettings, osdu_applications, common_helm_values, airflow_values)) } } @@ -367,13 +384,15 @@ values.yaml: | configEndpoint: {2} keyvaultUri: {3} keyvaultName: {4} - appId: {5} - appOid: {6} + appInsightsKey: {5} + appId: {6} + appOid: {7} + resourceGroup: {8} ingress: internalGateway: - enabled: {7} + enabled: {9} externalGateway: - enabled: {8} + enabled: {10} ''' } @@ -406,8 +425,10 @@ module appConfigMap './aks-config-map/main.bicep' = { app_config.outputs.endpoint, kvUri, kvName, + appInsightsKey, applicationClientId, applicationClientPrincipalOid, + resourceGroup().name, clusterIngress == 'Internal' || clusterIngress == 'Both' ? 'true' : 'false', clusterIngress == 'External' || clusterIngress == 'Both' ? 'true' : 'false') ] diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index 1f105725..8df7b7e9 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -504,7 +504,7 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti }) } - + // Apply Roles roleAssignments: [ { roleDefinitionIdOrName: 'Storage Blob Data Contributor' @@ -513,12 +513,12 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti } ] + enableHierarchicalNamespace: true + // Apply Security allowBlobPublicAccess: enableBlobPublicAccess publicNetworkAccess: 'Enabled' - - // TODO: Deployment Scripts don't support this yet. - // allowSharedKeyAccess: true + allowSharedKeyAccess: true // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts networkAcls: { bypass: 'AzureServices' @@ -547,47 +547,6 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti }] - -// module partitionStorage './storage-account-orig/main.bicep' = [for (partition, index) in partitions: { -// name: '${bladeConfig.sectionName}-azure-storage-${index}' -// params: { -// #disable-next-line BCP335 BCP332 -// name: '${replace('data${index}${substring(uniqueString(partition.name), 0, 6)}', '-', '')}${uniqueString(resourceGroup().id, 'data${index}${substring(uniqueString(partition.name), 0, 6)}')}' -// location: location - -// // Assign Tags - -// tags: union( -// tags, -// { -// layer: bladeConfig.displayName -// partition: partition.name -// purpose: 'data' -// } -// ) - -// // Hook up Diagnostics -// diagnosticWorkspaceId: workspaceResourceId -// diagnosticLogsRetentionInDays: 0 - -// // Apply Security -// allowBlobPublicAccess: enableBlobPublicAccess - -// // Configure Service -// sku: partitionLayerConfig.storage.sku -// containers: concat(partitionLayerConfig.storage.containers, [partition.name]) - -// // Hookup Customer Managed Encryption Key -// cmekConfiguration: cmekConfiguration - -// // Persist Secrets to Vault -// keyVaultName: kvName -// storageAccountSecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountName}' -// storageAccountKeySecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountKey}' -// storageAccountBlobEndpointSecretName: '${partition.name}-${partitionLayerConfig.secrets.storageAccountBlob}' -// } -// }] - module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-cosmos-db-${index}' params: { @@ -609,6 +568,15 @@ module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partiti diagnosticWorkspaceId: workspaceResourceId diagnosticLogsRetentionInDays: 0 + networkRestrictions: { + publicNetworkAccess: 'Enabled' + networkAclBypass: 'AzureServices' + ipRules: [ + '${natClusterIP}' + ] + virtualNetworkRules: [] + } + // Set isSystemPartition based on index isSystemPartition: index == 0 ? true : false @@ -702,21 +670,51 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.1' = [for }] -// Deployment Scripts are not enabled yet for Private Link -// https://github.com/Azure/bicep/issues/6540 -module blobUpload './script-blob-upload/main.bicep' = [for (partition, index) in partitions: { +// TODO: This should be moved to the Kubernetes Job. +module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-storage-blob-upload-${index}' params: { - storageAccountName: storage[index].outputs.name + name: 'script-${storage[index].outputs.name}-Legal_COO' location: location + cleanupPreference: 'Always' + retentionInterval: 'PT1H' + timeout: 'PT30M' + runOnce: true + + managedIdentities: { + userAssignedResourcesIds: [ + stampIdentity.id + ] + } - useExistingManagedIdentity: true - managedIdentityName: managedIdentityName - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name + kind: 'AzureCLI' + azCliVersion: '2.63.0' + + environmentVariables: [ + { name: 'CONTENT', value: loadTextContent('./deploy-scripts/Legal_COO.json') } + { name: 'FILE_NAME', value: 'Legal_COO.json' } + { name: 'CONTAINER', value: 'legal-service-azure-configuration' } + { name: 'AZURE_STORAGE_ACCOUNT', value: storage[index].outputs.name } + ] + scriptContent: loadTextContent('./deploy-scripts/blob_upload.sh') } }] + +// TODO: ACL can only be applied after the blob upload. +// module storageAcl './network_acl_storage.bicep' = [for (partition, index) in partitions: { +// name: '${bladeConfig.sectionName}-storage-acl-${index}' +// params: { +// storageName: storage[index].outputs.name +// location: location +// skuName: partitionLayerConfig.storage.sku +// natClusterIP: natClusterIP +// } +// dependsOn: [ +// blobUpload[index] +// ] +// }] + module partitionSecrets './keyvault_secrets_partition.bicep' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-secrets-${index}' params: { diff --git a/bicep/modules/cosmos-db/main.bicep b/bicep/modules/cosmos-db/main.bicep index be42a7e2..f4de069e 100644 --- a/bicep/modules/cosmos-db/main.bicep +++ b/bicep/modules/cosmos-db/main.bicep @@ -202,6 +202,26 @@ param kvKeyUri string = '' @description('Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment\'s principal object.') param crossTenant bool = false +@description('Optional. The network configuration of this module. Defaults to `{ ipRules: [], virtualNetworkRules: [], publicNetworkAccess: \'Disabled\' }`.') +param networkRestrictions networkRestrictionsType = { + ipRules: [] + virtualNetworkRules: [] + publicNetworkAccess: 'Disabled' +} + +var ipRules = [ + for i in (networkRestrictions.?ipRules ?? []): { + ipAddressOrRange: i + } +] + +var virtualNetworkRules = [ + for vnet in (networkRestrictions.?virtualNetworkRules ?? []): { + id: vnet.subnetResourceId + ignoreMissingVnetServiceEndpoint: false + } +] + var name = '${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' @@ -286,6 +306,13 @@ var databaseAccount_properties = union({ } ] : databaseAccount_locations + + ipRules: ipRules + virtualNetworkRules: virtualNetworkRules + networkAclBypass: networkRestrictions.?networkAclBypass ?? 'AzureServices' + publicNetworkAccess: networkRestrictions.?publicNetworkAccess ?? 'Enabled' + isVirtualNetworkFilterEnabled: !empty(ipRules) || !empty(virtualNetworkRules) + capabilities: capabilities backupPolicy: backupPolicy } : {}), (!empty(sqlDatabases) ? { @@ -557,3 +584,21 @@ module secretDatabaseConnectionString '.bicep/keyvault_secrets.bicep' = if (!em value: databaseAccount.listConnectionStrings().connectionStrings[0].connectionString } } + + +type networkRestrictionsType = { + @description('Required. A single IPv4 address or a single IPv4 address range in CIDR format. Provided IPs must be well-formatted and cannot be contained in one of the following ranges: 10.0.0.0/8, 100.64.0.0/10, 172.16.0.0/12, 192.168.0.0/16, since these are not enforceable by the IP address filter. Example of valid inputs: "23.40.210.245" or "23.40.210.0/8".') + ipRules: string[] + + @description('Optional. Default to AzureServices. Specifies the network ACL bypass for Azure services.') + networkAclBypass: ('AzureServices' | 'None')? + + @description('Optional. Default to Enabled. Whether requests from Public Network are allowed.') + publicNetworkAccess: ('Enabled' | 'Disabled')? + + @description('Required. List of Virtual Network ACL rules configured for the Cosmos DB account..') + virtualNetworkRules: { + @description('Required. Resource ID of a subnet.') + subnetResourceId: string + }[] +} diff --git a/bicep/modules/deploy-scripts/Legal_COO.json b/bicep/modules/deploy-scripts/Legal_COO.json new file mode 100644 index 00000000..4dfbe41f --- /dev/null +++ b/bicep/modules/deploy-scripts/Legal_COO.json @@ -0,0 +1,1472 @@ +[{ + "name": "Andorra", + "alpha2": "AD", + "numeric": 16, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United Arab Emirates", + "alpha2": "AE", + "numeric": 784, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Afghanistan", + "alpha2": "AF", + "numeric": 4, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Antigua and Barbuda", + "alpha2": "AG", + "numeric": 28, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Anguilla", + "alpha2": "AI", + "numeric": 660, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Albania", + "alpha2": "AL", + "numeric": 8, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Armenia", + "alpha2": "AM", + "numeric": 51, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Netherlands Antilles", + "alpha2": "AN", + "numeric": 530, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Angola", + "alpha2": "AO", + "numeric": 24, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Antarctica", + "alpha2": "AQ", + "numeric": 10, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Argentina", + "alpha2": "AR", + "numeric": 32, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "American Samoa", + "alpha2": "AS", + "numeric": 16, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Austria", + "alpha2": "AT", + "numeric": 40, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Australia", + "alpha2": "AU", + "numeric": 36, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Aruba", + "alpha2": "AW", + "numeric": 533, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Aland Islands", + "alpha2": "AX", + "numeric": 248, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Azerbaijan", + "alpha2": "AZ", + "numeric": 31, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Bosnia and Herzegovina", + "alpha2": "BA", + "numeric": 70, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Barbados", + "alpha2": "BB", + "numeric": 52, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bangladesh", + "alpha2": "BD", + "numeric": 50, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Belgium", + "alpha2": "BE", + "numeric": 56, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Burkina Faso", + "alpha2": "BF", + "numeric": 854, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bulgaria", + "alpha2": "BG", + "numeric": 100, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bahrain", + "alpha2": "BH", + "numeric": 48, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Burundi", + "alpha2": "BI", + "numeric": 108, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Benin", + "alpha2": "BJ", + "numeric": 204, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Barthelemy", + "alpha2": "BL", + "numeric": 652, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bermuda", + "alpha2": "BM", + "numeric": 60, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Brunei Darussalam", + "alpha2": "BN", + "numeric": 96, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bolivia", + "alpha2": "BO", + "numeric": 68, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Brazil", + "alpha2": "BR", + "numeric": 76, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bahamas", + "alpha2": "BS", + "numeric": 44, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bhutan", + "alpha2": "BT", + "numeric": 64, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Bouvet Island", + "alpha2": "BV", + "numeric": 74, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Botswana", + "alpha2": "BW", + "numeric": 72, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Belarus", + "alpha2": "BY", + "numeric": 112, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Belize", + "alpha2": "BZ", + "numeric": 84, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Canada", + "alpha2": "CA", + "numeric": 124, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cocos Islands", + "alpha2": "CC", + "numeric": 166, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "The Democratic Republic of the Congo", + "alpha2": "CD", + "numeric": 180, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Central African Republic", + "alpha2": "CF", + "numeric": 140, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Congo", + "alpha2": "CG", + "numeric": 178, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Switzerland", + "alpha2": "CH", + "numeric": 756, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cote d'Ivoire", + "alpha2": "CI", + "numeric": 384, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cook Islands", + "alpha2": "CK", + "numeric": 184, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Chile", + "alpha2": "CL", + "numeric": 152, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cameroon", + "alpha2": "CM", + "numeric": 120, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "China", + "alpha2": "CN", + "numeric": 156, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Colombia", + "alpha2": "CO", + "numeric": 170, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Costa Rica", + "alpha2": "CR", + "numeric": 188, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cuba", + "alpha2": "CU", + "numeric": 192, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Cape Verde", + "alpha2": "CV", + "numeric": 132, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Christmas Island", + "alpha2": "CX", + "numeric": 162, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cyprus", + "alpha2": "CY", + "numeric": 196, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Czech Republic", + "alpha2": "CZ", + "numeric": 203, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Germany", + "alpha2": "DE", + "numeric": 276, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Djibouti", + "alpha2": "DJ", + "numeric": 262, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Denmark", + "alpha2": "DK", + "numeric": 208, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Dominica", + "alpha2": "DM", + "numeric": 212, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Dominican Republic", + "alpha2": "DO", + "numeric": 214, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Algeria", + "alpha2": "DZ", + "numeric": 12, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ecuador", + "alpha2": "EC", + "numeric": 218, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Estonia", + "alpha2": "EE", + "numeric": 233, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Egypt", + "alpha2": "EG", + "numeric": 818, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Western Sahara", + "alpha2": "EH", + "numeric": 732, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Eritrea", + "alpha2": "ER", + "numeric": 232, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Spain", + "alpha2": "ES", + "numeric": 724, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ethiopia", + "alpha2": "ET", + "numeric": 231, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Finland", + "alpha2": "FI", + "numeric": 246, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Fiji", + "alpha2": "FJ", + "numeric": 242, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Falkland Islands", + "alpha2": "FK", + "numeric": 238, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Federated States of Micronesia", + "alpha2": "FM", + "numeric": 583, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Faroe Islands", + "alpha2": "FO", + "numeric": 234, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "France", + "alpha2": "FR", + "numeric": 250, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Gabon", + "alpha2": "GA", + "numeric": 266, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United Kingdom", + "alpha2": "GB", + "numeric": 826, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Grenada", + "alpha2": "GD", + "numeric": 308, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Georgia", + "alpha2": "GE", + "numeric": 268, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "French Guiana", + "alpha2": "GF", + "numeric": 254, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guernsey", + "alpha2": "GG", + "numeric": 831, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ghana", + "alpha2": "GH", + "numeric": 288, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Gibraltar", + "alpha2": "GI", + "numeric": 292, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Greenland", + "alpha2": "GL", + "numeric": 304, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Gambia", + "alpha2": "GM", + "numeric": 270, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guinea", + "alpha2": "GN", + "numeric": 324, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guadeloupe", + "alpha2": "GP", + "numeric": 312, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Equatorial Guinea", + "alpha2": "GQ", + "numeric": 226, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Greece", + "alpha2": "GR", + "numeric": 300, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "South Georgia and the South Sandwich Islands", + "alpha2": "GS", + "numeric": 239, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guatemala", + "alpha2": "GT", + "numeric": 320, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guam", + "alpha2": "GU", + "numeric": 316, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guinea-Bissau", + "alpha2": "GW", + "numeric": 624, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Guyana", + "alpha2": "GY", + "numeric": 328, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Hong Kong", + "alpha2": "HK", + "numeric": 344, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Heard Island and McDonald Islands", + "alpha2": "HM", + "numeric": 334, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Honduras", + "alpha2": "HN", + "numeric": 340, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Croatia", + "alpha2": "HR", + "numeric": 191, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Haiti", + "alpha2": "HT", + "numeric": 332, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Hungary", + "alpha2": "HU", + "numeric": 348, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Indonesia", + "alpha2": "ID", + "numeric": 360, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Ireland", + "alpha2": "IE", + "numeric": 372, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Israel", + "alpha2": "IL", + "numeric": 376, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Isle of Man", + "alpha2": "IM", + "numeric": 833, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "India", + "alpha2": "IN", + "numeric": 356, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "British Indian Ocean Territory", + "alpha2": "IO", + "numeric": 86, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Iraq", + "alpha2": "IQ", + "numeric": 368, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Islamic Republic of Iran", + "alpha2": "IR", + "numeric": 364, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Iceland", + "alpha2": "IS", + "numeric": 352, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Italy", + "alpha2": "IT", + "numeric": 380, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Jersey", + "alpha2": "JE", + "numeric": 832, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Jamaica", + "alpha2": "JM", + "numeric": 388, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Jordan", + "alpha2": "JO", + "numeric": 400, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Japan", + "alpha2": "JP", + "numeric": 392, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kenya", + "alpha2": "KE", + "numeric": 404, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kyrgyzstan", + "alpha2": "KG", + "numeric": 417, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Cambodia", + "alpha2": "KH", + "numeric": 116, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kiribati", + "alpha2": "KI", + "numeric": 296, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Comoros", + "alpha2": "KM", + "numeric": 174, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Kitts and Nevis", + "alpha2": "KN", + "numeric": 659, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Democratic People's Republic of Korea", + "alpha2": "KP", + "numeric": 408, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Republic of Korea", + "alpha2": "KR", + "numeric": 410, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kuwait", + "alpha2": "KW", + "numeric": 414, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Cayman Islands", + "alpha2": "KY", + "numeric": 136, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Kazakhstan", + "alpha2": "KZ", + "numeric": 398, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lao People's Democratic Republic", + "alpha2": "LA", + "numeric": 418, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lebanon", + "alpha2": "LB", + "numeric": 422, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Lucia", + "alpha2": "LC", + "numeric": 662, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Liechtenstein", + "alpha2": "LI", + "numeric": 438, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Sri Lanka", + "alpha2": "LK", + "numeric": 144, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Liberia", + "alpha2": "LR", + "numeric": 430, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lesotho", + "alpha2": "LS", + "numeric": 426, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Lithuania", + "alpha2": "LT", + "numeric": 440, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Luxembourg", + "alpha2": "LU", + "numeric": 442, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Latvia", + "alpha2": "LV", + "numeric": 428, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Libya", + "alpha2": "LY", + "numeric": 434, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Morocco", + "alpha2": "MA", + "numeric": 504, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Monaco", + "alpha2": "MC", + "numeric": 492, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Republic of Moldova", + "alpha2": "MD", + "numeric": 498, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Montenegro", + "alpha2": "ME", + "numeric": 499, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Martin", + "alpha2": "MF", + "numeric": 663, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Madagascar", + "alpha2": "MG", + "numeric": 450, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Marshall Islands", + "alpha2": "MH", + "numeric": 584, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "The former Yugoslav Republic of Macedonia", + "alpha2": "MK", + "numeric": 807, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mali", + "alpha2": "ML", + "numeric": 466, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Myanmar", + "alpha2": "MM", + "numeric": 104, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mongolia", + "alpha2": "MN", + "numeric": 496, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Macao", + "alpha2": "MO", + "numeric": 446, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Northern Mariana Islands", + "alpha2": "MP", + "numeric": 580, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Martinique", + "alpha2": "MQ", + "numeric": 474, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mauritania", + "alpha2": "MR", + "numeric": 478, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Montserrat", + "alpha2": "MS", + "numeric": 500, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Malta", + "alpha2": "MT", + "numeric": 470, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mauritius", + "alpha2": "MU", + "numeric": 480, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Maldives", + "alpha2": "MV", + "numeric": 462, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Malawi", + "alpha2": "MW", + "numeric": 454, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mexico", + "alpha2": "MX", + "numeric": 484, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Malaysia", + "alpha2": "MY", + "numeric": 458, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Mozambique", + "alpha2": "MZ", + "numeric": 508, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Namibia", + "alpha2": "NA", + "numeric": 516, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "New Caledonia", + "alpha2": "NC", + "numeric": 540, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Niger", + "alpha2": "NE", + "numeric": 562, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Norfolk Island", + "alpha2": "NF", + "numeric": 574, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nigeria", + "alpha2": "NG", + "numeric": 566, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nicaragua", + "alpha2": "NI", + "numeric": 558, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Netherlands", + "alpha2": "NL", + "numeric": 528, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Norway", + "alpha2": "NO", + "numeric": 578, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nepal", + "alpha2": "NP", + "numeric": 524, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Nauru", + "alpha2": "NR", + "numeric": 520, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Niue", + "alpha2": "NU", + "numeric": 570, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "New Zealand", + "alpha2": "NZ", + "numeric": 554, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Oman", + "alpha2": "OM", + "numeric": 512, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Panama", + "alpha2": "PA", + "numeric": 591, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Peru", + "alpha2": "PE", + "numeric": 604, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "French Polynesia", + "alpha2": "PF", + "numeric": 258, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Papua New Guinea", + "alpha2": "PG", + "numeric": 598, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Philippines", + "alpha2": "PH", + "numeric": 608, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Pakistan", + "alpha2": "PK", + "numeric": 586, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Poland", + "alpha2": "PL", + "numeric": 616, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Pierre and Miquelon", + "alpha2": "PM", + "numeric": 666, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Pitcairn", + "alpha2": "PN", + "numeric": 612, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Puerto Rico", + "alpha2": "PR", + "numeric": 630, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Palestinian Territory", + "alpha2": "PS", + "numeric": 275, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Portugal", + "alpha2": "PT", + "numeric": 620, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Palau", + "alpha2": "PW", + "numeric": 585, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Paraguay", + "alpha2": "PY", + "numeric": 600, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Qatar", + "alpha2": "QA", + "numeric": 634, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Reunion", + "alpha2": "RE", + "numeric": 638, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Romania", + "alpha2": "RO", + "numeric": 642, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Serbia", + "alpha2": "RS", + "numeric": 688, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Russian Federation", + "alpha2": "RU", + "numeric": 643, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Rwanda", + "alpha2": "RW", + "numeric": 646, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saudi Arabia", + "alpha2": "SA", + "numeric": 682, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Solomon Islands", + "alpha2": "SB", + "numeric": 90, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Seychelles", + "alpha2": "SC", + "numeric": 690, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Sudan", + "alpha2": "SD", + "numeric": 729, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Sweden", + "alpha2": "SE", + "numeric": 752, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Singapore", + "alpha2": "SG", + "numeric": 702, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Helena", + "alpha2": "SH", + "numeric": 654, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Slovenia", + "alpha2": "SI", + "numeric": 705, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Svalbard and Jan Mayen", + "alpha2": "SJ", + "numeric": 744, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Slovakia", + "alpha2": "SK", + "numeric": 703, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Sierra Leone", + "alpha2": "SL", + "numeric": 694, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "San Marino", + "alpha2": "SM", + "numeric": 674, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Senegal", + "alpha2": "SN", + "numeric": 686, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Somalia", + "alpha2": "SO", + "numeric": 706, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Suriname", + "alpha2": "SR", + "numeric": 740, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "South Sudan", + "alpha2": "SS", + "numeric": 728, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Sao Tome and Principe", + "alpha2": "ST", + "numeric": 678, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "El Salvador", + "alpha2": "SV", + "numeric": 222, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Syrian Arab Republic", + "alpha2": "SY", + "numeric": 760, + "residencyRisk": "Embargoed", + "typesNotApplyDataResidency": [] +}, { + "name": "Swaziland", + "alpha2": "SZ", + "numeric": 748, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Turks and Caicos Islands", + "alpha2": "TC", + "numeric": 796, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Chad", + "alpha2": "TD", + "numeric": 148, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Togo", + "alpha2": "TG", + "numeric": 768, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Thailand", + "alpha2": "TH", + "numeric": 764, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tajikistan", + "alpha2": "TJ", + "numeric": 762, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tokelau", + "alpha2": "TK", + "numeric": 772, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Timor-Leste", + "alpha2": "TL", + "numeric": 626, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Turkmenistan", + "alpha2": "TM", + "numeric": 795, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tunisia", + "alpha2": "TN", + "numeric": 788, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tonga", + "alpha2": "TO", + "numeric": 776, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Turkey", + "alpha2": "TR", + "numeric": 792, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Trinidad and Tobago", + "alpha2": "TT", + "numeric": 780, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Tuvalu", + "alpha2": "TV", + "numeric": 798, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Taiwan, Province of China", + "alpha2": "TW", + "numeric": 158, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United Republic of Tanzania", + "alpha2": "TZ", + "numeric": 834, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Ukraine", + "alpha2": "UA", + "numeric": 804, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Uganda", + "alpha2": "UG", + "numeric": 800, + "residencyRisk": "Not assigned", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United States Minor Outlying Islands", + "alpha2": "UM", + "numeric": 581, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "United States", + "alpha2": "US", + "numeric": 840, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Uruguay", + "alpha2": "UY", + "numeric": 858, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "Uzbekistan", + "alpha2": "UZ", + "numeric": 860, + "residencyRisk": "No restriction", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Saint Vincent and the Grenadines", + "alpha2": "VC", + "numeric": 670, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Venezuela", + "alpha2": "VE", + "numeric": 862, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "British Virgin Islands", + "alpha2": "VG", + "numeric": 92, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Virgin Islands, U.S.", + "alpha2": "VI", + "numeric": 850, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Vietnam", + "alpha2": "VN", + "numeric": 704, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Vanuatu", + "alpha2": "VU", + "numeric": 548, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Wallis and Futuna", + "alpha2": "WF", + "numeric": 876, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Samoa", + "alpha2": "WS", + "numeric": 882, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Yemen", + "alpha2": "YE", + "numeric": 887, + "residencyRisk": "Default", + "typesNotApplyDataResidency": [] +}, { + "name": "South Africa", + "alpha2": "ZA", + "numeric": 710, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Zambia", + "alpha2": "ZM", + "numeric": 894, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Zimbabwe", + "alpha2": "ZW", + "numeric": 716, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +}, { + "name": "Default", + "alpha2": "XX", + "numeric": 999, + "residencyRisk": "Default", + "typesNotApplyDataResidency": ["Transferred Data"] +} +] diff --git a/bicep/modules/script-blob-upload/script.sh b/bicep/modules/deploy-scripts/blob_upload.sh similarity index 74% rename from bicep/modules/script-blob-upload/script.sh rename to bicep/modules/deploy-scripts/blob_upload.sh index d259c604..e9dfb462 100644 --- a/bicep/modules/script-blob-upload/script.sh +++ b/bicep/modules/deploy-scripts/blob_upload.sh @@ -1,14 +1,12 @@ #!/bin/bash set -e -echo "Waiting on Identity RBAC replication ($initialDelay)" -sleep $initialDelay - # Installing curl apk add --no-cache curl echo "$CONTENT" > ${FILE_NAME} # Upload the blob, overwriting if it exists -az storage blob upload -f ${FILE_NAME} -c ${CONTAINER} -n ${FILE_NAME} --overwrite +az login --identity +az storage blob upload -f ${FILE_NAME} -c ${CONTAINER} -n ${FILE_NAME} --overwrite --auth-mode login echo "Blob ${CONTAINER} uploaded to container ${CONTAINER}, overwriting if it existed." \ No newline at end of file diff --git a/bicep/modules/deploy-scripts/software-upload.sh b/bicep/modules/deploy-scripts/software-upload.sh new file mode 100644 index 00000000..d0e1b72d --- /dev/null +++ b/bicep/modules/deploy-scripts/software-upload.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +# Installing required packages +apk add --no-cache curl zip unzip + +# Download the file using curl +echo "Downloading file from ${URL}" +curl -L -o repo.zip "${URL}" + +# Check if the download was successful +if [ $? -ne 0 ]; then + echo "Failed to download the file from ${URL}" + exit 1 +fi + +# Create a directory for extracted files +mkdir -p extracted_files + +# Unzip the file +echo "Extracting contents..." +unzip -q repo.zip -d extracted_files + +# Find and replace 'kind: GitRepository' with 'kind: Bucket' in all files +find extracted_files -type f -path "*/${UPLOAD_DIR}/*" -exec sed -i ' + /sourceRef:/{ + N;N;N + s/sourceRef:\n[[:space:]]*kind: GitRepository\n[[:space:]]*name: flux-system\n[[:space:]]*namespace: flux-system/sourceRef:\n kind: Bucket\n name: flux-system\n namespace: flux-system/g + }' {} + + +# Find the software directory +software_dir=$(find extracted_files -type d -name "${UPLOAD_DIR}" -exec dirname {} \;) + +if [ -z "$software_dir" ]; then + echo "Error: '${UPLOAD_DIR}' directory not found in the extracted contents." + exit 1 +fi + +# Upload the contents of the software directory +echo "Uploading files from ${software_dir} to blob container ${CONTAINER}" +az storage blob upload-batch --destination ${CONTAINER} --source "${software_dir}" --pattern "${UPLOAD_DIR}/**" --overwrite true --auth-mode login +echo "Files from software directory uploaded to blob container ${CONTAINER}." + +# Clean up +rm -rf extracted_files repo.zip diff --git a/bicep/modules/script-blob-upload/main.bicep b/bicep/modules/script-blob-upload/main.bicep deleted file mode 100644 index ea7d2507..00000000 --- a/bicep/modules/script-blob-upload/main.bicep +++ /dev/null @@ -1,94 +0,0 @@ -metadata name = 'Blob Upload' -metadata description = 'This module uploads a file to a blob storage account' -metadata owner = 'azure-global-energy' - -@description('Desired name of the storage account') -param storageAccountName string = uniqueString(resourceGroup().id, deployment().name, 'blob') - -@description('Name of the blob container') -param containerName string = 'legal-service-azure-configuration' - -@description('Name of the blob as it is stored in the blob container') -param filename string = 'Legal_COO.json' - -@description('The location of the Storage Account and where to deploy the module resources to') -param location string = resourceGroup().location - -@description('How the deployment script should be forced to execute') -param forceUpdateTag string = utcNow() - -@description('Azure RoleId that are required for the DeploymentScript resource to upload blobs') -param rbacRoleNeeded string = '' //Storage Blob Contributor is needed to upload secrets into Storage Account - -@description('Does the Managed Identity already exists, or should be created') -param useExistingManagedIdentity bool = false - -@description('Name of the Managed Identity resource') -param managedIdentityName string = 'id-storage-blob-${location}' - -@description('For an existing Managed Identity, the Subscription Id it is located in') -param existingManagedIdentitySubId string = subscription().subscriptionId - -@description('For an existing Managed Identity, the Resource Group it is located in') -param existingManagedIdentityResourceGroupName string = resourceGroup().name - -@description('A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate') -param initialScriptDelay string = '30s' - -@allowed([ 'OnSuccess', 'OnExpiration', 'Always' ]) -@description('When the script resource is cleaned up') -param cleanupPreference string = 'OnSuccess' - - -resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = { - name: storageAccountName -} - -resource newDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (!useExistingManagedIdentity) { - name: managedIdentityName - location: location -} - -resource existingDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (useExistingManagedIdentity) { - name: managedIdentityName - scope: resourceGroup(existingManagedIdentitySubId, existingManagedIdentityResourceGroupName) -} - -resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(rbacRoleNeeded)) { - name: guid(storageAccount.id, rbacRoleNeeded, useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id) - scope: storageAccount - properties: { - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', rbacRoleNeeded) - principalId: useExistingManagedIdentity ? existingDepScriptId.properties.principalId : newDepScriptId.properties.principalId - principalType: 'ServicePrincipal' - } -} - -resource uploadFile 'Microsoft.Resources/deploymentScripts@2023-08-01' = { - name: 'script-${storageAccount.name}-${replace(replace(filename, ':', ''), '/', '-')}' - location: location - identity: { - type: 'UserAssigned' - userAssignedIdentities: { '${useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id}': {} } - } - kind: 'AzureCLI' - dependsOn: [ rbac ] - properties: { - forceUpdateTag: forceUpdateTag - azCliVersion: '2.63.0' - timeout: 'PT30M' - retentionInterval: 'PT1H' - environmentVariables: [ - { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } - { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } - { name: 'CONTENT', value: loadTextContent('./Legal_COO.json') } - { name: 'FILE_NAME', value: filename } - { name: 'CONTAINER', value: containerName } - { name: 'initialDelay', value: initialScriptDelay } - ] - scriptContent: loadTextContent('script.sh') - // scriptContent: 'echo "$CONTENT" > ${filename} && az storage blob upload -f ${filename} -c ${containerName} -n ${filename}' - cleanupPreference: cleanupPreference - } -} - diff --git a/bicep/modules/script-kv-certificate/README.md b/bicep/modules/script-kv-certificate/README.md deleted file mode 100644 index ee6f6268..00000000 --- a/bicep/modules/script-kv-certificate/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# Key Vault Certificate - -This module creates a Key Vault Certificate and stores it in an Azure Key Vault - -## Details - -{{Add detailed information about the module}} - -## Parameters - -| Name | Type | Required | Description | -| :----------------------------------------- | :------------: | :------: | :------------------------------------------------------------------------------------------------------------ | -| `kvName` | `string` | Yes | The name of the Azure Key Vault | -| `location` | `string` | Yes | The location to deploy the resources to | -| `forceUpdateTag` | `string` | No | How the deployment script should be forced to execute | -| `rbacRolesNeededOnKV` | `string` | No | The RoleDefinitionId required for the DeploymentScript resource to interact with KeyVault | -| `useExistingManagedIdentity` | `bool` | No | Does the Managed Identity already exists, or should be created | -| `managedIdentityName` | `string` | No | Name of the Managed Identity resource | -| `existingManagedIdentitySubId` | `string` | No | For an existing Managed Identity, the Subscription Id it is located in | -| `existingManagedIdentityResourceGroupName` | `string` | No | For an existing Managed Identity, the Resource Group it is located in | -| `certificateNames` | `array` | Yes | The names of the certificate to create. Use when creating many certificates. | -| `certificateCommonNames` | `array` | No | The common names of the certificate to create. Use when creating many certificates. | -| `initialScriptDelay` | `string` | No | A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate | -| `cleanupPreference` | `string` | No | When the script resource is cleaned up | -| `issuerName` | `string` | No | Self, or user defined {IssuerName} for certificate signing | -| `issuerProvider` | `string` | No | Certificate Issuer Provider, DigiCert, GlobalSign, or internal options may be used. | -| `disabled` | `bool` | No | Create certificate in disabled state. Default: false | -| `accountId` | `string` | No | Account ID of Certificate Issuer Account | -| `issuerPassword` | `securestring` | No | Password of Certificate Issuer Account | -| `organizationId` | `string` | No | Organization ID of Certificate Issuer Account | -| `isCrossTenant` | `bool` | No | Override this parameter if using this in cross tenant scenarios | -| `reuseKey` | `bool` | No | The default policy might cause errors about CSR being used before, so set this to false if that happens | -| `validity` | `int` | No | Optional. Override default validityInMonths 12 value | -| `performRoleAssignment` | `bool` | No | Set to false to disable role assignments within this module. Default: true | - -## Outputs - -| Name | Type | Description | -| :-------------------------------- | :-----: | :------------------------------------------------- | -| `certificateNames` | `array` | Certificate names | -| `certificateSecretIds` | `array` | KeyVault secret ids to the created version | -| `certificateSecretIdUnversioneds` | `array` | KeyVault secret ids which uses the unversioned uri | -| `certificateThumbpints` | `array` | Certificate Thumbprints | -| `certificateThumbprintHexs` | `array` | Certificate Thumbprints (in hex) | - -## Examples - -### Single KeyVault Certificate - -Creates a single self-signed certificate in Azure KeyVault. - -```bicep -param location string = resourceGroup().location -param akvName string = 'yourAzureKeyVault' -param certificateName string = 'myapp' - -module kvCert 'br/public:deployment-scripts/create-kv-certificate:3.4' = { - name: 'akvCertSingle' - params: { - akvName: akvName - location: location - certificateName: certificateName - } -} -output SecretId string = akvCertSingle.outputs.certificateSecretId -output Thumbprint string = akvCertSingle.outputs.certificateThumbprintHex - -``` - -### Single KeyVault Certificate with fqdn common name - -Creates a single self-signed certificate in Azure KeyVault using a specific certificate common name. - -```bicep -param location string = resourceGroup().location -param akvName string = 'yourAzureKeyVault' -param certificateName string = 'myapp' -param certificateCommonName string = '${certificateName}.mydomain.local' - -module kvCert 'br/public:deployment-scripts/create-kv-certificate:3.4' = { - name: 'akvCertSingle' - params: { - akvName: akvName - location: location - certificateNames: [certificateName] - certificateCommonNames: [certificateCommonName] - } -} -output SecretId string = akvCertSingle.outputs.certificateSecretIds[0] -output Thumbprint string = akvCertSingle.outputs.certificateThumbprintHexs[0] - -``` - -### Multiple KeyVault Certificates - -Create multiple self-signed certificates in Azure KeyVault - -```bicep -param location string = resourceGroup().location -param akvName string = 'yourAzureKeyVault' -param certificateNames array = [ - 'myapp' - 'myotherapp' -] - -module kvCert 'br/public:deployment-scripts/create-kv-certificate:3.1.1' = { - name: 'akvCert-${certificateName}' - params: { - akvName: akvName - location: location - certificateNames: certificateNames - } -} - -@description('Array of info from each Certificate') -output createdCertificates array = [for (certificateName, i) in certificateNames: { - certificateName: kvCert.outputs.certificateNames[i] - certificateSecretId: kvCert.outputs.certificateSecretIds[i] - certificateThumbprint: kvCert.outputs.certificateThumbprintHexs[i] -}] -``` - -### Create Signed Certificate - -Using `DigiCert` or `GlobalSign` first requires account setup described [here](https://learn.microsoft.com/en-us/azure/key-vault/certificates/how-to-integrate-certificate-authority) - -```bicep -param accountId -@secure -param issuerPassword -param organizationId - -module signedCert 'br/public:deployment-scripts/create-kv-certificate:3.1.1' = { - name: 'akvCert-${certificateName}' - params: { - akvName: akvName - location: location - certificateName: [certificateName] - certificateCommonName: ['customdomain.com'] - issuerName: 'MyCert' - issuerProvider: 'DigiCert' - accountId: accountId - issuerPassword: issuerPassword - organizationId: organizationId - } -}] -``` \ No newline at end of file diff --git a/bicep/modules/script-kv-certificate/main.bicep b/bicep/modules/script-kv-certificate/main.bicep deleted file mode 100644 index c9b97a89..00000000 --- a/bicep/modules/script-kv-certificate/main.bicep +++ /dev/null @@ -1,170 +0,0 @@ -metadata name = 'Key Vault Certificate' -metadata description = 'This module creates a Key Vault Certificate and stores it in an Azure Key Vault' -metadata owner = 'azure-global-energy' - -@description('The name of the Azure Key Vault') -param kvName string - -@description('The location to deploy the resources to') -param location string - -@description('How the deployment script should be forced to execute') -param forceUpdateTag string = utcNow() - -@description('The RoleDefinitionId required for the DeploymentScript resource to interact with KeyVault') -param rbacRolesNeededOnKV string = 'a4417e6f-fecd-4de8-b567-7b0420556985' //KeyVault Certificate Officer - -@description('Does the Managed Identity already exists, or should be created') -param useExistingManagedIdentity bool = false - -@description('Name of the Managed Identity resource') -param managedIdentityName string = 'id-KeyVaultCertificateCreator-${location}' - -@description('For an existing Managed Identity, the Subscription Id it is located in') -param existingManagedIdentitySubId string = subscription().subscriptionId - -@description('For an existing Managed Identity, the Resource Group it is located in') -param existingManagedIdentityResourceGroupName string = resourceGroup().name - -@description('The names of the certificate to create. Use when creating many certificates.') -param certificateNames array - -@description('The common names of the certificate to create. Use when creating many certificates.') -param certificateCommonNames array = certificateNames - -@description('A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate') -param initialScriptDelay string = '0' - -@allowed([ - 'OnSuccess' - 'OnExpiration' - 'Always' -]) -@description('When the script resource is cleaned up') -param cleanupPreference string = 'OnSuccess' - -@description('Self, or user defined {IssuerName} for certificate signing') -param issuerName string = 'Self' - -@description('Certificate Issuer Provider, DigiCert, GlobalSign, or internal options may be used.') -param issuerProvider string = '' - -@description('Create certificate in disabled state. Default: false') -param disabled bool = false - -@description('Account ID of Certificate Issuer Account') -param accountId string = '' - -@description('Password of Certificate Issuer Account') -@secure() -param issuerPassword string = '' - -@description('Organization ID of Certificate Issuer Account') -param organizationId string = '' - -@description('Override this parameter if using this in cross tenant scenarios') -param isCrossTenant bool = false - -@description('The default policy might cause errors about CSR being used before, so set this to false if that happens') -param reuseKey bool = true - -@minValue(1) -@maxValue(1200) -@description('Optional. Override default validityInMonths 12 value') -param validity int = 12 - -@description('Set to false to disable role assignments within this module. Default: true') -param performRoleAssignment bool = true - -resource akv 'Microsoft.KeyVault/vaults@2022-07-01' existing = { - name: kvName -} - -@description('A new managed identity that will be created in this Resource Group, this is the default option') -resource newDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = if (!useExistingManagedIdentity) { - name: managedIdentityName - location: location -} - -@description('An existing managed identity that could exist in another sub/rg') -resource existingDepScriptId 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' existing = if (useExistingManagedIdentity) { - name: managedIdentityName - scope: resourceGroup(existingManagedIdentitySubId, existingManagedIdentityResourceGroupName) -} - -var delegatedManagedIdentityResourceId = useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id - -resource rbacKv 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (performRoleAssignment) { - name: guid(akv.id, rbacRolesNeededOnKV, managedIdentityName, string(useExistingManagedIdentity)) - scope: akv - properties: { - roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', rbacRolesNeededOnKV) - principalId: useExistingManagedIdentity ? existingDepScriptId.properties.principalId : newDepScriptId.properties.principalId - principalType: 'ServicePrincipal' - delegatedManagedIdentityResourceId: isCrossTenant ? delegatedManagedIdentityResourceId : null - } -} - -resource createImportCerts 'Microsoft.Resources/deploymentScripts@2023-08-01' = [for (certificateName, index) in certificateNames: { - name: 'script-${akv.name}-${replace(replace(certificateName, ':', ''), '/', '-')}' - location: location - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${useExistingManagedIdentity ? existingDepScriptId.id : newDepScriptId.id}': {} - } - } - kind: 'AzureCLI' - dependsOn: [ - rbacKv - ] - properties: { - forceUpdateTag: forceUpdateTag - azCliVersion: '2.63.0' - timeout: 'PT30M' - retentionInterval: 'PT1H' - environmentVariables: [ - { name: 'akvName', value: kvName } - { name: 'certName', value: certificateName } - { name: 'certCommonName', value: certificateCommonNames[index] } - { name: 'initialDelay', value: initialScriptDelay } - { name: 'issuerName', value: issuerName } - { name: 'issuerProvider', value: issuerProvider } - { name: 'disabled', value: toLower(string(disabled)) } - { name: 'retryMax', value: '10' } - { name: 'retrySleep', value: '5s' } - { name: 'accountId', value: accountId } - { name: 'issuerPassword', secureValue: issuerPassword } - { name: 'organizationId', value: organizationId } - { name: 'reuseKey', value: toLower(string(reuseKey)) } - { name: 'validity', value: string(validity) } - ] - scriptContent: loadTextContent('script.sh') - cleanupPreference: cleanupPreference - } -}] - -@description('Certificate names') -output certificateNames array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.name -]] - -@description('KeyVault secret ids to the created version') -output certificateSecretIds array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.certSecretId.versioned -]] - -@description('KeyVault secret ids which uses the unversioned uri') -output certificateSecretIdUnversioneds array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.certSecretId.unversioned -]] - -@description('Certificate Thumbprints') -output certificateThumbpints array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.thumbprint -]] - -@description('Certificate Thumbprints (in hex)') -output certificateThumbprintHexs array = [for (certificateName, index) in certificateNames: [ - createImportCerts[index].properties.outputs.thumbprintHex -]] diff --git a/bicep/modules/script-kv-certificate/script.sh b/bicep/modules/script-kv-certificate/script.sh deleted file mode 100644 index 93838ad5..00000000 --- a/bicep/modules/script-kv-certificate/script.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -set -e -initialDelay="${initialDelay:-5}" -retryMax="${retryMax:-5}" -certName="${certName:-default-cert}" -certCommonName="${certCommonName:-default}" -validity="${validity:-12}" -akvName="${akvName:-keyvault}" -issuerName="${issuerName:-}" -reuseKey="${reuseKey:-true}" -retrySleep="${retrySleep:-5}" - -echo "Waiting on Identity RBAC replication (\"$initialDelay\")" -sleep "$initialDelay" - -#Retry loop to catch errors (usually RBAC delays) -retryLoopCount=0 -until [ "$retryLoopCount" -ge "$retryMax" ] -do - echo "Creating AKV Cert $certName with CN $certCommonName (attempt $retryLoopCount)..." - - if [ -z "$issuerName" ] || [ -z "$issuerProvider" ]; then - policy=$(az keyvault certificate get-default-policy \ - | sed -e s/\"validityInMonths\":\ 12/\"validityInMonths\":\ "${validity}"/g \ - | sed -e s/CN=CLIGetDefaultPolicy/CN="${certCommonName}"/g ) - else - if [ "$issuerProvider" == "DigiCert" ] || [ "$issuerProvider" == "GlobalCert" ]; then - az keyvault certificate issuer create \ - --vault-name "$akvName" \ - --issuer-name "$issuerName" \ - --provider-name "$issuerProvider" \ - --account-id "$accountId" \ - --password "$issuerPassword" \ - --organizatiion-id "$organizationId" - else - az keyvault certificate issuer create \ - --vault-name "$akvName" \ - --issuer-name "$issuerName" \ - --provider-name "$issuerProvider" - fi - policy=$(az keyvault certificate get-default-policy \ - | sed -e s/\"validityInMonths\":\ 12/\"validityInMonths\":\ "${validity}"/g \ - | sed -e s/CN=CLIGetDefaultPolicy/CN="${certCommonName}"/g \ - | sed -e s/\"name\":\ \"Self\"/\"name\":\ \""${issuerName}"\"/g \ - | sed -e s/\"reuseKey\":\ true/\"reuseKey\":\ "${reuseKey}"/g ) - fi - az keyvault certificate create \ - --vault-name "$akvName" \ - -n "$certName" \ - -p "$policy" \ - --disabled "$disabled" \ - && break - - sleep "$retrySleep" - retryLoopCount=$((retryLoopCount+1)) -done - -echo "Getting Certificate $certName"; -retryLoopCount=0 -createdCert=$(az keyvault certificate show -n "$certName" --vault-name "$akvName" -o json) -while [ -z "$(echo "$createdCert" | jq -r '.x509ThumbprintHex')" ] && [ $retryLoopCount -lt "$retryMax" ] -do - echo "Waiting for cert creation (attempt $retryLoopCount)..." - sleep $retrySleep - createdCert=$(az keyvault certificate show -n $certName --vault-name $akvName -o json) - retryLoopCount=$((retryLoopCount+1)) -done - -unversionedSecretId=$(echo $createdCert | jq -r ".sid" | cut -d'/' -f-5) # remove the version from the url; -jsonOutputString=$(echo $createdCert | jq --arg usid $unversionedSecretId '{name: .name ,certSecretId: {versioned: .sid, unversioned: $usid }, thumbprint: .x509Thumbprint, thumbprintHex: .x509ThumbprintHex}') -echo $jsonOutputString > $AZ_SCRIPTS_OUTPUT_PATH \ No newline at end of file diff --git a/bicep/modules/script-kv-certificate/test/main.test.bicep b/bicep/modules/script-kv-certificate/test/main.test.bicep deleted file mode 100644 index 5bd3e973..00000000 --- a/bicep/modules/script-kv-certificate/test/main.test.bicep +++ /dev/null @@ -1,104 +0,0 @@ -targetScope = 'resourceGroup' - -@minLength(3) -@maxLength(10) -@description('Used to name all resources') -param resourceName string - -@description('Registry Location.') -param location string = resourceGroup().location - -//Prerequisites -module identity '../../user-managed-identity/main.bicep' = { - name: 'user-managed-identity' - params: { - resourceName: resourceName - location: location - } -} - -module kv '../../azure-keyvault/main.bicep' = { - name: 'azure_keyvault' - params: { - resourceName: resourceName - location: location - secretsObject: { secrets: [] } - - // Add Role Assignment - roleAssignments: [ - { - roleDefinitionIdOrName: 'Key Vault Administrator' - principals: [ - { - id: identity.outputs.principalId - } - ] - principalType: 'ServicePrincipal' - } - ] - } -} - -//Test 1. Just a single certificate -module kvCertSingle '../main.bicep' = { - name: 'kvCertSingle' - params: { - kvName: kv.outputs.name - location: location - - useExistingManagedIdentity: true - managedIdentityName: identity.outputs.name - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name - - certificateNames: [ 'mysingleapp' ] - certificateCommonNames: [ 'mysingleapp.mydomain.local' ] - validity: 11 - disabled: true - } -} -output singleSecretId string = kvCertSingle.outputs.certificateSecretIds[0][0] -output singleThumbprint string = kvCertSingle.outputs.certificateThumbprintHexs[0][0] - -//Test 2. Array of certificates -var certificateNames = [ - 'myapp' - 'myotherapp' -] - -module kvCertMultiple '../main.bicep' = { - name: 'kvCertMultiple-${uniqueString(kv.name)}' - params: { - kvName: kv.name - location: location - - useExistingManagedIdentity: true - managedIdentityName: identity.outputs.name - existingManagedIdentitySubId: subscription().subscriptionId - existingManagedIdentityResourceGroupName:resourceGroup().name - - certificateNames: certificateNames - initialScriptDelay: '0' - validity: 24 - } -} - -// Test 3. Test a signed cert -// module akvCertSigned '../main.bicep' = { -// name: 'akvCertSigned' -// params: { -// akvName: akv.name -// location: location -// certificateName: 'mysignedcert' -// certificateCommonName: 'sample-cert.gaming.azure.com' -// issuerName: 'Signed' -// issuerProvider: 'OneCertV2-PublicCA' -// } -// } - -@description('Array of info from each Certificate') -output createdCertificates array = [for (certificateName, i) in certificateNames: { - certificateName: certificateName - certificateSecretId: kvCertMultiple.outputs.certificateSecretIds[i] - certificateThumbprint: kvCertMultiple.outputs.certificateThumbprintHexs[i] -}] diff --git a/bicep/modules/script-kv-certificate/test/parameters.json b/bicep/modules/script-kv-certificate/test/parameters.json deleted file mode 100644 index 4d4c4d1d..00000000 --- a/bicep/modules/script-kv-certificate/test/parameters.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "template": "../main.bicep" - }, - "parameters": { - "location": { - "value": "southcentralus" - }, - "resourceName": { - "value": "adme" - } - } -} \ No newline at end of file diff --git a/bicep/modules/script-kv-certificate/version.json b/bicep/modules/script-kv-certificate/version.json deleted file mode 100644 index e215964e..00000000 --- a/bicep/modules/script-kv-certificate/version.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", - "version": "3.4", - "pathFilters": [ - "./main.json" - ] -} \ No newline at end of file diff --git a/bicep/modules/script-share-csvdag/main.bicep b/bicep/modules/script-share-csvdag/main.bicep index d05b7e8f..5975709e 100644 --- a/bicep/modules/script-share-csvdag/main.bicep +++ b/bicep/modules/script-share-csvdag/main.bicep @@ -130,7 +130,7 @@ module deploymentScript 'br/public:avm/res/resources/deployment-script:0.4.0' = environmentVariables: [ { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } - { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } + // { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } { name: 'FILE', value: filename } { name: 'URL', value: fileurl } { name: 'SHARE', value: shareName } diff --git a/bicep/modules/script-share-csvdag/script.sh b/bicep/modules/script-share-csvdag/script.sh index ebab9ba1..62d9e7e0 100644 --- a/bicep/modules/script-share-csvdag/script.sh +++ b/bicep/modules/script-share-csvdag/script.sh @@ -140,5 +140,5 @@ cd "extracted_files/${FILE}" || exit 1 zip -r "${current_dir}/${zip_filename}" . cd - || exit 1 -az storage file upload -s "${SHARE}" --source "${zip_filename}" -onone +az storage file upload -s "${SHARE}" --source "${zip_filename}" --enable-file-backup-request-intent --auth-mode login -onone echo "Zip file ${zip_filename} uploaded to file share ${SHARE}." \ No newline at end of file diff --git a/bicep/modules/script-share-upload/main.bicep b/bicep/modules/script-share-upload/main.bicep index cbc1d3f1..66698ad5 100644 --- a/bicep/modules/script-share-upload/main.bicep +++ b/bicep/modules/script-share-upload/main.bicep @@ -75,7 +75,7 @@ module deploymentScript 'br/public:avm/res/resources/deployment-script:0.4.0' = environmentVariables: [ { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } - { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } + // { name: 'AZURE_STORAGE_KEY', value: storageAccount.listKeys().keys[0].value } { name: 'FILE', value: filename } { name: 'URL', value: fileurl } { name: 'SHARE', value: shareName } diff --git a/bicep/modules/script-share-upload/script.sh b/bicep/modules/script-share-upload/script.sh index ea47f1c4..3e0404bf 100644 --- a/bicep/modules/script-share-upload/script.sh +++ b/bicep/modules/script-share-upload/script.sh @@ -43,17 +43,17 @@ if [[ ${URL} == *.tar.gz ]]; then # Navigate back to the original directory cd ${original_dir} # Upload the zip file to the file share - az storage file upload -s ${SHARE} --source ./${zip_filename} -onone + az storage file upload -s ${SHARE} --source ./${zip_filename} --enable-file-backup-request-intent --auth-mode login -onone echo "Zip file ${zip_filename} uploaded to file share ${SHARE}." else # Batch upload the extracted files to the file share using the specified pattern echo "Uploading extracted files to file share ${SHARE} with pattern ${FILE}/**" - az storage file upload-batch -d ${SHARE} --source extracted_files --pattern "${FILE}/**" --no-progress -onone + az storage file upload-batch -d ${SHARE} --source extracted_files --pattern "${FILE}/**" --no-progress --enable-file-backup-request-intent --auth-mode login -onone fi echo "Files from ${url_basename} uploaded to file share ${SHARE}." else # Upload the file to the file share, overwriting if it exists echo "Uploading file ${FILE} to file share ${SHARE}" - az storage file upload -s ${SHARE} --source ./${FILE} -onone + az storage file upload -s ${SHARE} --source ./${FILE} --enable-file-backup-request-intent --auth-mode login -onone echo "File ${FILE} uploaded to file share ${SHARE}, overwriting if it existed." fi \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/.bicep/keyvault_secrets.bicep b/bicep/modules/storage-account-orig/.bicep/keyvault_secrets.bicep deleted file mode 100644 index e457c8e7..00000000 --- a/bicep/modules/storage-account-orig/.bicep/keyvault_secrets.bicep +++ /dev/null @@ -1,31 +0,0 @@ -@description('Conditional. The name of the parent key vault. Required if the template is used in a standalone deployment.') -param keyVaultName string - -@description('Required. The name of the secret.') -param name string - -@description('Required. The value of the secret. NOTE: "value" will never be returned from the service, as APIs using this model are is intended for internal use in ARM deployments. Users should use the data-plane REST service for interaction with vault secrets.') -@secure() -param value string - -resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { - name: keyVaultName -} - -resource secret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { - name: name - parent: keyVault - - properties: { - value: value - } -} - -@description('The name of the secret.') -output name string = secret.name - -@description('The resource ID of the secret.') -output resourceId string = secret.id - -@description('The name of the resource group the secret was created in.') -output resourceGroupName string = resourceGroup().name diff --git a/bicep/modules/storage-account-orig/.bicep/nested_rbac.bicep b/bicep/modules/storage-account-orig/.bicep/nested_rbac.bicep deleted file mode 100644 index 7ee193d6..00000000 --- a/bicep/modules/storage-account-orig/.bicep/nested_rbac.bicep +++ /dev/null @@ -1,70 +0,0 @@ -param description string = '' -param principalType string = '' -param roleDefinitionIdOrName string -param resourceId string - -@sys.description('Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment\'s principal object.') -param crossTenant bool = false - -@sys.description('Required. The IDs of the principals to assign the role to. A resourceId is required when used in a cross tenant scenario (i.e. crossTenant is true)') -param principals array - /* example - [ - { - id: '222222-2222-2222-2222-2222222222' - resourceId: '/subscriptions/111111-1111-1111-1111-1111111111/resourcegroups/rg-osdu-bicep/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ManagedIdentityName' - } - ] - */ - -var builtInRoleNames = { - Owner: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') - Contributor: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') - Reader: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') - 'Avere Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4f8fab4f-1852-4a58-a46a-8eaf358af14a') - 'Avere Operator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c025889f-8102-4ebf-b32c-fc0c6f0c6bd9') - 'Backup Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5e467623-bb1f-42f4-a55d-6e525e11384b') - 'Backup Operator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '00c29273-979b-4161-815c-10b084fb9324') - 'Backup Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a795c7a0-d4a2-40c1-ae25-d81f01202912') - 'Classic Storage Account Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '') - 'Classic Storage Account Key Operator Service Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '') - 'Data Box Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'add466c9-e687-43fc-8d98-dfcf8d720be5') - 'Data Box Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '028f4ed7-e2a9-465e-a8f4-9c0ffdfdc027') - 'Data Lake Analytics Developer': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '47b7735b-770e-4598-a7da-8b91488b4c88') - 'Elastic SAN Owner': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '80dcbedb-47ef-405d-95bd-188a1b4ac406') - 'Elastic SAN Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'af6a70f8-3c9f-4105-acf1-d719e9fca4ca') - 'Elastic SAN Volume Group Owner': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a8281131-f312-4f34-8d98-ae12be9f0d23') - 'Reader and Data Access': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c12c1c16-33a1-487b-954d-41c89c60f349') - 'Storage Account Backup Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'e5e2a7ff-d759-4cd2-bb51-3152d37e2eb1') - 'Storage Account Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'e5e2a7ff-d759-4cd2-bb51-3152d37e2eb1') - 'Storage Account Key Operator Service Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '81a9662b-bebf-436f-a333-f67b29880f12') - 'Storage Blob Data Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') - 'Storage Blob Data Owner': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b') - 'Storage Blob Data Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b') - 'Storage Blob Delegator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'db58b8e5-c6ad-4a2a-8342-4190687cbf4a') - 'Storage File Data SMB Share Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0c867c2a-1d8c-454a-a3db-ab2ea1bdc8bb') - 'Storage File Data SMB Share Elevated Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'a7264617-510b-434b-a828-9731dc254ea7') - 'Storage File Data SMB Share Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'aba4ae5f-2193-4029-9191-0cb91df5e314') - 'Storage Queue Data Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '974c5e8b-45b9-4653-ba55-5f855dd0fb88') - 'Storage Queue Data Message Processor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8a0f0c08-91a1-4084-bc3d-661d67233fed') - 'Storage Queue Data Message Sender': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c6a89b2d-59bc-44d0-9896-0f6e12d7b80a') - 'Storage Queue Data Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '19e7f393-937e-4f77-808e-94535e297925') - 'Storage Table Data Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0a9a7e1f-b9d0-4cc4-a60d-0319b160aaa3') - 'Storage Table Data Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '76199698-9eea-4c19-bc75-cec21354c6b6') -} - -resource storage 'Microsoft.Storage/storageAccounts@2021-04-01' existing = { - name: last(split(resourceId, '/')) -} - -resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for principal in principals: { - name: guid(storage.name, principal.id, roleDefinitionIdOrName) - properties: { - description: description - roleDefinitionId: builtInRoleNames[?roleDefinitionIdOrName] ?? roleDefinitionIdOrName - principalId: principal.id - principalType: !empty(principalType) ? principalType : null - delegatedManagedIdentityResourceId: crossTenant ? principal.resourceId : null - } - scope: storage -}] diff --git a/bicep/modules/storage-account-orig/README.md b/bicep/modules/storage-account-orig/README.md deleted file mode 100644 index 8bcfc8dc..00000000 --- a/bicep/modules/storage-account-orig/README.md +++ /dev/null @@ -1,128 +0,0 @@ -# Azure Storage Module - -This module deploys an Azure Storage Account. - -## Description - -This module supports the following features. - -- Containers -- Tables -- Shares -- Role Assignments -- Diagnostics -- Private Link (Secure network) -- Customer Managed Encryption Keys -- Point in Time Restore -- SAS Secrets - -## Parameters - -| Name | Type | Required | Description | -| :-------------------------------------- | :------: | :------: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `resourceName` | `string` | Yes | Used to name all resources | -| `location` | `string` | No | Resource Location. | -| `tags` | `object` | No | Tags. | -| `lock` | `string` | No | Optional. Specify the type of lock. | -| `sku` | `string` | No | Specifies the storage account sku type. | -| `accessTier` | `string` | No | Specifies the storage account access tier. | -| `containers` | `array` | No | Optional. Array of Storage Containers to be created. | -| `tables` | `array` | No | Optional. Array of Storage Tables to be created. | -| `shares` | `array` | No | Optional. Array of Storage Shares to be created. | -| `shareQuota` | `int` | No | Optional. The maximum size of the share, in gigabytes. Must be greater than 0, and less than or equal to 5120 (5TB). For Large File Shares, the maximum size is 102400 (100TB). | -| `enabledProtocols` | `string` | No | Optional. The authentication protocol that is used for the file share. Can only be specified when creating a share. | -| `rootSquash` | `string` | No | Optional. Permissions for NFS file shares are enforced by the client OS rather than the Azure Files service. Toggling the root squash behavior reduces the rights of the root user for NFS shares. | -| `crossTenant` | `bool` | No | Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment's principal object. | -| `roleAssignments` | `array` | No | Optional. Array of objects that describe RBAC permissions, format { roleDefinitionResourceId (string), principalId (string), principalType (enum), enabled (bool) }. Ref: https://docs.microsoft.com/en-us/azure/templates/microsoft.authorization/roleassignments?tabs=bicep | -| `diagnosticWorkspaceId` | `string` | No | Optional. Resource ID of the diagnostic log analytics workspace. | -| `diagnosticStorageAccountId` | `string` | No | Optional. Resource ID of the diagnostic storage account. | -| `diagnosticEventHubAuthorizationRuleId` | `string` | No | Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to. | -| `diagnosticEventHubName` | `string` | No | Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category. | -| `diagnosticLogsRetentionInDays` | `int` | No | Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely. | -| `logsToEnable` | `array` | No | Optional. The name of logs that will be streamed. | -| `metricsToEnable` | `array` | No | Optional. The name of metrics that will be streamed. | -| `cmekConfiguration` | `object` | No | Optional. Customer Managed Encryption Key. | -| `deleteRetention` | `int` | No | Amount of days the soft deleted data is stored and available for recovery. 0 is off. | -| `privateLinkSettings` | `object` | No | Settings Required to Enable Private Link | -| `keyVaultName` | `string` | No | Optional: Key Vault Name to store secrets into | -| `storageAccountSecretName` | `string` | No | Optional: To save storage account name into vault set the secret name. | -| `storageAccountKeySecretName` | `string` | No | Optional: To save storage account key into vault set the secret name. | -| `storageAccountConnectionString` | `string` | No | Optional: To save storage account connectionstring into vault set the secret name. | -| `basetime` | `string` | No | Optional: Current Date Time | -| `sasProperties` | `object` | No | Optional: Default SAS TOken Properties to download Blob. | -| `saveToken` | `bool` | No | Optional: To save storage account sas token into vault set the properties. | - -## Outputs - -| Name | Type | Description | -| :--- | :----: | :------------------------ | -| id | string | The resource ID. | -| name | string | The name of the resource. | - -## Examples - -### Example 1 - -```bicep -module storage 'br:osdubicep.azurecr.io/public/storage-account:1.0.8' = { - name: 'storage_account' - params: { - resourceName: resourceName - location: location - sku: 'Standard_LRS' - } -} -``` - -### Example 2 - -```bicep -module storage 'br:osdubicep.azurecr.io/public/storage-account:1.0.8' = { - name: 'storage_account' - params: { - resourceName: resourceName - location: location - sku: 'Standard_LRS' - - containers: [ - 'container1' - 'another' - ] - - tables: [ - 'table1' - 'another' - ] - - // Add Role Assignment - roleAssignments: [ - { - roleDefinitionIdOrName: 'Storage Blob Data Contributor' - principalIds: [ - identity.outputs.principalId - ] - principalType: 'ServicePrincipal' - } - ] - - // Enable Diagnostics - diagnosticWorkspaceId: logs.outputs.id - - // Enable Private Link - privateLinkSettings:{ - vnetId: network.outputs.id - subnetId: network.outputs.subnetIds[0] - } - - // Enable Customer Managed Encryption Key - cmekConfiguration: { - kvUrl: 'https://akeyvault.vault.azure.net' - keyName: 'akey' - identityId: '/subscriptions/222222-2222-2222-2222-2222222222/resourcegroups/agroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aidentity' - } - - // Enable Point in Time Restore - deleteRetention: 7 - } -} -``` \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/main.bicep b/bicep/modules/storage-account-orig/main.bicep deleted file mode 100644 index 69046c69..00000000 --- a/bicep/modules/storage-account-orig/main.bicep +++ /dev/null @@ -1,574 +0,0 @@ -targetScope = 'resourceGroup' - -@description('Used to name all resources') -param name string - -@description('Resource Location.') -param location string = resourceGroup().location - -@description('Tags.') -param tags object = {} - -@allowed([ - 'CanNotDelete' - 'NotSpecified' - 'ReadOnly' -]) -@description('Optional. Specify the type of lock.') -param lock string = 'NotSpecified' - -@description('Specifies the storage account sku type.') -@allowed([ - 'Standard_LRS' - 'Premium_LRS' - 'Standard_GRS' -]) -param sku string = 'Standard_LRS' - -@description('Specifies the storage account access tier.') -@allowed([ - 'Cool' - 'Hot' -]) -param accessTier string = 'Hot' - - -@description('Optional. Array of Storage Containers to be created.') -param containers array = [ - /* example - 'one' - 'two' - */ -] - -@description('Optional. Array of Storage Tables to be created.') -param tables array = [ - /* example - 'one' - 'two' - */ -] - -@description('Optional. Array of Storage Shares to be created.') -param shares array = [ - /* example - 'one' - 'two' - */ -] - -@description('Optional. The maximum size of the share, in gigabytes. Must be greater than 0, and less than or equal to 5120 (5TB). For Large File Shares, the maximum size is 102400 (100TB).') -param shareQuota int = 5120 - -@allowed([ - 'NFS' - 'SMB' -]) -@description('Optional. The authentication protocol that is used for the file share. Can only be specified when creating a share.') -param enabledProtocols string = 'SMB' - -@allowed([ - 'AllSquash' - 'NoRootSquash' - 'RootSquash' -]) -@description('Optional. Permissions for NFS file shares are enforced by the client OS rather than the Azure Files service. Toggling the root squash behavior reduces the rights of the root user for NFS shares.') -param rootSquash string = 'NoRootSquash' - -@description('Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment\'s principal object.') -param crossTenant bool = false - -@description('Optional. Array of objects that describe RBAC permissions, format { roleDefinitionResourceId (string), principalId (string), principalType (enum), enabled (bool) }. Ref: https://docs.microsoft.com/en-us/azure/templates/microsoft.authorization/roleassignments?tabs=bicep') -param roleAssignments array = [ - /* example - { - roleDefinitionIdOrName: 'Reader' - principals: [ - { - id: '222222-2222-2222-2222-2222222222' - resourceId: '/subscriptions/111111-1111-1111-1111-1111111111/resourcegroups/rg-osdu-bicep/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ManagedIdentityName' - } - ] - principalType: 'ServicePrincipal' - } - */ -] - -@description('Optional. Resource ID of the diagnostic log analytics workspace.') -param diagnosticWorkspaceId string = '' - -@description('Optional. Resource ID of the diagnostic storage account.') -param diagnosticStorageAccountId string = '' - -@description('Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to.') -param diagnosticEventHubAuthorizationRuleId string = '' - -@description('Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category.') -param diagnosticEventHubName string = '' - -@description('Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely.') -@minValue(0) -@maxValue(365) -param diagnosticLogsRetentionInDays int = 365 - -@description('Optional. The name of logs that will be streamed.') -@allowed([ - 'StorageRead' - 'StorageWrite' - 'StorageDelete' -]) -param logsToEnable array = [ - 'StorageRead' - 'StorageWrite' - 'StorageDelete' -] - -@description('Optional. The name of metrics that will be streamed.') -@allowed([ - 'AllMetrics' -]) -param metricsToEnable array = [ - 'AllMetrics' -] - -@description('Optional. Customer Managed Encryption Key.') -param cmekConfiguration object = { - kvUrl: '' - keyName: '' - identityId: '' -} - -@description('Amount of days the soft deleted data is stored and available for recovery. 0 is off.') -@minValue(0) -@maxValue(7) -param deleteRetention int = 0 - -@description('Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false.') -param allowBlobPublicAccess bool = false - -@description('Optional. Indicates whether shared key access is enabled for the storage account.') -param allowSharedKeyAccess bool = true - -@description('Optional. Default to Microsoft Entra authorization in the Azure portal.') -param defaultToOAuthAuthentication bool = true - -var enableCMEK = !empty(cmekConfiguration.kvUrl) && !empty(cmekConfiguration.keyName) && !empty(cmekConfiguration.identityId) ? true : false - -var diagnosticsLogs = [for log in logsToEnable: { - category: log - enabled: true - retentionPolicy: { - enabled: true - days: diagnosticLogsRetentionInDays - } -}] - -var diagnosticsMetrics = [for metric in metricsToEnable: { - category: metric - timeGrain: null - enabled: true - retentionPolicy: { - enabled: true - days: diagnosticLogsRetentionInDays - } -}] - - - -// Create Storage Account -resource storage 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: length(name) > 24 ? substring(name, 0, 24) : name - location: location - tags: tags - sku: { - name: sku - } - kind: 'StorageV2' - - identity: enableCMEK ? { - type: 'UserAssigned' - userAssignedIdentities: { - '${cmekConfiguration.identityId}': {} - } - } : null - - properties: { - accessTier: accessTier - minimumTlsVersion: 'TLS1_2' - - encryption: enableCMEK ? { - identity: { - userAssignedIdentity: cmekConfiguration.identityId - } - services: { - blob: { - enabled: true - } - table: { - enabled: true - } - file: { - enabled: true - } - } - keySource: 'Microsoft.Keyvault' - keyvaultproperties: { - keyname: cmekConfiguration.keyName - keyvaulturi: cmekConfiguration.kvUrl - } - } : { - services: { - blob: { - enabled: true - } - table: { - enabled: true - } - file: { - enabled: true - } - } - keySource: 'Microsoft.Storage' - } - - allowBlobPublicAccess: allowBlobPublicAccess - defaultToOAuthAuthentication: defaultToOAuthAuthentication - allowSharedKeyAccess: allowSharedKeyAccess - - networkAcls: enablePrivateLink ? { - bypass: 'AzureServices' - defaultAction: 'Deny' - } : { - bypass: 'AzureServices' - defaultAction: 'Allow' - } - } -} - -resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = { - parent: storage - name: 'default' - properties: deleteRetention > 0 ? { - changeFeed: { - enabled: true - } - restorePolicy: { - enabled: true - days: 7 - } - isVersioningEnabled: true - deleteRetentionPolicy: { - enabled: true - days: max(deleteRetention, 1) - } - } : { - deleteRetentionPolicy: { - enabled: false - allowPermanentDelete: false - } - } -} - -resource tableServices 'Microsoft.Storage/storageAccounts/tableServices@2023-01-01' = { - name: 'default' - parent: storage - properties: {} -} - -resource fileServices 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' = { - name: 'default' - parent: storage - properties: { - protocolSettings: {} - shareDeleteRetentionPolicy: { - enabled: true - days: 7 - } - } -} - -resource storage_containers 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' = [for item in containers: { - parent: blobServices - name: item - properties: { - defaultEncryptionScope: '$account-encryption-key' - denyEncryptionScopeOverride: false - publicAccess: 'None' - } -}] - -resource storage_tables 'Microsoft.Storage/storageAccounts/tableServices/tables@2023-01-01' = [for item in tables: { - parent: tableServices - name: item -}] - -resource fileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = [for item in shares: { - parent: fileServices - name: item - properties: { - shareQuota: shareQuota - rootSquash: enabledProtocols == 'NFS' ? rootSquash : null - enabledProtocols: enabledProtocols - } -}] - -// Apply Resource Lock -resource resource_lock 'Microsoft.Authorization/locks@2020-05-01' = if (lock != 'NotSpecified') { - name: '${storage.name}-${lock}-lock' - properties: { - level: lock - notes: lock == 'CanNotDelete' ? 'Cannot delete resource or child resources.' : 'Cannot modify the resource or child resources.' - } - scope: storage -} - -module storage_rbac '.bicep/nested_rbac.bicep' = [for (roleAssignment, index) in roleAssignments: { - name: '${deployment().name}-rbac-${index}' - params: { - description: roleAssignment.?description ?? '' - principals: roleAssignment.principals - roleDefinitionIdOrName: roleAssignment.roleDefinitionIdOrName - principalType: roleAssignment.?principalType ?? '' - resourceId: storage.id - crossTenant: crossTenant - } -}] - - - -// Hook up Diagnostics -resource storage_diagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(diagnosticStorageAccountId) || !empty(diagnosticWorkspaceId) || !empty(diagnosticEventHubAuthorizationRuleId) || !empty(diagnosticEventHubName)) { - name: 'storage-diagnostics' - scope: blobServices - properties: { - storageAccountId: !empty(diagnosticStorageAccountId) ? diagnosticStorageAccountId : null - workspaceId: !empty(diagnosticWorkspaceId) ? diagnosticWorkspaceId : null - eventHubAuthorizationRuleId: !empty(diagnosticEventHubAuthorizationRuleId) ? diagnosticEventHubAuthorizationRuleId : null - eventHubName: !empty(diagnosticEventHubName) ? diagnosticEventHubName : null - metrics: diagnosticsMetrics - logs: diagnosticsLogs - } - dependsOn: [ - storage - ] -} - -@description('The resource ID.') -output id string = storage.id - -@description('The name of the resource.') -output name string = storage.name - -//////////////// -// Private Link -//////////////// - -@description('Settings Required to Enable Private Link') -param privateLinkSettings object = { - subnetId: '1' // Specify the Subnet for Private Endpoint - vnetId: '1' // Specify the Virtual Network for Virtual Network Link -} - -var enablePrivateLink = privateLinkSettings.vnetId != '1' && privateLinkSettings.subnetId != '1' - - -@description('Specifies the name of the private link to the Resource.') -var privateEndpointName = '${name}-PrivateEndpoint' - -var publicDNSZoneForwarder = 'blob.${environment().suffixes.storage}' -var privateDnsZoneName = 'privatelink.${publicDNSZoneForwarder}' - -resource privateDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (enablePrivateLink) { - name: privateDnsZoneName - location: 'global' - properties: {} -} - -resource privateEndpoint 'Microsoft.Network/privateEndpoints@2022-01-01' = if (enablePrivateLink) { - name: privateEndpointName - location: location - properties: { - privateLinkServiceConnections: [ - { - name: privateEndpointName - properties: { - privateLinkServiceId: storage.id - groupIds: [ - 'blob' - ] - } - } - ] - subnet: { - id: privateLinkSettings.subnetId - } - } - dependsOn: [ - storage - ] -} - -resource privateDNSZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2022-01-01' = if (enablePrivateLink) { - parent: privateEndpoint - name: 'dnsgroupname' - properties: { - privateDnsZoneConfigs: [ - { - name: 'dnsConfig' - properties: { - privateDnsZoneId: privateDNSZone.id - } - } - ] - } - dependsOn: [ - privateDNSZone - ] -} - -#disable-next-line BCP081 -resource virtualNetworkLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (enablePrivateLink) { - parent: privateDNSZone - name: 'link_to_vnet' - location: 'global' - properties: { - registrationEnabled: false - virtualNetwork: { - id: privateLinkSettings.vnetId - } - } - dependsOn: [ - privateDNSZone - ] -} - -//////////////// -// Secrets -//////////////// - -@description('Optional: Key Vault Name to store secrets into') -param keyVaultName string = '' - -@description('Optional: To save storage account name into vault set the secret name.') -param storageAccountSecretName string = '' - -@description('Optional: To save storage account key into vault set the secret name.') -param storageAccountKeySecretName string = '' - -@description('Optional: To save storage account table endpoint into vault set the secret name.') -param storageAccountTableEndpointSecretName string = '' - -@description('Optional: To save storage account blob endpoint into vault set the secret name.') -param storageAccountBlobEndpointSecretName string = '' - -@description('Optional: To save storage account connectionstring into vault set the secret name.') -param storageAccountConnectionString string = '' - -@description('Optional: Current Date Time') -param basetime string = utcNow('u') - -@description('Optional: Default SAS TOken Properties to download Blob.') -param sasProperties object = { - signedServices: 'b' - signedPermission: 'rl' - signedExpiry: dateTimeAdd(basetime, 'P1Y') - signedResourceTypes: 'sco' - signedProtocol: 'https' -} - -@description('Optional: To save storage account sas token into vault set the properties.') -param saveToken bool = false - -@description('Optional: Enable as System Storage.') -param isSystem bool = false - -module systemSecretStorageAccountName '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-name' - params: { - keyVaultName: keyVaultName - name: 'system-storage' - value: storage.name - } -} - -module systemSecretStorageAccountKey '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-key' - params: { - keyVaultName: keyVaultName - name: 'system-storage-key' - value: storage.listKeys().keys[0].value - } -} - -module systemSecretStorageAccountEndpoint '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-endpoint' - params: { - keyVaultName: keyVaultName - name: 'system-storage-blob-endpoint' - value: storage.properties.primaryEndpoints.blob - } -} - -module systemSecretStorageAccountConnection '.bicep/keyvault_secrets.bicep' = if (isSystem) { - name: '${deployment().name}-system-secret-connectionstring' - params: { - keyVaultName: keyVaultName - name: 'system-storage-connection' - value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[0].value};EndpointSuffix=${environment().suffixes.storage}' - } -} - -module secretStorageAccountName '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountSecretName)) { - name: '${deployment().name}-secret-name' - params: { - keyVaultName: keyVaultName - name: storageAccountSecretName - value: storage.name - } -} - -module secretStorageAccountKey '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountKeySecretName)) { - name: '${deployment().name}-secret-key' - params: { - keyVaultName: keyVaultName - name: storageAccountKeySecretName - value: storage.listKeys().keys[0].value - } -} - - -module secretStorageAccountTableEndpoint '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountTableEndpointSecretName)) { - name: '${deployment().name}-secret-table-endpoint' - params: { - keyVaultName: keyVaultName - name: storageAccountTableEndpointSecretName - value: storage.properties.primaryEndpoints.table - } -} - -module secretStorageAccountBlobEndpoint '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountBlobEndpointSecretName)) { - name: '${deployment().name}-secret-blob-endpoint' - params: { - keyVaultName: keyVaultName - name: storageAccountBlobEndpointSecretName - value: storage.properties.primaryEndpoints.blob - } -} - -module secretStorageAccountConnection '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && !empty(storageAccountConnectionString)) { - name: '${deployment().name}-secret-connectionstring' - params: { - keyVaultName: keyVaultName - name: storageAccountConnectionString - value: 'DefaultEndpointsProtocol=https;AccountName=${storage.name};AccountKey=${storage.listKeys().keys[0].value};EndpointSuffix=${environment().suffixes.storage}' - } -} - -module secretSASToken '.bicep/keyvault_secrets.bicep' = if (!empty(keyVaultName) && saveToken) { - name: '${deployment().name}-secret-sasToken' - params: { - keyVaultName: keyVaultName - name: '${storage.name}-SAS' - value: listAccountSAS(storage.name, '2022-05-01', sasProperties).accountSasToken - } -} diff --git a/bicep/modules/storage-account-orig/metadata.json b/bicep/modules/storage-account-orig/metadata.json deleted file mode 100644 index e1af3154..00000000 --- a/bicep/modules/storage-account-orig/metadata.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://aka.ms/bicep-registry-module-metadata-file-schema-v2#", - "name": "Azure Storage Module", - "summary": "This module deploys an Azure Storage Account.", - "owner": "azure-global-energy" -} \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/test/main.test.bicep b/bicep/modules/storage-account-orig/test/main.test.bicep deleted file mode 100644 index 804c4e7e..00000000 --- a/bicep/modules/storage-account-orig/test/main.test.bicep +++ /dev/null @@ -1,20 +0,0 @@ -targetScope = 'resourceGroup' - -@minLength(3) -@maxLength(22) -@description('Used to name all resources') -param resourceName string - -@description('Registry Location.') -param location string = resourceGroup().location - - -// Module --> Create Storage Account -module storage '../main.bicep' = { - name: 'storage_account' - params: { - name: 'sa${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' - location: location - sku: 'Standard_LRS' - } -} diff --git a/bicep/modules/storage-account-orig/test/parameters.json b/bicep/modules/storage-account-orig/test/parameters.json deleted file mode 100644 index 80279e80..00000000 --- a/bicep/modules/storage-account-orig/test/parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "template": "../main.bicep" - }, - "parameters": { - "resourceName": { - "value": "osdu" - } - } -} \ No newline at end of file diff --git a/bicep/modules/storage-account-orig/version.json b/bicep/modules/storage-account-orig/version.json deleted file mode 100644 index ba0bb44c..00000000 --- a/bicep/modules/storage-account-orig/version.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", - "version": "v1.0", - "pathFilters": [ - "./main.json", - "./metadata.json" - ] -} \ No newline at end of file diff --git a/charts/airflow-dags/Chart.yaml b/charts/airflow-dags/Chart.yaml new file mode 100644 index 00000000..850ed458 --- /dev/null +++ b/charts/airflow-dags/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: airflow-dags +type: application +description: Installs the OSDU Airflow DAGS +version: 0.0.1 +appVersion: 0.0.1 +maintainers: + - name: danielscholl + url: https://github.com/azure/osdu-developer \ No newline at end of file diff --git a/charts/airflow-dags/README.md b/charts/airflow-dags/README.md new file mode 100644 index 00000000..ca09f69c --- /dev/null +++ b/charts/airflow-dags/README.md @@ -0,0 +1,115 @@ +# Airflow DAGs Helm Chart + +This Helm chart installs and configures OSDU (Open Subsurface Data Universe) DAGs into an existing Airflow installation by ensuring the DAGs are downloaded values parsed, compressed and uploaded to the Storage Share. + +## Overview + +This chart manages two primary types of DAGs: +1. **Manifest DAGs**: Ingestion workflow DAGs for OSDU +2. **CSV Parser DAGs**: DAGs for parsing and processing CSV files in OSDU + +## Prerequisites + +### Infrastructure Requirements +- Kubernetes cluster with Helm installed +- Existing Airflow installation +- PVC named "airflow-dags-pvc" for DAG storage + +### Suggested Custom Values + +The best way to get values is by providing information in the ConfigMap and Secret values. + +#### ConfigMap Values (`airflow-configmap`) +***yaml +value.yaml: | + clientId: "" + tenantId: "" + keyvaultUri: "" +*** + +#### Secret Values (`airflow-secrets`) +***yaml +data: + client-key: "" + insights-key: "" +*** + +## Installation Methods + +### Using FluxCD HelmRelease (Recommended) + +The chart can be deployed using FluxCD's HelmRelease custom resource. Example configuration: + +```yaml +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: airflow-dags + namespace: airflow +spec: + targetNamespace: airflow + releaseName: airflow-dags + dependsOn: + - name: azure-keyvault-airflow + namespace: default + valuesFrom: + - kind: ConfigMap + name: airflow-configmap + valuesKey: value.yaml + - kind: Secret + name: airflow-secrets + valuesKey: client-key + targetPath: secrets.airflowSecrets.clientKey + values: + airflow: + manifestdag: + enabled: true + items: + - name: manifest + folder: "src/osdu_dags" + compress: true + url: "https://example.com/manifest-dags.tar.gz" + pvc: "airflow-dags-pvc" + csvdag: + enabled: true + folder: "airflowdags" + compress: true + url: "https://example.com/csv-parser.tar.gz" + pvc: "airflow-dags-pvc" +``` + +### 2. Manual Installation + +```bash +# Install/Upgrade the chart +helm upgrade --install airflow-dags -f custom_values.yaml . -n airflow +``` + +## Values Reference + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `airflow.manifestdag.enabled` | Enable manifest DAGs | `true` | +| `airflow.csvdag.enabled` | Enable CSV parser DAGs | `true` | +| `airflow.*.folder` | Installation folder | varies | +| `airflow.*.url` | Source URL for DAGs | varies | +| `airflow.*.pvc` | PVC for DAG storage | `airflow-dags-pvc` | + +## Troubleshooting + +For troubleshooting issues: + +1. Check pod logs: + +```bash +kubectl logs -n airflow $(kubectl get pods -n airflow | grep csvdag-upload | awk '{print $1}') +``` + +2. Verify ConfigMap and Secret values: + +```bash +kubectl describe configmap airflow-configmap -n airflow +kubectl describe secret airflow-secrets -n airflow +``` + +For detailed troubleshooting steps and implementation details, see the [IMPLEMENTATION.md](./IMPLEMENTATION.md) file. \ No newline at end of file diff --git a/charts/airflow-dags/scripts/README.md b/charts/airflow-dags/scripts/README.md new file mode 100644 index 00000000..e6970cf6 --- /dev/null +++ b/charts/airflow-dags/scripts/README.md @@ -0,0 +1,91 @@ +# CSV DAG Processing Scripts + +This directory contains scripts for processing and deploying CSV parser DAGs. + +## Manual Testing Instructions + +1. Create a test directory and copy both `csv-dag.sh` and `replace.py` into it: + +```bash +mkdir test-csvdag +cd test-csvdag +cp ../csv-dag.sh ../replace.py . +``` + +2. Create local directories for mounting: +```bash +mkdir scripts share +cp replace.py scripts/ +``` + +3. Export these environment variables by running the following commands: +```bash +export URL="https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz" +export FILE="airflowdags" +export SEARCH_AND_REPLACE='[ + { + "find": "{| K8S_POD_OPERATOR_KWARGS or {} |}", + "replace": { + "labels": { + "aadpodidbinding": "osdu-identity" + }, + "annotations": { + "sidecar.istio.io/inject": "false" + } + } + }, + { + "find": "{| ENV_VARS or {} |}", + "replace": { + "storage_service_endpoint": "http://storage.osdu-core.svc.cluster.local/api/storage/v2", + "schema_service_endpoint": "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1", + "search_service_endpoint": "http://search.osdu-core.svc.cluster.local/api/search/v2", + "partition_service_endpoint": "http://partition.osdu-core.svc.cluster.local/api/partition/v1", + "unit_service_endpoint": "http://unit.osdu-core.svc.cluster.local/api/unit/v2/unit/symbol", + "file_service_endpoint": "http://file.osdu-core.svc.cluster.local/api/file/v2", + "KEYVAULT_URI": "https://dummy-keyvault.vault.azure.net/", + "appinsights_key": "dummy-app-insights-key", + "azure_paas_podidentity_isEnabled": "false", + "AZURE_TENANT_ID": "dummy-tenant-id", + "AZURE_CLIENT_ID": "dummy-client-id", + "AZURE_CLIENT_SECRET": "dummy-client-secret", + "aad_client_id": "dummy-client-id" + } + }, + { + "find": "{| DAG_NAME |}", + "replace": "csv-parser" + }, + { + "find": "{| DOCKER_IMAGE |}", + "replace": "community.opengroup.org:5555/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/csv-parser-v0-27-0-azure-1:60747714ac490be0defe8f3e821497b3cce03390" + }, + { + "find": "{| NAMESPACE |}", + "replace": "airflow" + } +]' +``` + +4. Run the script in a CBL-Mariner container: +```bash +docker run -it --rm \ + -v "$(pwd)/scripts:/scripts" \ + -v "$(pwd)/share:/share" \ + -v "$(pwd)/csv-dag.sh:/csv-dag.sh" \ + -e URL="$URL" \ + -e FILE="$FILE" \ + -e SEARCH_AND_REPLACE="$SEARCH_AND_REPLACE" \ + mcr.microsoft.com/cbl-mariner/base/python:3 \ + /bin/bash /csv-dag.sh +``` + +5. Check the results in the local share directory: +```bash +ls -l share/ +``` + +### Notes +- The script will create a zip file in the mounted `share` directory +- All temporary files are created inside the container and cleaned up automatically +- The container is removed after execution (`--rm` flag) \ No newline at end of file diff --git a/charts/airflow-dags/scripts/csv-dag.sh b/charts/airflow-dags/scripts/csv-dag.sh new file mode 100644 index 00000000..089efced --- /dev/null +++ b/charts/airflow-dags/scripts/csv-dag.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +# Install required packages +tdnf install -y tar curl zip + +# Create and use working directory +WORK_DIR="/tmp/csvdag" +mkdir -p "${WORK_DIR}" +cd "${WORK_DIR}" + +# Download and extract +url_basename=$(basename "${URL}") +echo "Downloading file from ${URL}" +curl --insecure -so "${url_basename}" "${URL}" + +mkdir -p extracted_files +tar -xzf "${url_basename}" --strip-components=1 -C extracted_files + +# Set up file paths for Python script +export INPUT_FILE="extracted_files/${FILE}/csv_ingestion_all_steps.py" +export OUTPUT_FILE="extracted_files/${FILE}/csv-parser.py" + +# Run the Python replacement script +python3 /scripts/replace.py + +# Remove the template file +rm "${INPUT_FILE}" + +# Clean up and zip +rm "${url_basename}" +zip_filename="${url_basename%.tar.gz}.zip" +cd "extracted_files/${FILE}" +zip -r "${WORK_DIR}/${zip_filename}" . + +# Copy to shared volume +cp "${WORK_DIR}/${zip_filename}" /share/ +echo "Zip file ${zip_filename} copied to shared volume." \ No newline at end of file diff --git a/charts/airflow-dags/scripts/replace.py b/charts/airflow-dags/scripts/replace.py new file mode 100644 index 00000000..946d42e2 --- /dev/null +++ b/charts/airflow-dags/scripts/replace.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import json +import sys +import os +from typing import Dict, Any + +def process_replacements(content: str, replacements: Dict[str, Any]) -> str: + """Process the file content with the given replacements.""" + result = [] + for line in content.splitlines(): + modified_line = line + for item in replacements: + find = item['find'] + replace = item['replace'] + + if find in line: + if isinstance(replace, (dict, list)): + if '=' in line: + var_name, _ = line.split('=', 1) + + # Convert dictionary values + processed_dict = {} + for k, v in replace.items(): + # Handle template variables + if isinstance(v, str) and '{{' in v: + processed_dict[k] = v # Keep template syntax intact + # Handle boolean values + elif isinstance(v, str) and v.lower() in ['true', 'false']: + processed_dict[k] = v.lower() + else: + processed_dict[k] = str(v) + + # Format as Python code + replace_str = json.dumps(processed_dict, indent=2) + modified_line = f"{var_name.rstrip()} = {replace_str}" + else: + modified_line = line.replace(find, json.dumps(replace)) + else: + modified_line = line.replace(find, str(replace)) + break + if modified_line.strip(): + result.append(modified_line) + return '\n'.join(result) + +def main(): + input_file = os.environ['INPUT_FILE'] + output_file = os.environ['OUTPUT_FILE'] + raw_json = os.environ['SEARCH_AND_REPLACE'] + print("Raw JSON:", raw_json) + replacements = json.loads(raw_json) + print("Parsed replacements:", replacements) + + with open(input_file, 'r') as f: + content = f.read() + + processed_content = process_replacements(content, replacements) + + with open(output_file, 'w') as f: + f.write(processed_content) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/charts/airflow-dags/templates/_helpers.tpl b/charts/airflow-dags/templates/_helpers.tpl new file mode 100644 index 00000000..eb7c32d5 --- /dev/null +++ b/charts/airflow-dags/templates/_helpers.tpl @@ -0,0 +1,101 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "airflow-dags.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airflow-dags.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airflow-dags.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airflow-dags.labels" -}} +helm.sh/chart: {{ include "airflow-dags.chart" . }} +{{ include "airflow-dags.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airflow-dags.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airflow-dags.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Search and Replace Configuration +*/}} +{{- define "airflow-dags.searchAndReplace" -}} +[ + { + "find": "{| K8S_POD_OPERATOR_KWARGS or {} |}", + "replace": { + "annotations": { + "sidecar.istio.io/inject": "false" + }, + "labels": { + "aadpodidbinding": "osdu-identity" + } + } + }, + { + "find": "{| ENV_VARS or {} |}", + "replace": { + "AZURE_CLIENT_ID": {{ .Values.clientId | default "" | quote }}, + "AZURE_CLIENT_SECRET": {{ .Values.secrets.airflowSecrets.clientKey | default "" | quote }}, + "AZURE_TENANT_ID": {{ .Values.tenantId | default "" | quote }}, + "KEYVAULT_URI": {{ .Values.keyvaultUri | default "" | quote }}, + "aad_client_id": {{ .Values.clientId | default "" | quote }}, + "appinsights_key": {{ .Values.secrets.airflowSecrets.insightsKey | default "" | quote }}, + "azure_paas_podidentity_isEnabled": "false", + "file_service_endpoint": "http://file.osdu-core.svc.cluster.local/api/file/v2", + "partition_service_endpoint": "http://partition.osdu-core.svc.cluster.local/api/partition/v1", + "schema_service_endpoint": "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1", + "search_service_endpoint": "http://search.osdu-core.svc.cluster.local/api/search/v2", + "storage_service_endpoint": "http://storage.osdu-core.svc.cluster.local/api/storage/v2", + "unit_service_endpoint": "http://unit.osdu-core.svc.cluster.local/api/unit/v2/unit/symbol" + } + }, + { + "find": "{| DAG_NAME |}", + "replace": "csv-parser" + }, + { + "find": "{| DOCKER_IMAGE |}", + "replace": "community.opengroup.org:5555/osdu/platform/data-flow/ingestion/csv-parser/csv-parser-v0-27-0-azure-1:60747714ac490be0defe8f3e821497b3cce03390" + }, + { + "find": "{| NAMESPACE |}", + "replace": "airflow" + } +] +{{- end }} \ No newline at end of file diff --git a/charts/airflow-dags/templates/dag-csv-job.yaml b/charts/airflow-dags/templates/dag-csv-job.yaml new file mode 100644 index 00000000..5ea53b4e --- /dev/null +++ b/charts/airflow-dags/templates/dag-csv-job.yaml @@ -0,0 +1,50 @@ +{{- if .Values.airflow.csvdag.enabled -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-csvdag-upload + namespace: {{ .Release.Namespace }} +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: workload-identity-sa + volumes: + - name: scripts + configMap: + name: csvdag-scripts-{{ .Release.Name }} + defaultMode: 0500 + - name: share-storage + persistentVolumeClaim: + claimName: {{ .Values.airflow.csvdag.pvc }} + containers: + - name: csvdag-upload + image: mcr.microsoft.com/cbl-mariner/base/python:3.9-cm2.0 + command: ["/bin/bash"] + args: + - /scripts/csv-dag.sh + env: + - name: URL + value: {{ .Values.airflow.csvdag.url | quote }} + - name: FILE + value: {{ .Values.airflow.csvdag.folder | quote }} + - name: SEARCH_AND_REPLACE + value: {{ include "airflow-dags.searchAndReplace" . | quote }} + volumeMounts: + - name: scripts + mountPath: /scripts + - name: share-storage + mountPath: /share + restartPolicy: Never +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: csvdag-scripts-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +data: + replace.py: | + {{ .Files.Get "scripts/replace.py" | nindent 4 }} + csv-dag.sh: | + {{ .Files.Get "scripts/csv-dag.sh" | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/charts/airflow-dags/templates/dag-manifest-job.yaml b/charts/airflow-dags/templates/dag-manifest-job.yaml new file mode 100644 index 00000000..d6cdb93c --- /dev/null +++ b/charts/airflow-dags/templates/dag-manifest-job.yaml @@ -0,0 +1,108 @@ +{{- if .Values.airflow.manifestdag.enabled -}} +{{- range $index, $item := .Values.airflow.manifestdag.items }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $.Release.Name }}-file-upload-{{ $index }} + namespace: {{ $.Release.Namespace }} +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: workload-identity-sa + volumes: + - name: script + configMap: + name: file-upload-script-{{ $.Release.Name }} + defaultMode: 0500 + - name: share-storage + persistentVolumeClaim: + claimName: {{ $item.pvc }} + initContainers: + - name: file-upload + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + command: ["/bin/sh"] + args: + - -c + - | + tdnf install -y curl tar zip && \ + /script/init.sh + volumeMounts: + - name: script + mountPath: "/script" + - name: share-storage + mountPath: "/share" + env: + - name: URL + value: {{ $item.url | quote }} + - name: SHARE + value: {{ $item.name | quote }} + - name: COMPRESS + value: {{ $item.compress | default false | quote }} + - name: FILE + value: {{ $item.folder | quote }} + containers: + - name: completion + image: istio/base + command: ["/bin/sleep", "30"] + volumeMounts: + - name: script + mountPath: "/script" + restartPolicy: Never +--- +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-upload-script-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +data: + init.sh: | + #!/bin/bash + set -e + + url_basename=$(basename ${URL}) + echo "Derived filename from URL: ${url_basename}" + + echo "Downloading file from ${URL} to ${url_basename}" + retry_count=0 + max_retries=3 + while [ $retry_count -lt $max_retries ]; do + if curl -kso ${url_basename} ${URL}; then + break + else + retry_count=$((retry_count + 1)) + echo "Attempt $retry_count failed. Retrying in 5 seconds..." >&2 + sleep 5 + fi + done + + if [ $retry_count -eq $max_retries ]; then + echo "Error: Failed to download file from ${URL} after $max_retries attempts." >&2 + exit 1 + fi + + if [[ ${URL} == *.tar.gz ]]; then + echo "Processing tar.gz archive..." + mkdir -p extracted_files + tar -xzf ${url_basename} --strip-components=1 -C extracted_files + + if [[ ${COMPRESS} == "true" ]]; then + echo "Creating zip archive..." + rm ${url_basename} + zip_filename="${url_basename%.tar.gz}.zip" + original_dir=$(pwd) + cd extracted_files/${FILE} + zip -r ${original_dir}/${zip_filename} * + cd ${original_dir} + cp ${zip_filename} /share/ + echo "Zip file ${zip_filename} copied to share." + else + echo "Copying extracted files..." + cp -r extracted_files/${FILE} /share/ + fi + else + echo "Copying single file..." + cp ${FILE} /share/ + fi +{{- end }} \ No newline at end of file diff --git a/charts/airflow-dags/values.yaml b/charts/airflow-dags/values.yaml new file mode 100644 index 00000000..6ba38197 --- /dev/null +++ b/charts/airflow-dags/values.yaml @@ -0,0 +1,12 @@ + +# Default values for osdu-azure. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +fullnameOverride: airflow-dags + +airflow: + manifestdag: + enabled: false + csvdag: + enabled: false diff --git a/charts/blob-upload/Chart.yaml b/charts/blob-upload/Chart.yaml new file mode 100644 index 00000000..76c69e0c --- /dev/null +++ b/charts/blob-upload/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: blob-upload +type: application +description: Uploads files to Azure Blob Storage +version: 0.0.2 +appVersion: 0.0.1 +maintainers: + - name: danielscholl + url: https://github.com/azure/osdu-developer \ No newline at end of file diff --git a/charts/blob-upload/templates/_helpers.tpl b/charts/blob-upload/templates/_helpers.tpl new file mode 100644 index 00000000..7bc1d801 --- /dev/null +++ b/charts/blob-upload/templates/_helpers.tpl @@ -0,0 +1,52 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "blob-upload.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "blob-upload.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "blob-upload.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "blob-upload.labels" -}} +helm.sh/chart: {{ include "blob-upload.chart" . }} +{{ include "blob-upload.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "blob-upload.selectorLabels" -}} +app.kubernetes.io/name: {{ include "blob-upload.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml new file mode 100644 index 00000000..c2f0e658 --- /dev/null +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -0,0 +1,49 @@ +{{- if (default false .Values.blobUpload.enabled) }} +{{- $i := 0 }} +{{- range $key, $value := (lookup "v1" "ConfigMap" .Values.global.configmapNamespace "configmap-services").data }} +{{- if hasPrefix "partition_storage_name_" $key }} +{{- range $.Values.blobUpload.items }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $.Release.Name }}-blob-upload-{{ .name }}-{{ $i }} + namespace: {{ $.Release.Namespace }} +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: workload-identity-sa + containers: + - name: blob-upload + image: mcr.microsoft.com/azure-cli:cbl-mariner2.0-amd64 + command: ["/bin/bash"] + args: + - -c + - | + # Install curl + tdnf install -y curl + + # Download the file + echo "Downloading file from {{ .url }}" + curl -kso {{ .file }} "{{ .url }}" + + # Login using workload identity + az login --identity + + # Upload directly to blob storage using Azure CLI + az storage blob upload \ + -f {{ .file }} \ + -c {{ $.Values.blobUpload.container }} \ + -n {{ .file }} \ + --overwrite \ + --auth-mode login + + echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $value }}" + sleep 300000 + restartPolicy: Never +{{- end }} +{{- $i = add $i 1 }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/blob-upload/values.yaml b/charts/blob-upload/values.yaml new file mode 100644 index 00000000..b0764fb5 --- /dev/null +++ b/charts/blob-upload/values.yaml @@ -0,0 +1,8 @@ +fullnameOverride: blob-upload + +global: + configmapNamespace: osdu-core + +blobUpload: + enabled: true + container: "legal-service-azure-configuration" diff --git a/charts/config-maps/Chart.yaml b/charts/config-maps/Chart.yaml index 3b5dc0d7..b4bae630 100644 --- a/charts/config-maps/Chart.yaml +++ b/charts/config-maps/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.6 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/config-maps/templates/config-map-software.yaml b/charts/config-maps/templates/config-map-airflow.yaml similarity index 78% rename from charts/config-maps/templates/config-map-software.yaml rename to charts/config-maps/templates/config-map-airflow.yaml index a0df4cc5..02e89b7a 100644 --- a/charts/config-maps/templates/config-map-software.yaml +++ b/charts/config-maps/templates/config-map-airflow.yaml @@ -1,4 +1,5 @@ {{- $namespace := .Release.Namespace }} +{{- if .Values.configMaps.airflow }} apiVersion: azconfig.io/v1 kind: AzureAppConfigurationProvider metadata: @@ -7,7 +8,7 @@ metadata: spec: endpoint: {{ .Values.azure.configEndpoint }} target: - configMapName: configmap-software + configMapName: airflow-configmap configMapData: type: yaml key: value.yaml @@ -18,11 +19,12 @@ spec: configuration: selectors: - keyFilter: "*" - labelFilter: "configmap-osdu-applications" + labelFilter: "configmap-airflow-values" refresh: enabled: true interval: 1m monitoring: keyValues: - key: "osdu_sentinel" - label: "common" \ No newline at end of file + label: "common" +{{- end }} \ No newline at end of file diff --git a/charts/config-maps/templates/service-account.yaml b/charts/config-maps/templates/service-account.yaml index 346d059a..b5b864bf 100644 --- a/charts/config-maps/templates/service-account.yaml +++ b/charts/config-maps/templates/service-account.yaml @@ -1,3 +1,4 @@ +{{- if .Values.serviceAccount }} apiVersion: v1 kind: ServiceAccount metadata: @@ -6,4 +7,5 @@ metadata: azure.workload.identity/client-id: {{ .Values.azure.clientId }} azure.workload.identity/tenant-id: {{ .Values.azure.tenantId }} labels: - azure.workload.identity/use: "true" \ No newline at end of file + azure.workload.identity/use: "true" +{{- end }} \ No newline at end of file diff --git a/charts/config-maps/values.yaml b/charts/config-maps/values.yaml index fed16ab7..9eaf5a85 100644 --- a/charts/config-maps/values.yaml +++ b/charts/config-maps/values.yaml @@ -5,3 +5,8 @@ azure: configEndpoint: https://.azconfig.io clientId: keyvaultUri: https://.vault.azure.net/ + +serviceAccount: false + +configMaps: + airflow: true \ No newline at end of file diff --git a/charts/osdu-developer-base/templates/share-job.yaml b/charts/osdu-developer-base/templates/storage-share-job.yaml similarity index 100% rename from charts/osdu-developer-base/templates/share-job.yaml rename to charts/osdu-developer-base/templates/storage-share-job.yaml diff --git a/charts/osdu-developer-base/values.yaml b/charts/osdu-developer-base/values.yaml index c085a581..82db7a6b 100644 --- a/charts/osdu-developer-base/values.yaml +++ b/charts/osdu-developer-base/values.yaml @@ -1,20 +1,3 @@ -# Copyright © Microsoft Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for osdu-azure. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. fullnameOverride: osdu-svc @@ -36,6 +19,10 @@ resourceLimits: defaultCpuLimits: "1" defaultMemoryLimits: "4Gi" +blobUpload: + enabled: false + container: "legal-service-azure-configuration" + share: enabled: false items: @@ -45,4 +32,5 @@ share: - name: item2 url: "https://example.com/archive.tar.gz" file: "folder_in_archive" - compress: true \ No newline at end of file + compress: true + diff --git a/docs/src/feature_flags.md b/docs/src/feature_flags.md index 365add3c..b9afe571 100644 --- a/docs/src/feature_flags.md +++ b/docs/src/feature_flags.md @@ -50,7 +50,6 @@ Infrastructure customizations can be modified using the following feature flags. |---------------------------|-----------------------------------------------------------------------------| | CLUSTER_INGRESS | Specifies the Ingress type for the cluster (External, Internal, or Both) | | CLUSTER_VM_SIZE | Overrides the default server type with a custom VM size | -| ENABLE_BLOB_PUBLIC_ACCESS | Enables public access for storage account blob (False by default) | | ENABLE_NODE_AUTO_PROVISIONING | Enables node auto provisioning (True by default) | | ENABLE_PRIVATE_CLUSTER | Enables private cluster (False by default) | @@ -62,6 +61,7 @@ Software customizations can be modified using the following feature flags. | Feature Flag | Description | |---------------------------|-----------------------------------------------------------------------------| | ENABLE_SOFTWARE | Disables loading of all software when set to false (True by default) | +| ENABLE_PRIVATE_SOFTWARE | Software from Azure Blob instead of Git Repositories (False by default) | | ENABLE_OSDU_CORE | Disables loading of the osdu core services (True by default) | | ENABLE_OSDU_REFERENCE | Disables loading of the osdu reference services (True by default) | | SOFTWARE_VERSION | Sets the version (branch) of OSDU to be installed (release-0-27) | diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 76b91eb3..0741db96 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -31,4 +31,41 @@ spec: defaultCpuRequests: "0.5" defaultMemoryRequests: "1Gi" defaultCpuLimits: "2" - defaultMemoryLimits: "4Gi" \ No newline at end of file + defaultMemoryLimits: "4Gi" +--- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: blob-upload +# namespace: default +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-developer-base-core +# namespace: default +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/blob-upload +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: config-map-values +# valuesKey: values.yaml +# values: +# global: +# configmapNamespace: osdu-core +# blobUpload: +# enabled: true +# items: +# - name: legal +# file: "Legal_COO.json" +# url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" diff --git a/software/components/airflow/config-maps.yaml b/software/components/airflow/config-maps.yaml new file mode 100644 index 00000000..2baabddb --- /dev/null +++ b/software/components/airflow/config-maps.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: config-maps-airflow + namespace: default + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + targetNamespace: airflow + chart: + spec: + chart: ./charts/config-maps + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: config-map-values + valuesKey: values.yaml + values: + serviceAccount: false + configMaps: + airflow: true diff --git a/software/components/airflow/dag-jobs.yaml b/software/components/airflow/dag-jobs.yaml new file mode 100644 index 00000000..003122cf --- /dev/null +++ b/software/components/airflow/dag-jobs.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: airflow-dags + namespace: airflow + annotations: + clusterconfig.azure.com/use-managed-source: "true" + kustomize.toolkit.fluxcd.io/substitute: disabled +spec: + targetNamespace: airflow + releaseName: airflow-dags + dependsOn: + - name: azure-keyvault-airflow + namespace: default + - name: config-maps-airflow + namespace: default + chart: + spec: + chart: ./charts/airflow-dags + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: airflow-configmap + valuesKey: value.yaml + - kind: Secret + name: airflow-secrets + valuesKey: client-key + targetPath: secrets.airflowSecrets.clientKey + - kind: Secret + name: airflow-secrets + valuesKey: insights-key + targetPath: secrets.airflowSecrets.insightsKey + values: + airflow: + manifestdag: + enabled: true + items: + - name: manifest + folder: "src/osdu_dags" + compress: true + url: "https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/archive/master/ingestion-dags-master.tar.gz" + pvc: "airflow-dags-pvc" + csvdag: + enabled: true + folder: "airflowdags" + compress: true + url: "https://community.opengroup.org/osdu/platform/data-flow/ingestion/csv-parser/csv-parser/-/archive/master/csv-parser-master.tar.gz" + pvc: "airflow-dags-pvc" \ No newline at end of file diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml deleted file mode 100644 index 53a5a4e7..00000000 --- a/software/components/airflow/pvc.yaml +++ /dev/null @@ -1,82 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: airflow-logs-pv - namespace: airflow - labels: - usage: airflow-logs-pv -spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteMany - azureFile: - secretName: airflow-secrets - shareName: airflow-logs - volumeMode: Filesystem - mountOptions: - - dir_mode=0777 - - file_mode=0777 - - uid=50000 - - gid=0 - - mfsymlinks - - nobrl ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: airflow-logs-pvc - namespace: airflow - annotations: - volume.beta.kubernetes.io/storage-class: "" -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - selector: - matchLabels: - usage: airflow-logs-pv ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: airflow-dags-pv - namespace: airflow - labels: - usage: airflow-dags-pv -spec: - capacity: - storage: 5Gi - accessModes: - - ReadWriteMany - azureFile: - secretName: airflow-secrets - shareName: airflow-dags - volumeMode: Filesystem - mountOptions: - - dir_mode=0777 - - file_mode=0777 - - uid=1000 - - gid=1000 - - mfsymlinks - - nobrl ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: airflow-dags-pvc - namespace: airflow - annotations: - volume.beta.kubernetes.io/storage-class: "" -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - selector: - matchLabels: - usage: airflow-dags-pv \ No newline at end of file diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 105daef7..9c9c862d 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -10,8 +10,8 @@ spec: targetNamespace: airflow releaseName: airflow dependsOn: - - name: azure-keyvault-airflow - namespace: default + - name: airflow-dags + namespace: airflow install: remediation: retries: 3 @@ -38,6 +38,11 @@ spec: repository: apache/airflow tag: 2.10.1-python3.12 executor: KubernetesExecutor + # serviceAccount: + # create: false + # name: workload-identity-sa + # annotations: + # azure.workload.identity/use: "true" securityContext: fsGroup: 1000 runAsUser: 1000 diff --git a/software/components/airflow/storage.yaml b/software/components/airflow/storage.yaml new file mode 100644 index 00000000..a7976f26 --- /dev/null +++ b/software/components/airflow/storage.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: airflow-storage +provisioner: file.csi.azure.com +parameters: + skuName: Standard_LRS +allowVolumeExpansion: true +mountOptions: + - dir_mode=0777 + - file_mode=0777 + - uid=1000 + - gid=1000 + - mfsymlinks + - nobrl +reclaimPolicy: Delete +volumeBindingMode: Immediate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: airflow-logs-pvc + namespace: airflow + annotations: + csi.storage.k8s.io/share-name: "airflow-logs" +spec: + accessModes: + - ReadWriteMany + storageClassName: airflow-storage + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: airflow-dags-pvc + namespace: airflow + annotations: + csi.storage.k8s.io/share-name: "airflow-dags" +spec: + accessModes: + - ReadWriteMany + storageClassName: airflow-storage + resources: + requests: + storage: 5Gi \ No newline at end of file diff --git a/software/components/airflow/vault-secrets.yaml b/software/components/airflow/vault-secrets.yaml index 2e64524c..c405183b 100644 --- a/software/components/airflow/vault-secrets.yaml +++ b/software/components/airflow/vault-secrets.yaml @@ -32,10 +32,6 @@ spec: vaultSecret: airflow-fernet-key - key: webserver-key vaultSecret: airflow-webserver-key - - key: azurestorageaccountname - vaultSecret: system-storage - - key: azurestorageaccountkey - vaultSecret: system-storage-key - key: password vaultSecret: airflow-admin-password - key: connection @@ -50,6 +46,8 @@ spec: vaultSecret: app-dev-sp-id - key: client-key vaultSecret: app-dev-sp-password + - key: insights-key + vaultSecret: insights-key --- apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease diff --git a/tools/rest-scripts/partition.http b/tools/rest-scripts/partition.http index 9165da9b..bfe08101 100644 --- a/tools/rest-scripts/partition.http +++ b/tools/rest-scripts/partition.http @@ -203,12 +203,10 @@ GET {{PARTITION_HOST}}/partitions Authorization: Bearer {{access_token}} Accept: application/json -@partition = {{getPartitions.response.body.$[0]}} - ### # @name getPartition -GET {{PARTITION_HOST}}/partitions/{{partition}} +GET {{PARTITION_HOST}}/partitions/{{getPartitions.response.body.$[0]}} Authorization: Bearer {{access_token}} Content-Type: application/json