From 44c9e262125aed0459c1894fe848164606746c49 Mon Sep 17 00:00:00 2001 From: "Choi, James [USA]" Date: Mon, 25 Nov 2024 16:34:21 -0500 Subject: [PATCH] 484 Feature: integrate Universal Config store in the hive metastore service --- DRAFT_RELEASE_NOTES.md | 1 + .../README.md | 23 ++-- .../templates/configmap.yaml | 27 +++-- .../tests/configmap_test.yaml | 55 +++++++++ .../values.yaml | 8 ++ .../ModelAgnosticResourcesGenerator.java | 44 +++---- .../src/main/resources/profiles.json | 6 + .../src/main/resources/targets.json | 16 +++ .../v2/configuration-store.values-dev.yaml.vm | 2 - .../hive-metastore-service/hive.properties.vm | 4 + .../v2/spark.infrastructure.values.yaml.vm | 7 -- .../aiops/mda/generator/PropertiesStep.java | 76 ++++++++++++ .../properties-generation.feature | 16 +++ foundation/foundation-upgrade/pom.xml | 4 + .../HiveUniversalConfigYAMLMigration.java | 110 ++++++++++++++++++ .../src/main/resources/migrations.json | 11 ++ .../HiveUniversalConfigYAMLMigrationTest.java | 51 ++++++++ .../v1_11_0/hive-uc-migration.feature | 16 +++ .../migration/custom-values.yaml | 89 ++++++++++++++ .../migration/default-values.yaml | 89 ++++++++++++++ .../migration/updated-values.yaml | 83 +++++++++++++ 21 files changed, 690 insertions(+), 48 deletions(-) create mode 100644 foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive.properties.vm create mode 100644 foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/PropertiesStep.java create mode 100644 foundation/foundation-mda/src/test/resources/specifications/properties-generation.feature create mode 100644 foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigration.java create mode 100644 foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigrationTest.java create mode 100644 foundation/foundation-upgrade/src/test/resources/specifications/v1_11_0/hive-uc-migration.feature create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/custom-values.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/default-values.yaml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/updated-values.yaml diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md index 56afaa3fe..d05f624c8 100644 --- a/DRAFT_RELEASE_NOTES.md +++ b/DRAFT_RELEASE_NOTES.md @@ -33,6 +33,7 @@ To reduce burden of upgrading aiSSEMBLE, the Baton project is used to automate t | upgrade-tiltfile-aissemble-version-migration | Updates the aiSSEMBLE version within your project's Tiltfile | | upgrade-v2-chart-files-aissemble-version-migration | Updates the Helm chart dependencies within your project's deployment resources (`-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE | | upgrade-v1-chart-files-aissemble-version-migration | Updates the docker image tags within your project's deployment resources (`-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE | +| hive-universal-config-yaml-migration | Updates hive-metastore-service values.yaml to removes hive username to use values from configuration store service only if hive username has default value. | To deactivate any of these migrations, add the following configuration to the `baton-maven-plugin` within your root `pom.xml`: diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md index 85f3ba77a..69d275e59 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md @@ -48,6 +48,7 @@ configuration options. |------------------|---------------------| | fullnameOverride | "hive-metastore-db" | | auth.database | "metastore" | +| auth.username | "hive" | # Migration from aiSSEMBLE v1 Helm Charts @@ -66,17 +67,17 @@ In the table below, the notation `env[KEY]` refers the `env` list item whose `na **Note**: *all new property locations include the prefix `aissemble-hive-metastore-service-chart`* -| Old Property Location | New Property Location | Additional Notes | -|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|------------------| -| hive-metastore-db.service.spec.ports[].port | mysql.primary.service.ports.mysql | Default to 3306 | -| deployment.env[HADOOP_CLASSPATH] | deployment.baseEnv[HADOOP_CLASSPATH] | | -| deployment.env[JAVA_HOME] | deployment.baseEnv[JAVA_HOME] | | -| configMap.metastoreServiceConfig.configuration.property[metastore.thrift.uris] | configMap.metastoreServiceConfig.baseProperties[metastore.thrift.uris] | | -| configMap.metastoreServiceConfig.configuration.property[metastore.task.threads.always] | configMap.metastoreServiceConfig.baseProperties[metastore.task.threads.always] | | -| configMap.metastoreServiceConfig.configuration.property[metastore.expression.proxy] | configMap.metastoreServiceConfig.baseProperties[metastore.expression.proxy] | | -| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionDriverName] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionDriverName] | | -| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionURL] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionURL] | | - +| Old Property Location | New Property Location | Additional Notes | +|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|--------------------------------------------------------| +| hive-metastore-db.service.spec.ports[].port | mysql.primary.service.ports.mysql | Default to 3306 | +| deployment.env[HADOOP_CLASSPATH] | deployment.baseEnv[HADOOP_CLASSPATH] | | +| deployment.env[JAVA_HOME] | deployment.baseEnv[JAVA_HOME] | | +| configMap.metastoreServiceConfig.configuration.property[metastore.thrift.uris] | configMap.metastoreServiceConfig.baseProperties[metastore.thrift.uris] | | +| configMap.metastoreServiceConfig.configuration.property[metastore.task.threads.always] | configMap.metastoreServiceConfig.baseProperties[metastore.task.threads.always] | | +| configMap.metastoreServiceConfig.configuration.property[metastore.expression.proxy] | configMap.metastoreServiceConfig.baseProperties[metastore.expression.proxy] | | +| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionDriverName] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionDriverName] | | +| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionURL] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionURL] | | +| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionUserName] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionUserName] | Using Configuration store Service to inject this value | ## Property Removed The following properties no longer exist. diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml index 8c9cfae60..60329020b 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml @@ -2,20 +2,34 @@ apiVersion: v1 kind: ConfigMap metadata: name: metastore-service-config + labels: + aissemble-configuration-store: enabled data: # Add all the default properties from the local values.yaml to the ConfigMap # Then check if there are any downstream properties and add them as well metastore-site.xml: | + {{- $basePropDict := dict }} + {{- $propDict := dict }} + {{- range $baseProperty := .Values.configMap.metastoreServiceConfig.baseProperties }} + {{- $basePropDict := set $basePropDict $baseProperty.name $baseProperty.value }} + {{- end }} + {{- if .Values.configMap.metastoreServiceConfig.properties }} + {{- range $property := .Values.configMap.metastoreServiceConfig.properties }} + {{- $propDict := set $propDict $property.name $property.value }} + {{- end }} + {{- end }} + - {{- range $property := .Values.configMap.metastoreServiceConfig.baseProperties }} + {{- range $baseProperty := .Values.configMap.metastoreServiceConfig.baseProperties }} + {{- if and (hasKey $basePropDict $baseProperty.name) (not (hasKey $propDict $baseProperty.name))}} - {{ $property.name }} - {{ $property.value }} - - {{- if $property.description }} - {{ $property.description }} + {{ $baseProperty.name }} + {{ $baseProperty.value }} + {{- if $baseProperty.description }} + {{ $baseProperty.description }} {{- end }} + {{- end }} {{- end }} {{- if .Values.configMap.metastoreServiceConfig.properties }} @@ -23,7 +37,6 @@ data: {{ $property.name }} {{ $property.value }} - {{- if $property.description }} {{ $property.description }} {{- end }} diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml index 1688d5d3f..926abc621 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml @@ -33,6 +33,11 @@ tests: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false JDBC connect string for a JDBC metastore + + javax.jdo.option.ConnectionUserName + $getConfigValue(groupName=hive;propertyName=HIVE_USERNAME) + Username to use against metastore database + - it: Should override default properties appropriately set: @@ -90,6 +95,11 @@ tests: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false JDBC connect string for a JDBC metastore + + javax.jdo.option.ConnectionUserName + $getConfigValue(groupName=hive;propertyName=HIVE_USERNAME) + Username to use against metastore database + propertyName1 value1 @@ -100,3 +110,48 @@ tests: value2 + - it: Should override baseProperties with properties appropriately + set: + configMap: + metastoreServiceConfig: + properties: + - name: metastore.thrift.uris + value: thrift://0.0.0.0:8081 + description: Overridden metastore.thrift.uris + - name: metastore.task.threads.always + value: overridden metastore.task.threads.always + asserts: + - equal: + path: data + value: + metastore-site.xml: | + + + metastore.expression.proxy + org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy + + + javax.jdo.option.ConnectionDriverName + com.mysql.cj.jdbc.Driver + + + javax.jdo.option.ConnectionURL + jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false + JDBC connect string for a JDBC metastore + + + javax.jdo.option.ConnectionUserName + $getConfigValue(groupName=hive;propertyName=HIVE_USERNAME) + Username to use against metastore database + + + metastore.thrift.uris + thrift://0.0.0.0:8081 + Overridden metastore.thrift.uris + + + metastore.task.threads.always + overridden metastore.task.threads.always + + + diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml index 8a207519d..8a24969cf 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml @@ -15,11 +15,16 @@ image: dockerRepo: "ghcr.io/" mysql: + commonLabels: + aissemble-configuration-store: enabled enabled: true fullnameOverride: "hive-metastore-db" auth: # The schematool for the metastore service will create the database for us. database: "metastore" + # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the + # persisted database. + username: $getConfigValue(groupName=hive;propertyName=HIVE_USERNAME) hive: dbType: "mysql" @@ -72,4 +77,7 @@ configMap: - name: javax.jdo.option.ConnectionURL value: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false description: JDBC connect string for a JDBC metastore + - name: javax.jdo.option.ConnectionUserName + value: $getConfigValue(groupName=hive;propertyName=HIVE_USERNAME) + description: Username to use against metastore database properties: {} \ No newline at end of file diff --git a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java index e40e2e356..8b3a9981b 100644 --- a/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java +++ b/foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java @@ -22,27 +22,29 @@ public class ModelAgnosticResourcesGenerator extends AbstractModelAgnosticGenerator { /*--~-~-~~ * Usages: - * | Target | Template | Generated File | - * |----------------------------------|----------------------------------------------------|-------------------------------------------------| - * | cdiBeansXml | beans.xml.vm | META-INF/beans.xml | - * | cucumberPipelineFeature | cucumber.pipeline.feature.vm | specifications/pipeline.feature | - * | cucumberProperties | cucumber.properties.vm | cucumber.properties | - * | dataAccessApplicationProperties | data-access/data.access.application.properties.vm | application.properties | - * | mlflowStartScript | general-docker/mlflow.start.sh.vm | start.sh | - * | sparkDataDeliveryProperties | general-docker/spark.data.delivery.properties.vm | krausening/base/spark-data-delivery.properties | - * | authConfigResource | general-mlflow/auth.properties.vm | krausening/base/auth.properties | - * | inferenceConfigResource | general-mlflow/inference.config.properties.vm | krausening/base/inference.properties | - * | trainingPipelineConfigResource | general-mlflow/training.config.properties.vm | krausening/base/pipeline.properties | - * | itChartYaml | integration-test/it.chart.yaml.vm | test-chart/Chart.yaml | - * | itPipelineSpecification | integration-test/it.pipeline.spec.vm | specifications/pipeline.feature | - * | itServiceAccountYaml | integration-test/it.serviceaccount.yaml.vm | test-chart/templates/serviceaccount.yaml | - * | itTestYaml | integration-test/it.test.yaml.vm | test-chart/templates/test.yaml | - * | itTiltfile | integration-test/it.tiltfile.vm | Tiltfile | - * | itValuesCIYaml | integration-test/it.values.ci.yaml.vm | test-chart/values-ci.yaml | - * | itValuesPipelineYaml | integration-test/it.values.pipeline.yaml.vm | test-chart/values-pipeline.yaml | - * | itValuesYaml | integration-test/it.values.yaml.vm | test-chart/values.yaml | - * | testLog4jConfiguration | log4j2.xml.vm | log4j2.xml | - * | globalDeploymentConfigFile | pipeline-models/deployment-config.json.vm | deployment-config.json | + * | Target | Template | Generated File | + * |----------------------------------|-------------------------------------------------------|-------------------------------------------------| + * | cdiBeansXml | beans.xml.vm | META-INF/beans.xml | + * | cucumberPipelineFeature | cucumber.pipeline.feature.vm | specifications/pipeline.feature | + * | cucumberProperties | cucumber.properties.vm | cucumber.properties | + * | dataAccessApplicationProperties | data-access/data.access.application.properties.vm | application.properties | + * | baseHiveProperties | deployment/hive-metastore-service/hive.properties.vm | configurations/base/hive.properties | + * | envHiveProperties | deployment/hive-metastore-service/hive.properties.vm | configurations/env/hive.properties | + * | mlflowStartScript | general-docker/mlflow.start.sh.vm | start.sh | + * | sparkDataDeliveryProperties | general-docker/spark.data.delivery.properties.vm | krausening/base/spark-data-delivery.properties | + * | authConfigResource | general-mlflow/auth.properties.vm | krausening/base/auth.properties | + * | inferenceConfigResource | general-mlflow/inference.config.properties.vm | krausening/base/inference.properties | + * | trainingPipelineConfigResource | general-mlflow/training.config.properties.vm | krausening/base/pipeline.properties | + * | itChartYaml | integration-test/it.chart.yaml.vm | test-chart/Chart.yaml | + * | itPipelineSpecification | integration-test/it.pipeline.spec.vm | specifications/pipeline.feature | + * | itServiceAccountYaml | integration-test/it.serviceaccount.yaml.vm | test-chart/templates/serviceaccount.yaml | + * | itTestYaml | integration-test/it.test.yaml.vm | test-chart/templates/test.yaml | + * | itTiltfile | integration-test/it.tiltfile.vm | Tiltfile | + * | itValuesCIYaml | integration-test/it.values.ci.yaml.vm | test-chart/values-ci.yaml | + * | itValuesPipelineYaml | integration-test/it.values.pipeline.yaml.vm | test-chart/values-pipeline.yaml | + * | itValuesYaml | integration-test/it.values.yaml.vm | test-chart/values.yaml | + * | testLog4jConfiguration | log4j2.xml.vm | log4j2.xml | + * | globalDeploymentConfigFile | pipeline-models/deployment-config.json.vm | deployment-config.json | */ diff --git a/foundation/foundation-mda/src/main/resources/profiles.json b/foundation/foundation-mda/src/main/resources/profiles.json index 0bd3600f8..26ac19d53 100644 --- a/foundation/foundation-mda/src/main/resources/profiles.json +++ b/foundation/foundation-mda/src/main/resources/profiles.json @@ -322,6 +322,12 @@ }, { "name": "valuesCIFile" + }, + { + "name": "baseHiveProperties" + }, + { + "name": "envHiveProperties" } ] }, diff --git a/foundation/foundation-mda/src/main/resources/targets.json b/foundation/foundation-mda/src/main/resources/targets.json index 7c4e36b7c..5697399b9 100644 --- a/foundation/foundation-mda/src/main/resources/targets.json +++ b/foundation/foundation-mda/src/main/resources/targets.json @@ -2413,6 +2413,22 @@ "metadataContext": "targeted", "overwritable": false }, + { + "name": "baseHiveProperties", + "templateName": "templates/deployment/hive-metastore-service/hive.properties.vm", + "outputFile": "configurations/base/hive.properties", + "generator": "com.boozallen.aiops.mda.generator.ModelAgnosticResourcesGenerator", + "metadataContext": "targeted", + "overwritable": false + }, + { + "name": "envHiveProperties", + "templateName": "templates/deployment/hive-metastore-service/hive.properties.vm", + "outputFile": "configurations/env/hive.properties", + "generator": "com.boozallen.aiops.mda.generator.ModelAgnosticResourcesGenerator", + "metadataContext": "targeted", + "overwritable": false + }, { "name": "generatedPostActionInit", "templateName": "templates/python.init.py.vm", diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm index 27ddb2f5f..9a369fffa 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/configuration-store/v2/configuration-store.values-dev.yaml.vm @@ -9,8 +9,6 @@ aissemble-configuration-store-chart: deployment: image: imagePullPolicy: IfNotPresent - service: - type: LoadBalancer # Update 'volumePathOnNode' with the absolute path of the config files to be loaded into the configuration store. # Example '/Users/MyUser/my-project/resources/configs' diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive.properties.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive.properties.vm new file mode 100644 index 000000000..774364784 --- /dev/null +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive.properties.vm @@ -0,0 +1,4 @@ + +# Password and other sensitive information should be encrypted using krausening +# (See Here: https://github.com/TechnologyBrewery/krausening/tree/dev/krausening/#krausening-in-four-pints-leveraging-jasypt-for-encryptingdecrypting-properties) +HIVE_USERNAME=hive \ No newline at end of file diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm index f2fc8e4bb..a4edf50fd 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm @@ -50,8 +50,6 @@ aissemble-hive-metastore-service-chart: auth: # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the # persisted database. - username: hive - # NB: UCS rootPassword: hive replicationPassword: hive password: hive @@ -68,14 +66,9 @@ aissemble-hive-metastore-service-chart: secretKeyRef: name: remote-auth-config key: AWS_ACCESS_KEY_ID - configMap: metastoreServiceConfig: properties: - - name: javax.jdo.option.ConnectionUserName - value: hive - description: Username to use against metastore database - # NB: UCS - name: javax.jdo.option.ConnectionPassword value: hive description: Password to use against metastore database diff --git a/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/PropertiesStep.java b/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/PropertiesStep.java new file mode 100644 index 000000000..8f3c26471 --- /dev/null +++ b/foundation/foundation-mda/src/test/java/com/boozallen/aiops/mda/generator/PropertiesStep.java @@ -0,0 +1,76 @@ +package com.boozallen.aiops.mda.generator;/*- + * #%L + * aiSSEMBLE::Foundation::MDA + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import com.boozallen.aiops.mda.metamodel.element.*; +import io.cucumber.java.Before; +import io.cucumber.java.Scenario; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.technologybrewery.fermenter.mda.GenerateSourcesHelper; +import org.technologybrewery.fermenter.mda.element.ExpandedProfile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class PropertiesStep extends AbstractModelInstanceSteps { + private static final Logger logger = LoggerFactory.getLogger(PropertiesStep.class); + + @Before("@properties-generation") + public void setup(Scenario scenario) throws IOException { + this.scenario = scenario.getName(); + FileUtils.deleteDirectory(GENERATED_METADATA_DIRECTORY); + } + + @Given("project called {string}") + public void project_called(String projectName) throws IOException { + createProject(projectName, "shared"); + } + + @Given("{string} pipeline is using {string}") + public void pipeline_using(String typeName, String implName) throws IOException { + createAndSavePipeline(unique("TestPipeline"), typeName, implName); + } + + @When("the profile for {string} is generated") + public void profile_spark_infra_is_generated(String profileName) throws Exception { + readMetadata(projectName); + Map profiles = loadProfiles(); + GenerateSourcesHelper.performSourceGeneration(profileName, profiles, this::createGenerationContext, (missingProfile, foundProfiles) -> { + throw new RuntimeException("Missing profile: " + missingProfile); + }, new Slf4jDelegate(logger), projectDir.toFile()); + } + + @Then("hive.properties file is generated in {string}") + public void properties_file_generated(String propertiesPath) { + Path properties = projectDir.resolve(propertiesPath); + assertTrue("File not created: " + properties, Files.exists(properties) && Files.isRegularFile(properties)); + } + + @Then("hive.properties file generated in {string}, {string} properties are set to {string}") + public void properties_set_values_correctly(String propertiesPath, String propertyName, String propertyValue) throws IOException { + Path properties = projectDir.resolve(propertiesPath); + List lines = Files.readAllLines(properties); + String expectedPropertiesMap = propertyName + "=" + propertyValue; + assertTrue("Expected properties not found in " + propertiesPath, lines.contains(expectedPropertiesMap)); + } + + +} \ No newline at end of file diff --git a/foundation/foundation-mda/src/test/resources/specifications/properties-generation.feature b/foundation/foundation-mda/src/test/resources/specifications/properties-generation.feature new file mode 100644 index 000000000..fdee0c2b8 --- /dev/null +++ b/foundation/foundation-mda/src/test/resources/specifications/properties-generation.feature @@ -0,0 +1,16 @@ +@properties-generation +Feature: Generating properties file resources + + @module-generation + Scenario:hive properties generation + Given project called "example" + And "data-flow" pipeline is using "data-delivery-spark" + When the profile for "aissemble-spark-infrastructure-deploy-v2" is generated + Then hive.properties file is generated in "" + And hive.properties file generated in "main/resources/configurations/base/hive.properties", "HIVE_USERNAME" properties are set to "hive" + + + Examples: + | hivePropertiesPath | + | main/resources/configurations/base/hive.properties | + | main/resources/configurations/env/hive.properties | \ No newline at end of file diff --git a/foundation/foundation-upgrade/pom.xml b/foundation/foundation-upgrade/pom.xml index 32571170a..e0d612a6b 100644 --- a/foundation/foundation-upgrade/pom.xml +++ b/foundation/foundation-upgrade/pom.xml @@ -92,5 +92,9 @@ ${version.cucumber.reporting.plugin} test + + io.cucumber + cucumber-java + \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigration.java b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigration.java new file mode 100644 index 000000000..331ed81fd --- /dev/null +++ b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigration.java @@ -0,0 +1,110 @@ +package com.boozallen.aissemble.upgrade.migration.v1_11_0; + +/*- + * #%L + * aiSSEMBLE::Foundation::Upgrade + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import com.boozallen.aissemble.upgrade.migration.AbstractAissembleMigration; +import com.boozallen.aissemble.upgrade.util.YamlUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.error.YAMLException; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; + +/** + * This migration removes hive username in the hive-metastore-service values.yaml to use values from configuration store service only if values.yaml has default value. + */ +public class HiveUniversalConfigYAMLMigration extends AbstractAissembleMigration { + private static final Logger logger = LoggerFactory.getLogger(HiveUniversalConfigYAMLMigration.class); + private static final List propertiesForConfigStore = Arrays.asList("javax.jdo.option.ConnectionUserName"); + private static final List mysqlAuthForConfigStore = Arrays.asList("username"); + private static final HashSet linesToRemove = new HashSet<>(Arrays.asList("username: hive", "- name: javax.jdo.option.ConnectionUserName" ,"value: hive", "description: Username to use against metastore database", "# NB: UCS")); + private static final String DEFAULT_HIVE_USERNAME_AND_PASSWORD = "hive"; + @Override + protected boolean shouldExecuteOnFile(File file) { + try { + YamlUtils.YamlObject yaml = YamlUtils.loadYaml(file); + if(yaml.hasObject("aissemble-hive-metastore-service-chart")) + { + yaml = yaml.getObject("aissemble-hive-metastore-service-chart"); + }else{ + return false; + } + if (yaml.hasObject("mysql", "auth")) { + YamlUtils.YamlObject mysqlAuth = yaml.getObject("mysql", "auth"); + for(String key : mysqlAuth.keySet()) + { + if(mysqlAuthForConfigStore.contains(key) && mysqlAuth.get(key).equals(DEFAULT_HIVE_USERNAME_AND_PASSWORD)) + { + return true; + } + } + + } + + if (yaml.hasList("configMap", "metastoreServiceConfig", "properties")) { + List properties = yaml.getListOfObjects("configMap", "metastoreServiceConfig", "properties"); + for(YamlUtils.YamlObject property : properties) + { + if(propertiesForConfigStore.contains(property.get("name")) && property.get("value").equals(DEFAULT_HIVE_USERNAME_AND_PASSWORD)) + { + return true; + } + } + } + return false; + + } catch (YAMLException e) { + if (logger.isDebugEnabled()) { + logger.debug("Failed to parse values yaml file: {}", file.getPath(), e); + } else { + logger.info("Failed to parse values yaml file: {}", file.getName()); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + return false; + } + + @Override + protected boolean performMigration(File file) { + try { + File tempFile = new File("tempValuesFile.yaml"); + BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile)); + List lines = Files.readAllLines(file.toPath()); + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i).trim(); + boolean skip = true; + if(i >= 1 && lines.get(i-1).trim().equals("- name: javax.jdo.option.ConnectionPassword")) + { + skip = false; + } + if(linesToRemove.contains(line) && skip) + { + continue; + } + writer.write(lines.get(i) + System.getProperty("line.separator")); + } + writer.close(); + + return tempFile.renameTo(file); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/foundation/foundation-upgrade/src/main/resources/migrations.json b/foundation/foundation-upgrade/src/main/resources/migrations.json index e2b7f33f6..f4ed689d4 100644 --- a/foundation/foundation-upgrade/src/main/resources/migrations.json +++ b/foundation/foundation-upgrade/src/main/resources/migrations.json @@ -52,6 +52,17 @@ ] } ] + }, + { + "name": "hive-universal-config-yaml-migration", + "implementation": "com.boozallen.aissemble.upgrade.migration.v1_11_0.HiveUniversalConfigYAMLMigration", + "fileSets": [ + { + "includes": [ + "*-deploy/src/main/resources/apps/spark-infrastructure/values.yaml" + ] + } + ] } ] } diff --git a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigrationTest.java b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigrationTest.java new file mode 100644 index 000000000..837b7789b --- /dev/null +++ b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_11_0/HiveUniversalConfigYAMLMigrationTest.java @@ -0,0 +1,51 @@ +package com.boozallen.aissemble.upgrade.migration.v1_11_0; + +/*- + * #%L + * aiSSEMBLE::Foundation::Upgrade + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import com.boozallen.aissemble.upgrade.migration.AbstractMigrationTest; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; + + +public class HiveUniversalConfigYAMLMigrationTest extends AbstractMigrationTest { + @Given("default values.yaml of spark infrastructure") + public void aSparkInfrastructureValuesFileWithDefaultValues() + { + testFile = getTestFile("v1_11_0/HiveUniversalConfigYAMLMigration/migration/default-values.yaml"); + + } + @Given("values.yaml of spark infrastructure with hive username and password changed from \"hive\"") + public void aSparkInfrastructureValuesFileModifiedToCustomValues() + { + testFile = getTestFile("v1_11_0/HiveUniversalConfigYAMLMigration/migration/custom-values.yaml"); + + } + + @When("the 1.11.0 hive configuration migration executes") + public void theHiveConfigurationMigrationExecutes() + { + HiveUniversalConfigYAMLMigration migration = new HiveUniversalConfigYAMLMigration(); + performMigration(migration); + } + + @Then("values.yaml is updated to remove hive username properties") + public void theHiveUsernamePasswordRemoved() { + assertMigrationSuccess(); + var updatedValue = getTestFile("v1_11_0/HiveUniversalConfigYAMLMigration/migration/updated-values.yaml"); + assertLinesMatch("Yaml file not updated with removed hive userrname properties: " + testFile.getName(), testFile, updatedValue); + } + + @Then("hive configuration migration is skipped") + public void theHiveConfigMigrationIsSkipped() { + assertMigrationSkipped(); + } +} diff --git a/foundation/foundation-upgrade/src/test/resources/specifications/v1_11_0/hive-uc-migration.feature b/foundation/foundation-upgrade/src/test/resources/specifications/v1_11_0/hive-uc-migration.feature new file mode 100644 index 000000000..b7a62fe87 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/specifications/v1_11_0/hive-uc-migration.feature @@ -0,0 +1,16 @@ +Feature: Migrate Hive Universal Configuration Migration + The Hive Metastore Service starts to implement Universal Configuration Store to inject hive credentials. We make configuration store call + by the default. If consumer keep default value of hive credentials, it will be migrated to use configuration store, if not it would override to use custom value without migration. + hive.username refers to javax.jdo.option.ConnectionUserName and aissemble-hive-metastore-service-chart.mysql.auth.username + +Scenario: Default Spark infrastructure file is migrated + Given default values.yaml of spark infrastructure + When the 1.11.0 hive configuration migration executes + Then values.yaml is updated to remove hive username properties + + + +Scenario: Spark infrastructure file with modified hive credentials is not migrated + Given values.yaml of spark infrastructure with hive username and password changed from "hive" + When the 1.11.0 hive configuration migration executes + Then hive configuration migration is skipped \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/custom-values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/custom-values.yaml new file mode 100644 index 000000000..c8f9883f0 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/custom-values.yaml @@ -0,0 +1,89 @@ +# This file contains the default values for the aiSSEMBLE Spark Infrastructure Helm chart. + +aissemble-spark-history-chart: + app: + name: "spark-history" + eventVolume: + enabled: true + +aissemble-thrift-server-chart: + app: + name: "thrift-server" + dependencies: + packages: + - org.apache.hadoop:hadoop-aws:3.3.4 + + deployment: + metadata: + labels: + aissemble-configuration-store: enabled + envFromSecret: + AWS_ACCESS_KEY_ID: + secretName: remote-auth-config + key: AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + secretName: remote-auth-config + key: AWS_SECRET_ACCESS_KEY + + sparkConf: | + spark.hadoop.fs.s3a.endpoint=http://s3-local:4566 + spark.hadoop.fs.s3a.access.key=$getConfigValue(groupName=aws-credentials;propertyName=AWS_SECRET_ACCESS_KEY) + spark.hadoop.fs.s3a.secret.key=$getConfigValue(groupName=aws-credentials;propertyName=AWS_SECRET_ACCESS_KEY) + spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem + spark.hive.server2.thrift.port=10000 + spark.hive.server2.thrift.http.port=10001 + spark.hive.server2.transport.mode=http + spark.hive.metastore.warehouse.dir=s3a://spark-infrastructure/warehouse + spark.hadoop.fs.s3a.path.style.access=true + spark.hive.server2.thrift.http.path=cliservice + spark.hive.metastore.schema.verification=false + spark.hive.metastore.uris=thrift://hive-metastore-service:9083/default + + hiveSite: | + + + datanucleus.schema.autoCreateAll + true + Creates necessary schema on a startup if one does not exist + + + +aissemble-hive-metastore-service-chart: + mysql: + auth: + # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the + # persisted database. + username: custom + # NB: UCS + rootPassword: hive + replicationPassword: hive + password: hive + + deployment: + env: + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_SECRET_ACCESS_KEY + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_ACCESS_KEY_ID + configMap: + metastoreServiceConfig: + properties: + - name: javax.jdo.option.ConnectionUserName + value: custom + description: Username to use against metastore database + # NB: UCS + - name: javax.jdo.option.ConnectionPassword + value: hive + description: Password to use against metastore database + - name: metastore.warehouse.dir + value: s3a://spark-infrastructure/warehouse + - name: fs.s3a.endpoint + value: http://s3-local:4566 + - name: fs.s3a.path.style.access + value: true diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/default-values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/default-values.yaml new file mode 100644 index 000000000..20b1a1cbe --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/default-values.yaml @@ -0,0 +1,89 @@ +# This file contains the default values for the aiSSEMBLE Spark Infrastructure Helm chart. + +aissemble-spark-history-chart: + app: + name: "spark-history" + eventVolume: + enabled: true + +aissemble-thrift-server-chart: + app: + name: "thrift-server" + dependencies: + packages: + - org.apache.hadoop:hadoop-aws:3.3.4 + + deployment: + metadata: + labels: + aissemble-configuration-store: enabled + envFromSecret: + AWS_ACCESS_KEY_ID: + secretName: remote-auth-config + key: AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + secretName: remote-auth-config + key: AWS_SECRET_ACCESS_KEY + + sparkConf: | + spark.hadoop.fs.s3a.endpoint=http://s3-local:4566 + spark.hadoop.fs.s3a.access.key=$getConfigValue(groupName=aws-credentials;propertyName=AWS_SECRET_ACCESS_KEY) + spark.hadoop.fs.s3a.secret.key=$getConfigValue(groupName=aws-credentials;propertyName=AWS_SECRET_ACCESS_KEY) + spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem + spark.hive.server2.thrift.port=10000 + spark.hive.server2.thrift.http.port=10001 + spark.hive.server2.transport.mode=http + spark.hive.metastore.warehouse.dir=s3a://spark-infrastructure/warehouse + spark.hadoop.fs.s3a.path.style.access=true + spark.hive.server2.thrift.http.path=cliservice + spark.hive.metastore.schema.verification=false + spark.hive.metastore.uris=thrift://hive-metastore-service:9083/default + + hiveSite: | + + + datanucleus.schema.autoCreateAll + true + Creates necessary schema on a startup if one does not exist + + + +aissemble-hive-metastore-service-chart: + mysql: + auth: + # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the + # persisted database. + username: hive + # NB: UCS + rootPassword: hive + replicationPassword: hive + password: hive + + deployment: + env: + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_SECRET_ACCESS_KEY + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_ACCESS_KEY_ID + configMap: + metastoreServiceConfig: + properties: + - name: javax.jdo.option.ConnectionUserName + value: hive + description: Username to use against metastore database + # NB: UCS + - name: javax.jdo.option.ConnectionPassword + value: hive + description: Password to use against metastore database + - name: metastore.warehouse.dir + value: s3a://spark-infrastructure/warehouse + - name: fs.s3a.endpoint + value: http://s3-local:4566 + - name: fs.s3a.path.style.access + value: true diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/updated-values.yaml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/updated-values.yaml new file mode 100644 index 000000000..acb2cac02 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_11_0/HiveUniversalConfigYAMLMigration/migration/updated-values.yaml @@ -0,0 +1,83 @@ +# This file contains the default values for the aiSSEMBLE Spark Infrastructure Helm chart. + +aissemble-spark-history-chart: + app: + name: "spark-history" + eventVolume: + enabled: true + +aissemble-thrift-server-chart: + app: + name: "thrift-server" + dependencies: + packages: + - org.apache.hadoop:hadoop-aws:3.3.4 + + deployment: + metadata: + labels: + aissemble-configuration-store: enabled + envFromSecret: + AWS_ACCESS_KEY_ID: + secretName: remote-auth-config + key: AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: + secretName: remote-auth-config + key: AWS_SECRET_ACCESS_KEY + + sparkConf: | + spark.hadoop.fs.s3a.endpoint=http://s3-local:4566 + spark.hadoop.fs.s3a.access.key=$getConfigValue(groupName=aws-credentials;propertyName=AWS_SECRET_ACCESS_KEY) + spark.hadoop.fs.s3a.secret.key=$getConfigValue(groupName=aws-credentials;propertyName=AWS_SECRET_ACCESS_KEY) + spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem + spark.hive.server2.thrift.port=10000 + spark.hive.server2.thrift.http.port=10001 + spark.hive.server2.transport.mode=http + spark.hive.metastore.warehouse.dir=s3a://spark-infrastructure/warehouse + spark.hadoop.fs.s3a.path.style.access=true + spark.hive.server2.thrift.http.path=cliservice + spark.hive.metastore.schema.verification=false + spark.hive.metastore.uris=thrift://hive-metastore-service:9083/default + + hiveSite: | + + + datanucleus.schema.autoCreateAll + true + Creates necessary schema on a startup if one does not exist + + + +aissemble-hive-metastore-service-chart: + mysql: + auth: + # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the + # persisted database. + rootPassword: hive + replicationPassword: hive + password: hive + + deployment: + env: + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_SECRET_ACCESS_KEY + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_ACCESS_KEY_ID + configMap: + metastoreServiceConfig: + properties: + - name: javax.jdo.option.ConnectionPassword + value: hive + description: Password to use against metastore database + - name: metastore.warehouse.dir + value: s3a://spark-infrastructure/warehouse + - name: fs.s3a.endpoint + value: http://s3-local:4566 + - name: fs.s3a.path.style.access + value: true