484 Feature: integrate Universal Config store in the hive metastore s…

…ervice
boozallen · Dec 6, 2024 · 44c9e26 · 44c9e26
1 parent 7fbe697
commit 44c9e26
Show file tree

Hide file tree

Showing 21 changed files with 690 additions and 48 deletions.
diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md
@@ -33,6 +33,7 @@ To reduce burden of upgrading aiSSEMBLE, the Baton project is used to automate t
 | upgrade-tiltfile-aissemble-version-migration       | Updates the aiSSEMBLE version within your project's Tiltfile                                                                                                                 |
 | upgrade-v2-chart-files-aissemble-version-migration | Updates the Helm chart dependencies within your project's deployment resources (`<YOUR_PROJECT>-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE |
 | upgrade-v1-chart-files-aissemble-version-migration | Updates the docker image tags within your project's deployment resources (`<YOUR_PROJECT>-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE       |
+| hive-universal-config-yaml-migration               | Updates hive-metastore-service values.yaml to removes hive username to use values from configuration store service only if hive username has default value.                  |
 
 To deactivate any of these migrations, add the following configuration to the `baton-maven-plugin` within your root `pom.xml`:
 

diff --git a/...ions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md b/...ions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md
@@ -48,6 +48,7 @@ configuration options.
 |------------------|---------------------|
 | fullnameOverride | "hive-metastore-db" |
 | auth.database    | "metastore"         |
+| auth.username    | "hive"              |
 
 # Migration from aiSSEMBLE v1 Helm Charts
 
@@ -66,17 +67,17 @@ In the table below, the notation `env[KEY]` refers the `env` list item whose `na
 
 **Note**: *all new property locations include the prefix `aissemble-hive-metastore-service-chart`*
 
-| Old Property Location                                                                          | New Property Location                                                                  | Additional Notes |
-|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|------------------|
-| hive-metastore-db.service.spec.ports[].port                                                    | mysql.primary.service.ports.mysql                                                      | Default to 3306  |
-| deployment.env[HADOOP_CLASSPATH]                                                               | deployment.baseEnv[HADOOP_CLASSPATH]                                                   |                  |
-| deployment.env[JAVA_HOME]                                                                      | deployment.baseEnv[JAVA_HOME]                                                          |                  |
-| configMap.metastoreServiceConfig.configuration.property[metastore.thrift.uris]                 | configMap.metastoreServiceConfig.baseProperties[metastore.thrift.uris]                 |                  |
-| configMap.metastoreServiceConfig.configuration.property[metastore.task.threads.always]         | configMap.metastoreServiceConfig.baseProperties[metastore.task.threads.always]         |                  |
-| configMap.metastoreServiceConfig.configuration.property[metastore.expression.proxy]            | configMap.metastoreServiceConfig.baseProperties[metastore.expression.proxy]            |                  |
-| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionDriverName] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionDriverName] |                  |
-| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionURL]        | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionURL]        |                  |
-
+| Old Property Location                                                                          | New Property Location                                                                  | Additional Notes                                       |
+|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|--------------------------------------------------------|
+| hive-metastore-db.service.spec.ports[].port                                                    | mysql.primary.service.ports.mysql                                                      | Default to 3306                                        |
+| deployment.env[HADOOP_CLASSPATH]                                                               | deployment.baseEnv[HADOOP_CLASSPATH]                                                   |                                                        |
+| deployment.env[JAVA_HOME]                                                                      | deployment.baseEnv[JAVA_HOME]                                                          |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[metastore.thrift.uris]                 | configMap.metastoreServiceConfig.baseProperties[metastore.thrift.uris]                 |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[metastore.task.threads.always]         | configMap.metastoreServiceConfig.baseProperties[metastore.task.threads.always]         |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[metastore.expression.proxy]            | configMap.metastoreServiceConfig.baseProperties[metastore.expression.proxy]            |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionDriverName] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionDriverName] |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionURL]        | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionURL]        |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionUserName]   | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionUserName]   | Using Configuration store Service to inject this value |
 ## Property Removed
 
 The following properties no longer exist.

diff --git a/...helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml b/...helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml
@@ -2,28 +2,41 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: metastore-service-config
+  labels:
+    aissemble-configuration-store: enabled
 data:
   # Add all the default properties from the local values.yaml to the ConfigMap
   # Then check if there are any downstream properties and add them as well
   metastore-site.xml: |
+    {{- $basePropDict := dict }}
+    {{- $propDict := dict }}
+    {{- range $baseProperty := .Values.configMap.metastoreServiceConfig.baseProperties }}
+      {{- $basePropDict := set $basePropDict $baseProperty.name $baseProperty.value }}
+    {{- end }}
+    {{- if .Values.configMap.metastoreServiceConfig.properties }}
+      {{- range $property := .Values.configMap.metastoreServiceConfig.properties }}
+        {{- $propDict := set $propDict $property.name $property.value }}
+      {{- end }}
+    {{- end }}
+    
     <configuration>
-    {{- range $property := .Values.configMap.metastoreServiceConfig.baseProperties }}
+    {{- range $baseProperty := .Values.configMap.metastoreServiceConfig.baseProperties }}
+      {{- if and (hasKey $basePropDict $baseProperty.name) (not (hasKey $propDict $baseProperty.name))}}
       <property>
-        <name>{{ $property.name }}</name>
-        <value>{{ $property.value }}</value>
-
-      {{- if $property.description }}
-        <description>{{ $property.description }}</description>
+        <name>{{ $baseProperty.name }}</name>
+        <value>{{ $baseProperty.value }}</value>
+      {{- if $baseProperty.description }}
+        <description>{{ $baseProperty.description }}</description>
       {{- end }}
       </property>
+      {{- end }}
     {{- end }}
 
     {{- if .Values.configMap.metastoreServiceConfig.properties }}
       {{- range $property := .Values.configMap.metastoreServiceConfig.properties }}
       <property>
         <name>{{ $property.name }}</name>
         <value>{{ $property.value }}</value>
-
         {{- if $property.description }}
         <description>{{ $property.description }}</description>
         {{- end }}

diff --git a/...elm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml b/...elm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml
@@ -33,6 +33,11 @@ tests:
                   <value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
                   <description>JDBC connect string for a JDBC metastore</description>
                 </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionUserName</name>
+                  <value>$getConfigValue(groupName=hive;propertyName=HIVE_USERNAME)</value>
+                  <description>Username to use against metastore database</description>
+                </property>
               </configuration>
   - it: Should override default properties appropriately
     set:
@@ -90,6 +95,11 @@ tests:
                   <value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
                   <description>JDBC connect string for a JDBC metastore</description>
                 </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionUserName</name>
+                  <value>$getConfigValue(groupName=hive;propertyName=HIVE_USERNAME)</value>
+                  <description>Username to use against metastore database</description>
+                </property>
                 <property>
                   <name>propertyName1</name>
                   <value>value1</value>
@@ -100,3 +110,48 @@ tests:
                   <value>value2</value>
                 </property>
               </configuration>
+  - it: Should override baseProperties with properties appropriately
+    set:
+      configMap:
+        metastoreServiceConfig:
+          properties:
+            - name: metastore.thrift.uris
+              value: thrift://0.0.0.0:8081
+              description: Overridden metastore.thrift.uris
+            - name: metastore.task.threads.always
+              value: overridden metastore.task.threads.always
+    asserts:
+      - equal:
+          path: data
+          value:
+            metastore-site.xml: |
+              <configuration>
+                <property>
+                  <name>metastore.expression.proxy</name>
+                  <value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value>
+                </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionDriverName</name>
+                  <value>com.mysql.cj.jdbc.Driver</value>
+                </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionURL</name>
+                  <value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
+                  <description>JDBC connect string for a JDBC metastore</description>
+                </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionUserName</name>
+                  <value>$getConfigValue(groupName=hive;propertyName=HIVE_USERNAME)</value>
+                  <description>Username to use against metastore database</description>
+                </property>
+                <property>
+                  <name>metastore.thrift.uris</name>
+                  <value>thrift://0.0.0.0:8081</value>
+                  <description>Overridden metastore.thrift.uris</description>
+                </property>
+                <property>
+                  <name>metastore.task.threads.always</name>
+                  <value>overridden metastore.task.threads.always</value>
+                </property>
+              </configuration>
+              
diff --git a/...m/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml b/...m/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml
@@ -15,11 +15,16 @@ image:
   dockerRepo: "ghcr.io/"
 
 mysql:
+  commonLabels:
+    aissemble-configuration-store: enabled
   enabled: true
   fullnameOverride: "hive-metastore-db"
   auth:
     # The schematool for the metastore service will create the database for us.
     database: "metastore"
+    # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the
+    # persisted database.
+    username: $getConfigValue(groupName=hive;propertyName=HIVE_USERNAME)
 
 hive:
   dbType: "mysql"
@@ -72,4 +77,7 @@ configMap:
       - name: javax.jdo.option.ConnectionURL
         value: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false
         description: JDBC connect string for a JDBC metastore
+      - name: javax.jdo.option.ConnectionUserName
+        value: $getConfigValue(groupName=hive;propertyName=HIVE_USERNAME)
+        description: Username to use against metastore database
     properties: {}
diff --git a/...-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java b/...-mda/src/main/java/com/boozallen/aiops/mda/generator/ModelAgnosticResourcesGenerator.java
@@ -22,27 +22,29 @@
 public class ModelAgnosticResourcesGenerator extends AbstractModelAgnosticGenerator {
     /*--~-~-~~
      * Usages:
-     * | Target                           | Template                                           | Generated File                                  |
-     * |----------------------------------|----------------------------------------------------|-------------------------------------------------|
-     * | cdiBeansXml                      | beans.xml.vm                                       | META-INF/beans.xml                              |
-     * | cucumberPipelineFeature          | cucumber.pipeline.feature.vm                       | specifications/pipeline.feature                 |
-     * | cucumberProperties               | cucumber.properties.vm                             | cucumber.properties                             |
-     * | dataAccessApplicationProperties  | data-access/data.access.application.properties.vm  | application.properties                          |
-     * | mlflowStartScript                | general-docker/mlflow.start.sh.vm                  | start.sh                                        |
-     * | sparkDataDeliveryProperties      | general-docker/spark.data.delivery.properties.vm   | krausening/base/spark-data-delivery.properties  |
-     * | authConfigResource               | general-mlflow/auth.properties.vm                  | krausening/base/auth.properties                 |
-     * | inferenceConfigResource          | general-mlflow/inference.config.properties.vm      | krausening/base/inference.properties            |
-     * | trainingPipelineConfigResource   | general-mlflow/training.config.properties.vm       | krausening/base/pipeline.properties             |
-     * | itChartYaml                      | integration-test/it.chart.yaml.vm                  | test-chart/Chart.yaml                           |
-     * | itPipelineSpecification          | integration-test/it.pipeline.spec.vm               | specifications/pipeline.feature                 |
-     * | itServiceAccountYaml             | integration-test/it.serviceaccount.yaml.vm         | test-chart/templates/serviceaccount.yaml        |
-     * | itTestYaml                       | integration-test/it.test.yaml.vm                   | test-chart/templates/test.yaml                  |
-     * | itTiltfile                       | integration-test/it.tiltfile.vm                    | Tiltfile                                        |
-     * | itValuesCIYaml                   | integration-test/it.values.ci.yaml.vm              | test-chart/values-ci.yaml                       |
-     * | itValuesPipelineYaml             | integration-test/it.values.pipeline.yaml.vm        | test-chart/values-pipeline.yaml                 |
-     * | itValuesYaml                     | integration-test/it.values.yaml.vm                 | test-chart/values.yaml                          |
-     * | testLog4jConfiguration           | log4j2.xml.vm                                      | log4j2.xml                                      |
-     * | globalDeploymentConfigFile       | pipeline-models/deployment-config.json.vm          | deployment-config.json                          |
+     * | Target                           | Template                                              | Generated File                                  |
+     * |----------------------------------|-------------------------------------------------------|-------------------------------------------------|
+     * | cdiBeansXml                      | beans.xml.vm                                          | META-INF/beans.xml                              |
+     * | cucumberPipelineFeature          | cucumber.pipeline.feature.vm                          | specifications/pipeline.feature                 |
+     * | cucumberProperties               | cucumber.properties.vm                                | cucumber.properties                             |
+     * | dataAccessApplicationProperties  | data-access/data.access.application.properties.vm     | application.properties                          |
+     * | baseHiveProperties               | deployment/hive-metastore-service/hive.properties.vm  | configurations/base/hive.properties             |
+     * | envHiveProperties                | deployment/hive-metastore-service/hive.properties.vm  | configurations/env/hive.properties              |
+     * | mlflowStartScript                | general-docker/mlflow.start.sh.vm                     | start.sh                                        |
+     * | sparkDataDeliveryProperties      | general-docker/spark.data.delivery.properties.vm      | krausening/base/spark-data-delivery.properties  |
+     * | authConfigResource               | general-mlflow/auth.properties.vm                     | krausening/base/auth.properties                 |
+     * | inferenceConfigResource          | general-mlflow/inference.config.properties.vm         | krausening/base/inference.properties            |
+     * | trainingPipelineConfigResource   | general-mlflow/training.config.properties.vm          | krausening/base/pipeline.properties             |
+     * | itChartYaml                      | integration-test/it.chart.yaml.vm                     | test-chart/Chart.yaml                           |
+     * | itPipelineSpecification          | integration-test/it.pipeline.spec.vm                  | specifications/pipeline.feature                 |
+     * | itServiceAccountYaml             | integration-test/it.serviceaccount.yaml.vm            | test-chart/templates/serviceaccount.yaml        |
+     * | itTestYaml                       | integration-test/it.test.yaml.vm                      | test-chart/templates/test.yaml                  |
+     * | itTiltfile                       | integration-test/it.tiltfile.vm                       | Tiltfile                                        |
+     * | itValuesCIYaml                   | integration-test/it.values.ci.yaml.vm                 | test-chart/values-ci.yaml                       |
+     * | itValuesPipelineYaml             | integration-test/it.values.pipeline.yaml.vm           | test-chart/values-pipeline.yaml                 |
+     * | itValuesYaml                     | integration-test/it.values.yaml.vm                    | test-chart/values.yaml                          |
+     * | testLog4jConfiguration           | log4j2.xml.vm                                         | log4j2.xml                                      |
+     * | globalDeploymentConfigFile       | pipeline-models/deployment-config.json.vm             | deployment-config.json                          |
      */
 
 

diff --git a/foundation/foundation-mda/src/main/resources/profiles.json b/foundation/foundation-mda/src/main/resources/profiles.json
@@ -322,6 +322,12 @@
 			},
 			{
 				"name": "valuesCIFile"
+			},
+			{
+				"name": "baseHiveProperties"
+			},
+			{
+				"name": "envHiveProperties"
 			}
 		]
 	},
-Original file line number
+Diff line change
@@ Expand Up / @@ -322,6 +322,12 @@ @@
     			},
     			{
     				"name": "valuesCIFile"
+    			},
+    			{
+    				"name": "baseHiveProperties"
+    			},
+    			{
+    				"name": "envHiveProperties"
     			}
     		]
     	},
@@ Expand Down @@