Merge pull request #494 from boozallen/484-integrate-config-store-hive

#484 Feature: integrate Universal Config store in the hive metastore service
boozallen · Dec 9, 2024 · f46877e · f46877e
2 parents 7fbe697 + ddbc3ad
commit f46877e
Show file tree

Hide file tree

Showing 22 changed files with 712 additions and 53 deletions.
diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md
@@ -28,11 +28,12 @@ The following steps will upgrade your project to `1.11`. These instructions cons
 ## Automatic Upgrades
 To reduce burden of upgrading aiSSEMBLE, the Baton project is used to automate the migration of some files to the new version.  These migrations run automatically when you build your project, and are included by default when you update the `build-parent` version in your root POM.  Below is a description of all of the Baton migrations that are included with this version of aiSSEMBLE.
 
-| Migration Name                                     | Description                                                                                                                                                                  |
-|----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| upgrade-tiltfile-aissemble-version-migration       | Updates the aiSSEMBLE version within your project's Tiltfile                                                                                                                 |
-| upgrade-v2-chart-files-aissemble-version-migration | Updates the Helm chart dependencies within your project's deployment resources (`<YOUR_PROJECT>-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE |
-| upgrade-v1-chart-files-aissemble-version-migration | Updates the docker image tags within your project's deployment resources (`<YOUR_PROJECT>-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE       |
+| Migration Name                                       | Description                                                                                                                                                                 |
+|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| upgrade-tiltfile-aissemble-version-migration         | Updates the aiSSEMBLE version within your project's Tiltfile                                                                                                                |
+| upgrade-v2-chart-files-aissemble-version-migration   | Updates the Helm chart dependencies within your project's deployment resources (`<YOUR_PROJECT>-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE |
+| upgrade-v1-chart-files-aissemble-version-migration   | Updates the docker image tags within your project's deployment resources (`<YOUR_PROJECT>-deploy/src/main/resources/apps/`) to use the latest version of the aiSSEMBLE      |
+| spark-infrastructure-universal-config-yaml-migration | Removes the default hive username (if present) from hive-metastore-service values.yaml so that it can be set by the configuration store service                             |
 
 To deactivate any of these migrations, add the following configuration to the `baton-maven-plugin` within your root `pom.xml`:
 

diff --git a/...ions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md b/...ions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md
@@ -48,6 +48,7 @@ configuration options.
 |------------------|---------------------|
 | fullnameOverride | "hive-metastore-db" |
 | auth.database    | "metastore"         |
+| auth.username    | "hive"              |
 
 # Migration from aiSSEMBLE v1 Helm Charts
 
@@ -66,17 +67,17 @@ In the table below, the notation `env[KEY]` refers the `env` list item whose `na
 
 **Note**: *all new property locations include the prefix `aissemble-hive-metastore-service-chart`*
 
-| Old Property Location                                                                          | New Property Location                                                                  | Additional Notes |
-|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|------------------|
-| hive-metastore-db.service.spec.ports[].port                                                    | mysql.primary.service.ports.mysql                                                      | Default to 3306  |
-| deployment.env[HADOOP_CLASSPATH]                                                               | deployment.baseEnv[HADOOP_CLASSPATH]                                                   |                  |
-| deployment.env[JAVA_HOME]                                                                      | deployment.baseEnv[JAVA_HOME]                                                          |                  |
-| configMap.metastoreServiceConfig.configuration.property[metastore.thrift.uris]                 | configMap.metastoreServiceConfig.baseProperties[metastore.thrift.uris]                 |                  |
-| configMap.metastoreServiceConfig.configuration.property[metastore.task.threads.always]         | configMap.metastoreServiceConfig.baseProperties[metastore.task.threads.always]         |                  |
-| configMap.metastoreServiceConfig.configuration.property[metastore.expression.proxy]            | configMap.metastoreServiceConfig.baseProperties[metastore.expression.proxy]            |                  |
-| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionDriverName] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionDriverName] |                  |
-| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionURL]        | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionURL]        |                  |
-
+| Old Property Location                                                                          | New Property Location                                                                  | Additional Notes                                       |
+|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|--------------------------------------------------------|
+| hive-metastore-db.service.spec.ports[].port                                                    | mysql.primary.service.ports.mysql                                                      | Default to 3306                                        |
+| deployment.env[HADOOP_CLASSPATH]                                                               | deployment.baseEnv[HADOOP_CLASSPATH]                                                   |                                                        |
+| deployment.env[JAVA_HOME]                                                                      | deployment.baseEnv[JAVA_HOME]                                                          |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[metastore.thrift.uris]                 | configMap.metastoreServiceConfig.baseProperties[metastore.thrift.uris]                 |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[metastore.task.threads.always]         | configMap.metastoreServiceConfig.baseProperties[metastore.task.threads.always]         |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[metastore.expression.proxy]            | configMap.metastoreServiceConfig.baseProperties[metastore.expression.proxy]            |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionDriverName] | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionDriverName] |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionURL]        | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionURL]        |                                                        |
+| configMap.metastoreServiceConfig.configuration.property[javax.jdo.option.ConnectionUserName]   | configMap.metastoreServiceConfig.baseProperties[javax.jdo.option.ConnectionUserName]   | Using Configuration store Service to inject this value |
 ## Property Removed
 
 The following properties no longer exist.

diff --git a/...helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml b/...helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml
@@ -2,28 +2,41 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: metastore-service-config
+  labels:
+    aissemble-configuration-store: {{ .Values.configMap.metadata.labels.configStore}}
 data:
   # Add all the default properties from the local values.yaml to the ConfigMap
   # Then check if there are any downstream properties and add them as well
   metastore-site.xml: |
+    {{- $basePropDict := dict }}
+    {{- $propDict := dict }}
+    {{- range $baseProperty := .Values.configMap.metastoreServiceConfig.baseProperties }}
+      {{- $basePropDict := set $basePropDict $baseProperty.name $baseProperty.value }}
+    {{- end }}
+    {{- if .Values.configMap.metastoreServiceConfig.properties }}
+      {{- range $property := .Values.configMap.metastoreServiceConfig.properties }}
+        {{- $propDict := set $propDict $property.name $property.value }}
+      {{- end }}
+    {{- end }}
+    
     <configuration>
-    {{- range $property := .Values.configMap.metastoreServiceConfig.baseProperties }}
+    {{- range $baseProperty := .Values.configMap.metastoreServiceConfig.baseProperties }}
+      {{- if and (hasKey $basePropDict $baseProperty.name) (not (hasKey $propDict $baseProperty.name))}}
       <property>
-        <name>{{ $property.name }}</name>
-        <value>{{ $property.value }}</value>
-
-      {{- if $property.description }}
-        <description>{{ $property.description }}</description>
+        <name>{{ $baseProperty.name }}</name>
+        <value>{{ $baseProperty.value }}</value>
+      {{- if $baseProperty.description }}
+        <description>{{ $baseProperty.description }}</description>
       {{- end }}
       </property>
+      {{- end }}
     {{- end }}
 
     {{- if .Values.configMap.metastoreServiceConfig.properties }}
       {{- range $property := .Values.configMap.metastoreServiceConfig.properties }}
       <property>
         <name>{{ $property.name }}</name>
         <value>{{ $property.value }}</value>
-
         {{- if $property.description }}
         <description>{{ $property.description }}</description>
         {{- end }}

diff --git a/...elm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml b/...elm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml
@@ -33,6 +33,11 @@ tests:
                   <value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
                   <description>JDBC connect string for a JDBC metastore</description>
                 </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionUserName</name>
+                  <value>$getConfigValue(groupName=spark-infrastructure;propertyName=metastore.db.username)</value>
+                  <description>Username to use against metastore database</description>
+                </property>
               </configuration>
   - it: Should override default properties appropriately
     set:
@@ -90,6 +95,11 @@ tests:
                   <value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
                   <description>JDBC connect string for a JDBC metastore</description>
                 </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionUserName</name>
+                  <value>$getConfigValue(groupName=spark-infrastructure;propertyName=metastore.db.username)</value>
+                  <description>Username to use against metastore database</description>
+                </property>
                 <property>
                   <name>propertyName1</name>
                   <value>value1</value>
@@ -100,3 +110,48 @@ tests:
                   <value>value2</value>
                 </property>
               </configuration>
+  - it: Should override baseProperties with properties appropriately
+    set:
+      configMap:
+        metastoreServiceConfig:
+          properties:
+            - name: metastore.thrift.uris
+              value: thrift://0.0.0.0:8081
+              description: Overridden metastore.thrift.uris
+            - name: metastore.task.threads.always
+              value: overridden metastore.task.threads.always
+    asserts:
+      - equal:
+          path: data
+          value:
+            metastore-site.xml: |
+              <configuration>
+                <property>
+                  <name>metastore.expression.proxy</name>
+                  <value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value>
+                </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionDriverName</name>
+                  <value>com.mysql.cj.jdbc.Driver</value>
+                </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionURL</name>
+                  <value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
+                  <description>JDBC connect string for a JDBC metastore</description>
+                </property>
+                <property>
+                  <name>javax.jdo.option.ConnectionUserName</name>
+                  <value>$getConfigValue(groupName=spark-infrastructure;propertyName=metastore.db.username)</value>
+                  <description>Username to use against metastore database</description>
+                </property>
+                <property>
+                  <name>metastore.thrift.uris</name>
+                  <value>thrift://0.0.0.0:8081</value>
+                  <description>Overridden metastore.thrift.uris</description>
+                </property>
+                <property>
+                  <name>metastore.task.threads.always</name>
+                  <value>overridden metastore.task.threads.always</value>
+                </property>
+              </configuration>
+              
diff --git a/...m/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml b/...m/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml
@@ -15,11 +15,16 @@ image:
   dockerRepo: "ghcr.io/"
 
 mysql:
+  commonLabels:
+    aissemble-configuration-store: enabled
   enabled: true
   fullnameOverride: "hive-metastore-db"
   auth:
     # The schematool for the metastore service will create the database for us.
     database: "metastore"
+    # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the
+    # persisted database.
+    username: $getConfigValue(groupName=spark-infrastructure;propertyName=metastore.db.username)
 
 hive:
   dbType: "mysql"
@@ -58,6 +63,9 @@ service:
 
 # hive-metastore-service Config Map
 configMap:
+  metadata:
+    labels:
+      configStore: enabled
   metastoreServiceConfig:
     baseProperties:
       - name: metastore.thrift.uris
@@ -72,4 +80,7 @@ configMap:
       - name: javax.jdo.option.ConnectionURL
         value: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false
         description: JDBC connect string for a JDBC metastore
+      - name: javax.jdo.option.ConnectionUserName
+        value: $getConfigValue(groupName=spark-infrastructure;propertyName=metastore.db.username)
+        description: Username to use against metastore database
     properties: {}