[Merged by Bors] - Consolidated CRD to top level cluster config

CHANGELOG.md

@@ -11,12 +11,16 @@ All notable changes to this project will be documented in this file.
 - Consolidated security context user, group and fs group ([#277]).
 - [BREAKING] Use Product image selection instead of version. `spec.version` has been replaced by `spec.image` ([#280]).
 - Fix role group node selector ([#283]).
+- [BREAKING] Moved `database` specification from role / role-group level to top-level `clusterConfig` ([#292]).
+- [BREAKING] Moved `s3`, `serviceType` and `hdfs` discovery to top-level `clusterConfig` ([#292]).
+- `operator-rs` `0.30.1` -> `0.30.2` ([#292])
 
 [#271]: https://github.com/stackabletech/hive-operator/pull/271
 [#274]: https://github.com/stackabletech/hive-operator/pull/274
 [#277]: https://github.com/stackabletech/hive-operator/pull/277
 [#280]: https://github.com/stackabletech/hive-operator/pull/280
 [#283]: https://github.com/stackabletech/hive-operator/pull/283
+[#292]: https://github.com/stackabletech/hive-operator/pull/292
 
 ## [0.8.0] - 2022-11-07
 

docs/modules/ROOT/pages/usage.adoc

@@ -9,12 +9,13 @@ Please open an https://github.com/stackabletech/hive-operator/issues[issue] if y
 == S3 Support
 
 Hive supports creating tables in S3 compatible object stores.
-To use this feature you need to provide connection details for the object store using the xref:home:concepts:s3.adoc[S3Connection].
+To use this feature you need to provide connection details for the object store using the xref:home:concepts:s3.adoc[S3Connection] in the top level `clusterConfig`.
 
 An example usage can look like this:
 
 [source,yaml]
 ----
+clusterConfig:
   s3:
     inline:
       host: minio
@@ -27,10 +28,11 @@ An example usage can look like this:
 == Apache HDFS Support
 
 As well as S3, Hive also supports creating tables in HDFS.
-You can add the HDFS connection as follows:
+You can add the HDFS connection in the top level `clusterConfig` as follows:
 
 [source,yaml]
 ----
+clusterConfig:
   hdfs:
     configMap: my-hdfs-cluster # Name of the HdfsCluster
 ----
@@ -185,21 +187,20 @@ spec:
   image:
     productVersion: 3.1.3
     stackableVersion: 0.2.0
+  clusterConfig:
+    database:
+      connString: jdbc:derby:;databaseName=/tmp/metastore_db;create=true
+      user: APP
+      password: mine
+      dbType: derby
   metastore:
     roleGroups:
       default:
-        selector:
-          matchLabels:
-            kubernetes.io/os: linux
         replicas: 1
-        config:
-          database:
-            connString: jdbc:derby:;databaseName=/tmp/metastore_db;create=true
-            user: APP
-            password: mine
-            dbType: derby
 ----
 
+WARNING: You should not use the `Derby` database with more than one replica or in production. Derby stores data locally and therefore the data is not shared between different metastore Pods and lost after Pod restarts.
+
 To create a single node Apache Hive Metastore (v2.3.9) cluster with derby and S3 access, deploy a minio (or use any available S3 bucket):
 [source,bash]
 ----
@@ -229,26 +230,23 @@ spec:
   image:
     productVersion: 3.1.3
     stackableVersion: 0.2.0
-  s3:
-    inline:
-      host: minio
-      port: 9000
-      accessStyle: Path
-      credentials:
-        secretClass: simple-hive-s3-secret-class
+  clusterConfig:
+    database:
+      connString: jdbc:derby:;databaseName=/stackable/metastore_db;create=true
+      user: APP
+      password: mine
+      dbType: derby
+    s3:
+      inline:
+        host: minio
+        port: 9000
+        accessStyle: Path
+        credentials:
+          secretClass: simple-hive-s3-secret-class
   metastore:
     roleGroups:
       default:
-        selector:
-          matchLabels:
-            kubernetes.io/os: linux
         replicas: 1
-        config:
-          database:
-            connString: jdbc:derby:;databaseName=/stackable/metastore_db;create=true
-            user: APP
-            password: mine
-            dbType: derby
 ---
 apiVersion: secrets.stackable.tech/v1alpha1
 kind: SecretClass
@@ -302,17 +300,14 @@ spec:
   image:
     productVersion: 3.1.3
     stackableVersion: 0.2.0
+  clusterConfig:
+    database:
+      connString: jdbc:postgresql://hive-postgresql.default.svc.cluster.local:5432/hive
+      user: hive
+      password: hive
+      dbType: postgres
   metastore:
     roleGroups:
       default:
-        selector:
-          matchLabels:
-            kubernetes.io/os: linux
         replicas: 1
-        config:
-          database:
-            connString: jdbc:postgresql://hive-postgresql.default.svc.cluster.local:5432/hive
-            user: hive
-            password: hive
-            dbType: postgres
 ----

docs/modules/getting_started/examples/code/hive-postgres-s3.yaml

@@ -7,18 +7,15 @@ spec:
   image:
     productVersion: 3.1.3
     stackableVersion: 0.2.0
-  s3:
-    reference: minio
+  clusterConfig:
+    database:
+      connString: jdbc:postgresql://postgresql:5432/hive
+      user: hive
+      password: hive
+      dbType: postgres
+    s3:
+      reference: minio
   metastore:
     roleGroups:
       default:
-        selector:
-          matchLabels:
-            kubernetes.io/os: linux
         replicas: 1
-        config:
-          database:
-            connString: jdbc:postgresql://postgresql:5432/hive
-            user: hive
-            password: hive
-            dbType: postgres

examples/simple-hive-cluster-postgres-s3.yaml

@@ -19,26 +19,23 @@ spec:
   image:
     productVersion: 3.1.3
     stackableVersion: 0.2.0
-  s3:
-    inline:
-      host: test-minio
-      port: 9000
-      accessStyle: Path
-      credentials:
-        secretClass: simple-hive-s3-secret-class
+  clusterConfig:
+    database:
+      connString: jdbc:derby:;databaseName=/tmp/hive;create=true
+      user: APP
+      password: mine
+      dbType: derby
+    s3:
+      inline:
+        host: test-minio
+        port: 9000
+        accessStyle: Path
+        credentials:
+          secretClass: simple-hive-s3-secret-class
   metastore:
     roleGroups:
       default:
-        selector:
-          matchLabels:
-            kubernetes.io/os: linux
         replicas: 1
-        config:
-          database:
-            connString: jdbc:postgresql://hive-postgresql.default.svc.cluster.local:5432/hive
-            user: hive
-            password: hive
-            dbType: postgres
 ---
 apiVersion: secrets.stackable.tech/v1alpha1
 kind: SecretClass

examples/simple-hive-cluster.yaml

@@ -7,16 +7,17 @@ spec:
   image:
     productVersion: 2.3.9
     stackableVersion: 0.6.0
+  clusterConfig:
+    database:
+      connString: jdbc:derby:;databaseName=/tmp/hive;create=true
+      user: APP
+      password: mine
+      dbType: derby
   metastore:
     roleGroups:
       default:
         replicas: 1
         config:
-          database:
-            connString: jdbc:derby:;databaseName=/tmp/hive;create=true
-            user: APP
-            password: mine
-            dbType: derby
           resources:
             storage:
               data:

rust/crd/Cargo.toml

@@ -13,6 +13,6 @@ indoc = "1.0.8"
 serde = "1.0"
 serde_json = "1.0"
 snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" }
 strum = { version = "0.24", features = ["derive"] }
 tracing = "0.1"

rust/crd/src/lib.rs

@@ -70,17 +70,28 @@ pub enum Error {
     )
 )]
 pub struct HiveClusterSpec {
-    /// Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would)
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub stopped: Option<bool>,
+    /// General Hive metastore cluster settings
+    pub cluster_config: HiveClusterConfig,
     /// The Hive metastore image to use
     pub image: ProductImage,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub metastore: Option<Role<MetaStoreConfig>>,
+    /// Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would)
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub s3: Option<S3ConnectionDef>,
+    pub stopped: Option<bool>,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HiveClusterConfig {
+    /// Database connection specification
+    pub database: DatabaseConnectionSpec,
+    /// HDFS connection specification
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub hdfs: Option<HdfsConnection>,
+    /// S3 connection specification
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub s3: Option<S3ConnectionDef>,
     /// Specify the type of the created kubernetes service.
     /// This attribute will be removed in a future release when listener-operator is finished.
     /// Use with caution.
@@ -95,7 +106,7 @@ pub struct HdfsConnection {
     pub config_map: String,
 }
 
-#[derive(strum::Display)]
+#[derive(Display)]
 #[strum(serialize_all = "camelCase")]
 pub enum HiveRole {
     #[strum(serialize = "metastore")]
@@ -149,9 +160,9 @@ pub struct MetastoreStorageConfig {
 #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct MetaStoreConfig {
+    /// The location of default database for the Hive warehouse.
+    /// Maps to the `hive.metastore.warehouse.dir` setting.
     pub warehouse_dir: Option<String>,
-    #[serde(default)]
-    pub database: DatabaseConnectionSpec,
     pub resources: Option<ResourcesFragment<MetastoreStorageConfig, NoRuntimeLimits>>,
 }
 
@@ -263,7 +274,7 @@ impl Configuration for MetaStoreConfig {
 
     fn compute_env(
         &self,
-        _resource: &Self::Configurable,
+        _hive: &Self::Configurable,
         _role_name: &str,
     ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
         let mut result = BTreeMap::new();
@@ -283,20 +294,20 @@ impl Configuration for MetaStoreConfig {
 
     fn compute_cli(
         &self,
-        _resource: &Self::Configurable,
+        hive: &Self::Configurable,
         _role_name: &str,
     ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
         let mut result = BTreeMap::new();
         result.insert(
             Self::DB_TYPE_CLI.to_string(),
-            Some(self.database.db_type.to_string()),
+            Some(hive.spec.cluster_config.database.db_type.to_string()),
         );
         Ok(result)
     }
 
     fn compute_files(
         &self,
-        _resource: &Self::Configurable,
+        hive: &Self::Configurable,
         _role_name: &str,
         _file: &str,
     ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
@@ -310,19 +321,26 @@ impl Configuration for MetaStoreConfig {
         }
         result.insert(
             Self::CONNECTION_URL.to_string(),
-            Some(self.database.conn_string.clone()),
+            Some(hive.spec.cluster_config.database.conn_string.clone()),
         );
         result.insert(
             Self::CONNECTION_USER_NAME.to_string(),
-            Some(self.database.user.clone()),
+            Some(hive.spec.cluster_config.database.user.clone()),
         );
         result.insert(
             Self::CONNECTION_PASSWORD.to_string(),
-            Some(self.database.password.clone()),
+            Some(hive.spec.cluster_config.database.password.clone()),
         );
         result.insert(
             Self::CONNECTION_DRIVER_NAME.to_string(),
-            Some(self.database.db_type.get_jdbc_driver_class().to_string()),
+            Some(
+                hive.spec
+                    .cluster_config
+                    .database
+                    .db_type
+                    .get_jdbc_driver_class()
+                    .to_string(),
+            ),
         );
 
         result.insert(

rust/operator-binary/Cargo.toml

@@ -19,12 +19,12 @@ serde = "1.0"
 serde_json = "1.0"
 snafu = "0.7"
 stackable-hive-crd = { path = "../crd" }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" }
 strum = { version = "0.24", features = ["derive"] }
 tokio = { version = "1.23", features = ["full"] }
 tracing = "0.1"
 
 [build-dependencies]
 built = { version =  "0.5", features = ["chrono", "git2"] }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" }
 stackable-hive-crd = { path = "../crd" }