Refactor etcd config as discovery section with convenience types (#2843)

m3db · Nov 9, 2020 · ec68e83 · ec68e83
1 parent 5a40a30
commit ec68e83
Show file tree

Hide file tree

Showing 49 changed files with 803 additions and 1,410 deletions.
diff --git a/kube/bundle.yaml b/kube/bundle.yaml
diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml
@@ -65,15 +65,16 @@ data:
       filesystem:
         filePathPrefix: /var/lib/m3db
 
-      config:
-        service:
-            env: default_env
-            zone: embedded
-            service: m3db
-            cacheDir: /var/lib/m3kv
-            etcdClusters:
-            - zone: embedded
-              endpoints:
-              - http://etcd-0.etcd:2379
-              - http://etcd-1.etcd:2379
-              - http://etcd-2.etcd:2379
+      discovery:
+        config:
+          service:
+              env: default_env
+              zone: embedded
+              service: m3db
+              cacheDir: /var/lib/m3kv
+              etcdClusters:
+              - zone: embedded
+                endpoints:
+                - http://etcd-0.etcd:2379
+                - http://etcd-1.etcd:2379
+                - http://etcd-2.etcd:2379
diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml
@@ -12,20 +12,25 @@ db:
     resolver: environment
     envVarName: M3DB_HOST_ID
 
-  config:
+  # Note: cannot use type: "m3db_single_node" since sometimes
+  # multiple DB nodes spawned using the m3_stack start script
+  # and as such the non-seed nodes need to point etcd not to 
+  # localhost but to m3db_seed:2379 specifically.
+  discovery:
+    config:
       service:
-          env: default_env
-          zone: embedded
-          service: m3db
-          cacheDir: /var/lib/m3kv
-          etcdClusters:
-              - zone: embedded
-                endpoints:
-                    - m3db_seed:2379
-      seedNodes:
+        env: default_env
+        zone: embedded
+        service: m3db
+        cacheDir: /var/lib/m3kv
+        etcdClusters:
+          - zone: embedded
+            endpoints:
+              - m3db_seed:2379
+        seedNodes:
           initialCluster:
-              - hostID: m3db_seed
-                endpoint: http://m3db_seed:2380
+            - hostID: m3db_seed
+              endpoint: http://m3db_seed:2380
 
   # proto:
   #   schemaFilePath: /etc/m3dbnode/schema.proto
diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml
@@ -1,21 +1,5 @@
 listenAddress: 0.0.0.0:7202
 
-logging:
-  level: info
-
-metrics:
-  scope:
-    prefix: "coordinator"
-  prometheus:
-    handlerPath: /metrics
-    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
-  sanitization: prometheus
-  samplingRate: 1.0
-  extended: none
-
-tagOptions:
-  idScheme: quoted
-
 carbon:
   ingester:
     listenAddress: "0.0.0.0:7204"
@@ -47,8 +31,6 @@ clusters:
             - zone: embedded
               endpoints:
                 - dbnode01:2379
-      writeConsistencyLevel: majority
-      readConsistencyLevel: unstrict_majority
 
 downsample:
   rules:

diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
@@ -1,21 +1,5 @@
 listenAddress: 0.0.0.0:7202
 
-logging:
-  level: info
-
-metrics:
-  scope:
-    prefix: "coordinator"
-  prometheus:
-    handlerPath: /metrics
-    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
-  sanitization: prometheus
-  samplingRate: 1.0
-  extended: none
-
-tagOptions:
-  idScheme: quoted
-
 carbon:
   ingester:
     listenAddress: "0.0.0.0:7204"
@@ -47,8 +31,6 @@ clusters:
             - zone: embedded
               endpoints:
                 - dbnode01:2379
-      writeConsistencyLevel: majority
-      readConsistencyLevel: unstrict_majority
 
 downsample:
   remoteAggregator:

diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml
@@ -1,18 +1,3 @@
-listenAddress: 0.0.0.0:7201
-
-logging:
-  level: info
-
-metrics:
-  scope:
-    prefix: "coordinator"
-  prometheus:
-    handlerPath: /metrics
-    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
-  sanitization: prometheus
-  samplingRate: 1.0
-  extended: none
-
 clusters:
   - namespaces:
       - namespace: agg
@@ -33,8 +18,6 @@ clusters:
             - zone: embedded
               endpoints:
                 - dbnode01:2379
-      writeConsistencyLevel: majority
-      readConsistencyLevel: unstrict_majority
 
 carbon:
   ingester:
@@ -56,6 +39,3 @@ carbon:
         policies:
           - resolution: 5s
             retention: 10h
-
-tagOptions:
-  idScheme: quoted
diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
@@ -1,18 +1,3 @@
-listenAddress: 0.0.0.0:7201
-
-logging:
-  level: info
-
-metrics:
-  scope:
-    prefix: "coordinator"
-  prometheus:
-    handlerPath: /metrics
-    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
-  sanitization: prometheus
-  samplingRate: 1.0
-  extended: none
-
 limits:
   perQuery:
     maxFetchedSeries: 100
@@ -37,8 +22,3 @@ clusters:
             - zone: embedded
               endpoints:
                 - dbnode01:2379
-      writeConsistencyLevel: majority
-      readConsistencyLevel: unstrict_majority
-
-tagOptions:
-  idScheme: quoted
diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml
@@ -1,18 +1,3 @@
-listenAddress: 0.0.0.0:7201
-
-logging:
-  level: info
-
-metrics:
-  scope:
-    prefix: "coordinator"
-  prometheus:
-    handlerPath: /metrics
-    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
-  sanitization: prometheus
-  samplingRate: 1.0
-  extended: none
-
 clusters:
   - namespaces:
       - namespace: agg
@@ -41,8 +26,6 @@ clusters:
             - zone: embedded
               endpoints:
                 - dbnode01:2379
-      writeConsistencyLevel: majority
-      readConsistencyLevel: unstrict_majority
 
 downsample:
   rules:
@@ -77,6 +60,3 @@ downsample:
   bufferPastLimits:
     - resolution: 0s
       bufferPast: 90s
-
-tagOptions:
-  idScheme: quoted
diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml
@@ -18,56 +18,14 @@ coordinator:
     idScheme: quoted
 
 db:
-  logging:
-    level: info
-
-  metrics:
-    prometheus:
-      handlerPath: /metrics
-    sanitization: prometheus
-    samplingRate: 1.0
-    extended: detailed
-
-  listenAddress: 0.0.0.0:9000
-  clusterListenAddress: 0.0.0.0:9001
-  httpNodeListenAddress: 0.0.0.0:9002
-  httpClusterListenAddress: 0.0.0.0:9003
-  debugListenAddress: 0.0.0.0:9004
-
   hostID:
     resolver: environment
     envVarName: M3DB_HOST_ID
 
-  client:
-    writeConsistencyLevel: majority
-    readConsistencyLevel: unstrict_majority
-
-  gcPercentage: 100
-
-  writeNewSeriesAsync: true
-  writeNewSeriesBackoffDuration: 2ms
-
-  cache:
-    series:
-      policy: lru
-
-  commitlog:
-    flushMaxBytes: 524288
-    flushEvery: 1s
-    queue:
-      calculationType: fixed
-      size: 2097152
-
-  filesystem:
-    filePathPrefix: /var/lib/m3db
-
-  config:
-    service:
+  discovery:
+    type: m3db_cluster
+    m3dbCluster:
       env: foo-namespace/foo-cluster
       zone: bar-zone
-      service: m3db
-      cacheDir: /var/lib/m3kv
-      etcdClusters:
-      - zone: bar-zone
-        endpoints:
-        - etcd01:2379
+      endpoints:
+      - etcd01:2379
diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml
@@ -1,18 +1,3 @@
-listenAddress: 0.0.0.0:7201
-
-logging:
-  level: info
-
-metrics:
-  scope:
-    prefix: "coordinator"
-  prometheus:
-    handlerPath: /metrics
-    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
-  sanitization: prometheus
-  samplingRate: 1.0
-  extended: none
-
 limits:
   perQuery:
     maxFetchedSeries: 100
@@ -37,8 +22,3 @@ clusters:
             - zone: embedded
               endpoints:
                 - cluster_a_dbnode01:2379
-      writeConsistencyLevel: majority
-      readConsistencyLevel: unstrict_majority
-
-tagOptions:
-  idScheme: quoted
diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml
@@ -1,18 +1,3 @@
-listenAddress: 0.0.0.0:7201
-
-logging:
-  level: info
-
-metrics:
-  scope:
-    prefix: "coordinator"
-  prometheus:
-    handlerPath: /metrics
-    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
-  sanitization: prometheus
-  samplingRate: 1.0
-  extended: none
-
 limits:
   perQuery:
     maxFetchedSeries: 100
@@ -37,8 +22,3 @@ clusters:
             - zone: embedded
               endpoints:
                 - cluster_b_dbnode01:2379
-      writeConsistencyLevel: majority
-      readConsistencyLevel: unstrict_majority
-
-tagOptions:
-  idScheme: quoted