[config] Remove deprecated configuration fields (#2771)

m3db · Oct 29, 2020 · d86e3ef · d86e3ef
1 parent 72fd76b
commit d86e3ef
Show file tree

Hide file tree

Showing 165 changed files with 459 additions and 5,181 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,39 @@
 # Changelog
 
+# 1.0.0 (PROVISIONAL - STILL WORK IN PROGRESS)
+
+## Backwards Incompatible Changes
+
+### Configuration
+- **M3DB**: `db.bootstrap.bootstrappers` removed
+- **M3Coordinator**: `cluster.namespaces.storageMetricsType` removed
+- **M3Coordinator**: `tagOptions.tagOptions` no longer supports `legacy` type
+- **M3Query**: `limits.perQuery.maxComputedDatapoints` removed
+- **M3Query**: `limits.perQuery.maxFetchedDatapoints` removed
+- **M3Query**: `limits.global.maxFetchedDatapoints` removed
+- **M3Query**: `cache` removed
+- **M3Query**: `listenAddress` changed to always be resolved as a string from config. Format changed from
+```
+listenAddress:
+  config: "..."
+  value: "..."
+```
+to 
+```
+listenAddress: "..."
+```
+
+### API
+- **M3DB**: `/services/m3db/database/config/bootstrappers` dynamic bootstappers endpoint removed
+- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace` in favor of stable preferred URL `/api/v1/services/m3db/namespace`
+- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace/init` in favor of stable preferred URL `/api/v1/services/m3db/namespace/init`
+- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace/unagg` in favor of stable preferred URL `/api/v1/services/m3db/namespace/unagg`
+- **M3Coordinator**: Removed deprecated URL `/api/v1/placement` in favor of stable preferred URL `/api/v1/services/m3db/placement`
+- **M3Coordinator**: Removed deprecated URL `/api/v1/placement/init` in favor of stable preferred URL `/api/v1/services/m3db/placement/init`
+
+### Misc
+- **M3Query**: Concept of data point limit enforcers removed in favor of the other remaining query limits (e.g. max series). This also removed metrics `cost_reporter_datapoints`, `cost_reporter_datapoints_counter`, and `cost_reporter_over_datapoints_limit`.
+
 # 0.15.17
 
 ## Features 

diff --git a/config/m3db/clustered-etcd/generated.yaml b/config/m3db/clustered-etcd/generated.yaml
@@ -1,7 +1,5 @@
 "coordinator":
-  "listenAddress":
-    "type": "config"
-    "value": "0.0.0.0:7201"
+  "listenAddress": "0.0.0.0:7201"
   "local":
     "namespaces":
     - "namespace": "default"
@@ -22,11 +20,6 @@
     "idScheme": "quoted"
 "db":
   "bootstrap":
-    "bootstrappers":
-    - "filesystem"
-    - "commitlog"
-    - "peers"
-    - "uninitialized_topology"
     "commitlog":
       "returnUnfulfilledForCorruptCommitLogFiles": false
   "cache":

diff --git a/config/m3db/clustered-etcd/m3dbnode.libsonnet b/config/m3db/clustered-etcd/m3dbnode.libsonnet
@@ -104,12 +104,6 @@ function(cluster, coordinator={}, db={}) {
     "writeNewSeriesLimitPerSecond": 1048576,
     "writeNewSeriesBackoffDuration": "2ms",
     "bootstrap": {
-      "bootstrappers": [
-        "filesystem",
-        "commitlog",
-        "peers",
-        "uninitialized_topology"
-      ],
       "commitlog": {
         "returnUnfulfilledForCorruptCommitLogFiles": false
       }

diff --git a/config/m3db/local-etcd/generated.yaml b/config/m3db/local-etcd/generated.yaml
@@ -1,9 +1,5 @@
 "coordinator":
-  "limits":
-    "maxComputedDatapoints": 10000
-  "listenAddress":
-    "type": "config"
-    "value": "0.0.0.0:7201"
+  "listenAddress": "0.0.0.0:7201"
   "local":
     "namespaces":
     - "namespace": "default"
@@ -24,11 +20,6 @@
     "idScheme": "quoted"
 "db":
   "bootstrap":
-    "bootstrappers":
-    - "filesystem"
-    - "commitlog"
-    - "peers"
-    - "uninitialized_topology"
     "commitlog":
       "returnUnfulfilledForCorruptCommitLogFiles": false
   "cache":

diff --git a/config/m3db/local-etcd/m3dbnode.libsonnet b/config/m3db/local-etcd/m3dbnode.libsonnet
@@ -28,9 +28,6 @@ function(coordinator={}, db={}) {
       "samplingRate": 1.0,
       "extended": "none"
     },
-    "limits": {
-      "maxComputedDatapoints": 10000
-    },
     "tagOptions": {
       // Configuration setting for generating metric IDs from tags.
       "idScheme": "quoted"
@@ -66,12 +63,6 @@ function(coordinator={}, db={}) {
     "writeNewSeriesLimitPerSecond": 1048576,
     "writeNewSeriesBackoffDuration": "2ms",
     "bootstrap": {
-      "bootstrappers": [
-        "filesystem",
-        "commitlog",
-        "peers",
-        "uninitialized_topology"
-      ],
       "commitlog": {
         "returnUnfulfilledForCorruptCommitLogFiles": false
       }

diff --git a/kube/bundle.yaml b/kube/bundle.yaml
diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml
@@ -6,9 +6,7 @@ metadata:
 data:
   m3dbnode.yml: |+
     coordinator:
-      listenAddress:
-        type: "config"
-        value: "0.0.0.0:7201"
+      listenAddress: 0.0.0.0:7201
       local:
         namespaces:
             - namespace: default
@@ -59,11 +57,6 @@ data:
       writeNewSeriesBackoffDuration: 2ms
 
       bootstrap:
-        bootstrappers:
-            - filesystem
-            - commitlog
-            - peers
-            - uninitialized_topology
         commitlog:
           returnUnfulfilledForCorruptCommitLogFiles: false
 
@@ -74,7 +67,7 @@ data:
           calculationType: fixed
           size: 2097152
 
-      fs:
+      filesystem:
         filePathPrefix: /var/lib/m3db
 
       config:

diff --git a/kube/terraform/main.tf b/kube/terraform/main.tf
@@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" {
     namespace = "m3db"
   }
   data {
-    m3dbnode.yml = "coordinator:\n  listenAddress:\n    type: \"config\"\n    value: \"0.0.0.0:7201\"\n  local:\n    namespaces:\n        - namespace: default\n          type: unaggregated\n          retention: 48h\n  metrics:\n    scope:\n      prefix: \"coordinator\"\n    prometheus:\n      handlerPath: /metrics\n      listenAddress: 0.0.0.0:7203\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: none\n  tagOptions:\n    idScheme: quoted\n\ndb:\n  logging:\n    level: info\n\n  metrics:\n    prometheus:\n      handlerPath: /metrics\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: detailed\n\n  listenAddress: 0.0.0.0:9000\n  clusterListenAddress: 0.0.0.0:9001\n  httpNodeListenAddress: 0.0.0.0:9002\n  httpClusterListenAddress: 0.0.0.0:9003\n  debugListenAddress: 0.0.0.0:9004\n\n  hostID:\n    resolver: hostname\n\n  client:\n    writeConsistencyLevel: majority\n    readConsistencyLevel: unstrict_majority\n\n  gcPercentage: 100\n\n  writeNewSeriesAsync: true\n  writeNewSeriesLimitPerSecond: 1048576\n  writeNewSeriesBackoffDuration: 2ms\n\n  bootstrap:\n    bootstrappers:\n        - filesystem\n        - commitlog\n        - peers\n        - uninitialized_topology\n    fs:\n      numProcessorsPerCPU: 0.125\n    commitlog:\n      returnUnfulfilledForCorruptCommitLogFiles: false\n\n  commitlog:\n    flushMaxBytes: 524288\n    flushEvery: 1s\n    queue:\n      calculationType: fixed\n      size: 2097152\n\n  fs:\n    filePathPrefix: /var/lib/m3db\n\n  config:\n    service:\n        env: default_env\n        zone: embedded\n        service: m3db\n        cacheDir: /var/lib/m3kv\n        etcdClusters:\n        - zone: embedded\n          endpoints:\n          - http://etcd-0.etcd:2379\n          - http://etcd-1.etcd:2379\n          - http://etcd-2.etcd:2379\n"
+    m3dbnode.yml = "coordinator:\n  listenAddress: \"0.0.0.0:7201\"\n  local:\n    namespaces:\n        - namespace: default\n          type: unaggregated\n          retention: 48h\n  metrics:\n    scope:\n      prefix: \"coordinator\"\n    prometheus:\n      handlerPath: /metrics\n      listenAddress: 0.0.0.0:7203\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: none\n  tagOptions:\n    idScheme: quoted\n\ndb:\n  logging:\n    level: info\n\n  metrics:\n    prometheus:\n      handlerPath: /metrics\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: detailed\n\n  listenAddress: 0.0.0.0:9000\n  clusterListenAddress: 0.0.0.0:9001\n  httpNodeListenAddress: 0.0.0.0:9002\n  httpClusterListenAddress: 0.0.0.0:9003\n  debugListenAddress: 0.0.0.0:9004\n\n  hostID:\n    resolver: hostname\n\n  client:\n    writeConsistencyLevel: majority\n    readConsistencyLevel: unstrict_majority\n\n  gcPercentage: 100\n\n  writeNewSeriesAsync: true\n  writeNewSeriesLimitPerSecond: 1048576\n  writeNewSeriesBackoffDuration: 2ms\n\n  bootstrap:\n    filesystem:\n      numProcessorsPerCPU: 0.125\n    commitlog:\n      returnUnfulfilledForCorruptCommitLogFiles: false\n\n  commitlog:\n    flushMaxBytes: 524288\n    flushEvery: 1s\n    queue:\n      calculationType: fixed\n      size: 2097152\n\n  filesystem:\n    filePathPrefix: /var/lib/m3db\n\n  config:\n    service:\n        env: default_env\n        zone: embedded\n        service: m3db\n        cacheDir: /var/lib/m3kv\n        etcdClusters:\n        - zone: embedded\n          endpoints:\n          - http://etcd-0.etcd:2379\n          - http://etcd-1.etcd:2379\n          - http://etcd-2.etcd:2379\n"
   }
 }
 

diff --git a/scripts/comparator/m3query.yml b/scripts/comparator/m3query.yml
@@ -1,6 +1,4 @@
-listenAddress:
-  type: "config"
-  value: "0.0.0.0:7201"
+listenAddress: 0.0.0.0:7201
 
 backend: grpc
 

diff --git a/scripts/development/m3_stack/m3collector.yml b/scripts/development/m3_stack/m3collector.yml
@@ -1,6 +1,4 @@
-listenAddress:
-  type: config
-  value: 0.0.0.0:7206
+listenAddress: 0.0.0.0:7206
 
 metrics:
   scope:

diff --git a/scripts/development/m3_stack/m3coordinator-aggregator.yml b/scripts/development/m3_stack/m3coordinator-aggregator.yml
@@ -1,5 +1,4 @@
-listenAddress:
-  value: "0.0.0.0:7201"
+listenAddress: 0.0.0.0:7201
 
 logging:
   level: info

diff --git a/scripts/development/m3_stack/m3coordinator-standard.yml b/scripts/development/m3_stack/m3coordinator-standard.yml
@@ -1,5 +1,4 @@
-listenAddress:
-  value: "0.0.0.0:7201"
+listenAddress: 0.0.0.0:7201
 
 logging:
   level: info

diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml
@@ -39,11 +39,6 @@ db:
   writeNewSeriesBackoffDuration: 2ms
 
   bootstrap:
-    bootstrappers:
-        - filesystem
-        - peers
-        - commitlog
-        - uninitialized_topology
     commitlog:
       returnUnfulfilledForCorruptCommitLogFiles: false
 
@@ -60,7 +55,7 @@ db:
       calculationType: fixed
       size: 2097152
 
-  fs:
+  filesystem:
     filePathPrefix: /var/lib/m3db
 
   config:

diff --git a/scripts/development/m3_stack/start_m3.sh b/scripts/development/m3_stack/start_m3.sh
@@ -172,7 +172,7 @@ else
 fi
 
 echo "Initializing namespaces"
-curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{
+curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{
   "name": "metrics_0_30m",
   "options": {
     "bootstrapEnabled": true,
@@ -205,7 +205,7 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{
     }
   }
 }'
-curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{
+curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{
   "name": "metrics_30s_24h",
   "options": {
     "bootstrapEnabled": true,
@@ -244,8 +244,8 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{
 echo "Done initializing namespaces"
 
 echo "Validating namespace"
-[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.metrics_0_30m.indexOptions.enabled)" == true ]
-[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.metrics_30s_24h.indexOptions.enabled)" == true ]
+[ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_0_30m.indexOptions.enabled)" == true ]
+[ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_30s_24h.indexOptions.enabled)" == true ]
 echo "Done validating namespace"
 
 echo "Waiting for namespaces to be ready"
@@ -255,7 +255,7 @@ echo "Done waiting for namespaces to be ready"
 
 echo "Initializing topology"
 if [[ "$USE_MULTI_DB_NODES" = true ]] ; then
-    curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{
+    curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
         "num_shards": 64,
         "replication_factor": 3,
         "instances": [
@@ -289,7 +289,7 @@ if [[ "$USE_MULTI_DB_NODES" = true ]] ; then
         ]
     }'
 else
-    curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{
+    curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
         "num_shards": 64,
         "replication_factor": 1,
         "instances": [
@@ -307,12 +307,12 @@ else
 fi
 
 echo "Validating topology"
-[ "$(curl -sSf localhost:7201/api/v1/placement | jq .placement.instances.m3db_seed.id)" == '"m3db_seed"' ]
+[ "$(curl -sSf localhost:7201/api/v1/services/m3db/placement | jq .placement.instances.m3db_seed.id)" == '"m3db_seed"' ]
 echo "Done validating topology"
 
 echo "Waiting until shards are marked as available"
 ATTEMPTS=100 TIMEOUT=2 retry_with_backoff  \
-  '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | grep -c INITIALIZING)" -eq 0 ]'
+  '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | grep -c INITIALIZING)" -eq 0 ]'
 
 if [[ "$USE_AGGREGATOR" = true ]]; then
     echo "Initializing M3Coordinator topology"

diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml
@@ -1,5 +1,4 @@
-listenAddress:
-  value: "0.0.0.0:7202"
+listenAddress: 0.0.0.0:7202
 
 logging:
   level: info

diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
@@ -1,5 +1,4 @@
-listenAddress:
-  value: "0.0.0.0:7202"
+listenAddress: 0.0.0.0:7202
 
 logging:
   level: info

diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml
@@ -1,5 +1,4 @@
-listenAddress:
-  value: "0.0.0.0:7201"
+listenAddress: 0.0.0.0:7201
 
 logging:
   level: info

diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
@@ -1,5 +1,4 @@
-listenAddress:
-  value: "0.0.0.0:7201"
+listenAddress: 0.0.0.0:7201
 
 logging:
   level: info