From ec68e83b38a7834083a839558fbfaae8c7d61a38 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Mon, 9 Nov 2020 16:07:56 -0500 Subject: [PATCH] Refactor etcd config as discovery section with convenience types (#2843) --- kube/bundle.yaml | 25 +- kube/m3dbnode-configmap.yaml | 25 +- scripts/development/m3_stack/m3dbnode.yml | 29 ++- .../aggregator/m3coordinator.yml | 18 -- .../aggregator_legacy/m3coordinator.yml | 18 -- .../carbon/m3coordinator.yml | 20 -- .../cold_writes_simple/m3coordinator.yml | 20 -- .../m3coordinator.yml | 20 -- .../m3dbnode.yml | 52 +--- .../m3coordinator-cluster-a.yml | 20 -- .../m3coordinator-cluster-b.yml | 20 -- .../m3dbnode-cluster-a.yml | 77 ++---- .../m3dbnode-cluster-b.yml | 79 ++---- .../prometheus/m3coordinator.yml | 20 -- .../m3coordinator01.yml | 20 -- .../m3coordinator02.yml | 20 -- .../query_fanout/m3coordinator-cluster-a.yml | 20 -- .../query_fanout/m3coordinator-cluster-b.yml | 21 -- .../query_fanout/m3coordinator-cluster-c.yml | 20 -- .../repair/m3coordinator.yml | 20 -- .../repair/m3dbnode.yml | 76 +----- .../m3coordinator-cluster-a.yml | 5 - .../m3coordinator-cluster-b.yml | 20 -- .../m3dbnode-cluster-a.yml | 79 ++---- .../m3dbnode-cluster-b.yml | 75 +----- .../replication/m3coordinator-cluster-a.yml | 20 -- .../replication/m3coordinator-cluster-b.yml | 20 -- .../replication/m3dbnode-cluster-a.yml | 76 +----- .../replication/m3dbnode-cluster-b.yml | 76 +----- .../simple_v2_batch_apis/m3coordinator.yml | 20 -- src/cmd/services/m3dbnode/config/config.go | 15 +- .../services/m3dbnode/config/config_test.go | 185 +++++++------- .../services/m3dbnode/main/main_index_test.go | 26 +- src/cmd/services/m3dbnode/main/main_test.go | 80 +++--- .../m3query/config/testdata/config_test.yml | 32 --- .../resources/config/m3coordinator.yml | 20 -- .../harness/resources/config/m3dbnode.yml | 31 +-- src/dbnode/config/m3dbnode-all-config.yml | 41 +-- .../config/m3dbnode-cluster-template.yml | 60 ----- .../config/m3dbnode-local-etcd-proto.yml | 94 +------ src/dbnode/config/m3dbnode-local-etcd.yml | 16 +- src/dbnode/discovery/config.go | 235 ++++++++++++++++++ src/dbnode/discovery/config_test.go | 166 +++++++++++++ src/dbnode/server/server.go | 31 ++- .../config/m3coordinator-cluster-template.yml | 18 -- src/query/config/m3coordinator-local-etcd.yml | 20 -- src/query/config/m3query-dev-etcd.yml | 42 ---- src/query/config/m3query-local-etcd.yml | 41 --- src/query/server/query.go | 29 ++- 49 files changed, 803 insertions(+), 1410 deletions(-) create mode 100644 src/dbnode/discovery/config.go create mode 100644 src/dbnode/discovery/config_test.go diff --git a/kube/bundle.yaml b/kube/bundle.yaml index 8449fcf925..bc2bd61b36 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -175,18 +175,19 @@ data: filesystem: filePathPrefix: /var/lib/m3db - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - http://etcd-0.etcd:2379 - - http://etcd-1.etcd:2379 - - http://etcd-2.etcd:2379 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd-0.etcd:2379 + - http://etcd-1.etcd:2379 + - http://etcd-2.etcd:2379 --- # Headless service for the statefulset apiVersion: v1 diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index 6ccd3084e3..0536215c4a 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -65,15 +65,16 @@ data: filesystem: filePathPrefix: /var/lib/m3db - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - http://etcd-0.etcd:2379 - - http://etcd-1.etcd:2379 - - http://etcd-2.etcd:2379 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd-0.etcd:2379 + - http://etcd-1.etcd:2379 + - http://etcd-2.etcd:2379 diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml index 35a384a193..fb3db8d151 100644 --- a/scripts/development/m3_stack/m3dbnode.yml +++ b/scripts/development/m3_stack/m3dbnode.yml @@ -12,20 +12,25 @@ db: resolver: environment envVarName: M3DB_HOST_ID - config: + # Note: cannot use type: "m3db_single_node" since sometimes + # multiple DB nodes spawned using the m3_stack start script + # and as such the non-seed nodes need to point etcd not to + # localhost but to m3db_seed:2379 specifically. + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - m3db_seed:2379 - seedNodes: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - m3db_seed:2379 + seedNodes: initialCluster: - - hostID: m3db_seed - endpoint: http://m3db_seed:2380 + - hostID: m3db_seed + endpoint: http://m3db_seed:2380 # proto: # schemaFilePath: /etc/m3dbnode/schema.proto diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index 35a42c248f..8d3afe98b6 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -1,21 +1,5 @@ listenAddress: 0.0.0.0:7202 -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - carbon: ingester: listenAddress: "0.0.0.0:7204" @@ -47,8 +31,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority downsample: rules: diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index db8feefcd7..3649b73580 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -1,21 +1,5 @@ listenAddress: 0.0.0.0:7202 -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - carbon: ingester: listenAddress: "0.0.0.0:7204" @@ -47,8 +31,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority downsample: remoteAggregator: diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index ba69263bc6..a883cdbc0c 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: agg @@ -33,8 +18,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -56,6 +39,3 @@ carbon: policies: - resolution: 5s retention: 10h - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index 64ae45cdca..37ff9ff20f 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 1ad54297c1..46a9f4cdc7 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: agg @@ -41,8 +26,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority downsample: rules: @@ -77,6 +60,3 @@ downsample: bufferPastLimits: - resolution: 0s bufferPast: 90s - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index 23e7bdf9eb..11795536c2 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -18,56 +18,14 @@ coordinator: idScheme: quoted db: - logging: - level: info - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: + discovery: + type: m3db_cluster + m3dbCluster: env: foo-namespace/foo-cluster zone: bar-zone - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: bar-zone - endpoints: - - etcd01:2379 + endpoints: + - etcd01:2379 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml index 1a165ab6b9..9435c4aaf6 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml index 0f434a0865..7f567e157d 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index 124973365a..13aac06336 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -1,36 +1,9 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority config: services: - service: @@ -53,38 +26,18 @@ db: - cluster_b_dbnode01:2379 async: true - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_a_dbnode01:2379 - seedNodes: - initialCluster: - - hostID: cluster_a_m3db_local_1 - endpoint: http://cluster_a_dbnode01:2380 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_a_dbnode01:2379 + seedNodes: + initialCluster: + - hostID: cluster_a_m3db_local_1 + endpoint: http://cluster_a_dbnode01:2380 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index e1c13baba6..9e59d22898 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -1,69 +1,20 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_b_dbnode01:2379 - seedNodes: - initialCluster: - - hostID: cluster_b_m3db_local_1 - endpoint: http://cluster_b_dbnode01:2380 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_b_dbnode01:2379 + seedNodes: + initialCluster: + - hostID: cluster_b_m3db_local_1 + endpoint: http://cluster_b_dbnode01:2380 diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index f9d46f31b3..92edee33fc 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,11 +22,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted query: restrictTags: diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml index 83d61c1b74..690e81eac6 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - writeForwarding: promRemoteWrite: targets: @@ -38,8 +23,3 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml index 8d15b050d8..b593362683 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: agg @@ -33,8 +18,3 @@ clusters: - zone: embedded endpoints: - dbnode02:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml index a3ef780416..85bf17a51d 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - # Fanout queries to remote clusters rpc: enabled: true @@ -43,8 +28,6 @@ clusters: - zone: embedded endpoints: - dbnode-cluster-a:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -55,9 +38,6 @@ carbon: - resolution: 5s retention: 10h -tagOptions: - idScheme: quoted - # Use tag consolidation here; other integration tests handle id consolidations. query: consolidation: diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml index 2464d98685..07be9bbc8e 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - # Fanout queries to remote clusters rpc: enabled: true @@ -43,8 +28,6 @@ clusters: - zone: embedded endpoints: - dbnode-cluster-b:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -54,10 +37,6 @@ carbon: policies: - resolution: 5s retention: 10h - -tagOptions: - idScheme: quoted - query: consolidation: matchType: tags \ No newline at end of file diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml index 034036b6ef..21e70608e7 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - # Fanout queries to remote clusters rpc: enabled: true @@ -43,8 +28,6 @@ clusters: - zone: embedded endpoints: - dbnode-cluster-c:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -55,9 +38,6 @@ carbon: - resolution: 5s retention: 10h -tagOptions: - idScheme: quoted - query: consolidation: matchType: tags \ No newline at end of file diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index 64ae45cdca..37ff9ff20f 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index eb307f1e81..29dcc22d4b 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -1,76 +1,26 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - dbnode01:2379 seedNodes: - initialCluster: - - hostID: m3db_local_1 - endpoint: http://dbnode01:2380 + initialCluster: + - hostID: m3db_local_1 + endpoint: http://dbnode01:2380 # Enable repairs. repair: enabled: true throttle: 1ms checkInterval: 1ms - diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml index 1a165ab6b9..fdfefaeb3a 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml @@ -37,8 +37,3 @@ clusters: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml index 0f434a0865..7f567e157d 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml index f2403c8890..3c3b91f4bf 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_a_dbnode01:2379 - seedNodes: - initialCluster: - - hostID: cluster_a_m3db_local_1 - endpoint: http://cluster_a_dbnode01:2380 + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_a_dbnode01:2379 + seedNodes: + initialCluster: + - hostID: cluster_a_m3db_local_1 + endpoint: http://cluster_a_dbnode01:2380 # Enable repairs (within cluster a). repair: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml index bcdd21cb21..920ab4db81 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_b_dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_b_dbnode01:2379 seedNodes: - initialCluster: - - hostID: cluster_b_m3db_local_1 - endpoint: http://cluster_b_dbnode01:2380 + initialCluster: + - hostID: cluster_b_m3db_local_1 + endpoint: http://cluster_b_dbnode01:2380 # Enable repairs (within cluster b). repair: diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml index 1a165ab6b9..9435c4aaf6 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml index 0f434a0865..7f567e157d 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,8 +22,3 @@ clusters: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index e89976d488..52e75125c5 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_a_dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_a_dbnode01:2379 seedNodes: - initialCluster: - - hostID: cluster_a_m3db_local_1 - endpoint: http://cluster_a_dbnode01:2380 + initialCluster: + - hostID: cluster_a_m3db_local_1 + endpoint: http://cluster_a_dbnode01:2380 # Disable repairs (within cluster a). repair: @@ -90,4 +41,3 @@ db: - zone: embedded endpoints: - cluster_b_dbnode01:2379 - diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index 195497e169..2d04a09719 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -1,72 +1,23 @@ db: - logging: - level: info - - tracing: - backend: jaeger - jaeger: - reporter: - localAgentHostPort: jaeger:6831 - sampler: - type: const - param: 1 - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: environment envVarName: M3DB_HOST_ID - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: + discovery: + config: service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - cluster_b_dbnode01:2379 + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - cluster_b_dbnode01:2379 seedNodes: - initialCluster: - - hostID: cluster_b_m3db_local_1 - endpoint: http://cluster_b_dbnode01:2380 + initialCluster: + - hostID: cluster_b_m3db_local_1 + endpoint: http://cluster_b_dbnode01:2380 # Disable repairs (within cluster b). repair: @@ -92,4 +43,3 @@ db: - zone: embedded endpoints: - cluster_a_dbnode01:2379 - diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 68f407d984..83e9cf4df4 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - limits: perQuery: maxFetchedSeries: 100 @@ -37,9 +22,4 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority useV2BatchAPIs: true - -tagOptions: - idScheme: quoted diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index 9f438cec6b..1ec7e0a165 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -31,6 +31,7 @@ import ( coordinatorcfg "github.com/m3db/m3/src/cmd/services/m3query/config" "github.com/m3db/m3/src/dbnode/client" + "github.com/m3db/m3/src/dbnode/discovery" "github.com/m3db/m3/src/dbnode/environment" "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/x/config/hostid" @@ -164,8 +165,8 @@ type DBConfiguration struct { // The pooling policy. PoolingPolicy *PoolingPolicy `yaml:"pooling"` - // The environment (static or dynamic) configuration. - EnvironmentConfig environment.Configuration `yaml:"config"` + // The discovery configuration. + DiscoveryConfig discovery.Configuration `yaml:"discovery"` // The configuration for hashing Hashing HashingConfiguration `yaml:"hashing"` @@ -587,12 +588,18 @@ func (c *ProtoConfiguration) Validate() error { // NewEtcdEmbedConfig creates a new embedded etcd config from kv config. func NewEtcdEmbedConfig(cfg DBConfiguration) (*embed.Config, error) { newKVCfg := embed.NewConfig() - kvCfg := cfg.EnvironmentConfig.SeedNodes hostID, err := cfg.HostID.Resolve() if err != nil { - return nil, err + return nil, fmt.Errorf("failed resolving hostID %w", err) } + + envCfg, err := cfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + return nil, fmt.Errorf("failed getting env config from discovery config %w", err) + } + + kvCfg := envCfg.SeedNodes newKVCfg.Name = hostID dir := kvCfg.RootDir diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 2db737a5df..5c98086e00 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -274,37 +274,38 @@ db: lowWatermark: 0.01 highWatermark: 0.02 - config: - service: - env: production - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 1.1.1.1:2379 - - 1.1.1.2:2379 - - 1.1.1.3:2379 - - seedNodes: - listenPeerUrls: - - http://0.0.0.0:2380 - listenClientUrls: - - http://0.0.0.0:2379 - rootDir: /var/lib/etcd - initialAdvertisePeerUrls: - - http://1.1.1.1:2380 - advertiseClientUrls: - - http://1.1.1.1:2379 - initialCluster: - - hostID: host1 - endpoint: http://1.1.1.1:2380 - clusterState: existing - - hostID: host2 - endpoint: http://1.1.1.2:2380 - - hostID: host3 - endpoint: http://1.1.1.3:2380 + discovery: + config: + service: + env: production + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - 1.1.1.1:2379 + - 1.1.1.2:2379 + - 1.1.1.3:2379 + + seedNodes: + listenPeerUrls: + - http://0.0.0.0:2380 + listenClientUrls: + - http://0.0.0.0:2379 + rootDir: /var/lib/etcd + initialAdvertisePeerUrls: + - http://1.1.1.1:2380 + advertiseClientUrls: + - http://1.1.1.1:2379 + initialCluster: + - hostID: host1 + endpoint: http://1.1.1.1:2380 + clusterState: existing + - hostID: host2 + endpoint: http://1.1.1.2:2380 + - hostID: host3 + endpoint: http://1.1.1.3:2380 hashing: seed: 42 writeNewSeriesAsync: true @@ -602,65 +603,69 @@ func TestConfiguration(t *testing.T) { size: 8 lowWatermark: 0 highWatermark: 0 - config: - services: - - async: false - clientOverrides: - hostQueueFlushInterval: null - targetHostQueueFlushSize: null - service: - zone: embedded - env: production - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 1.1.1.1:2379 - - 1.1.1.2:2379 - - 1.1.1.3:2379 - keepAlive: null - tls: null - autoSyncInterval: 0s - m3sd: - initTimeout: null - watchWithRevision: 0 - newDirectoryMode: null - statics: [] - seedNodes: - rootDir: /var/lib/etcd - initialAdvertisePeerUrls: - - http://1.1.1.1:2380 - advertiseClientUrls: - - http://1.1.1.1:2379 - listenPeerUrls: - - http://0.0.0.0:2380 - listenClientUrls: - - http://0.0.0.0:2379 - initialCluster: - - hostID: host1 - endpoint: http://1.1.1.1:2380 - clusterState: existing - - hostID: host2 - endpoint: http://1.1.1.2:2380 - clusterState: "" - - hostID: host3 - endpoint: http://1.1.1.3:2380 - clusterState: "" - clientTransportSecurity: - caFile: "" - certFile: "" - keyFile: "" - trustedCaFile: "" - clientCertAuth: false - autoTls: false - peerTransportSecurity: - caFile: "" - certFile: "" - keyFile: "" - trustedCaFile: "" - clientCertAuth: false - autoTls: false + discovery: + type: null + m3dbCluster: null + m3AggregatorCluster: null + config: + services: + - async: false + clientOverrides: + hostQueueFlushInterval: null + targetHostQueueFlushSize: null + service: + zone: embedded + env: production + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - 1.1.1.1:2379 + - 1.1.1.2:2379 + - 1.1.1.3:2379 + keepAlive: null + tls: null + autoSyncInterval: 0s + m3sd: + initTimeout: null + watchWithRevision: 0 + newDirectoryMode: null + statics: [] + seedNodes: + rootDir: /var/lib/etcd + initialAdvertisePeerUrls: + - http://1.1.1.1:2380 + advertiseClientUrls: + - http://1.1.1.1:2379 + listenPeerUrls: + - http://0.0.0.0:2380 + listenClientUrls: + - http://0.0.0.0:2379 + initialCluster: + - hostID: host1 + endpoint: http://1.1.1.1:2380 + clusterState: existing + - hostID: host2 + endpoint: http://1.1.1.2:2380 + clusterState: "" + - hostID: host3 + endpoint: http://1.1.1.3:2380 + clusterState: "" + clientTransportSecurity: + caFile: "" + certFile: "" + keyFile: "" + trustedCaFile: "" + clientCertAuth: false + autoTls: false + peerTransportSecurity: + caFile: "" + certFile: "" + keyFile: "" + trustedCaFile: "" + clientCertAuth: false + autoTls: false hashing: seed: 42 writeNewSeriesAsync: true diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go index 276c01bfde..bd302eba66 100644 --- a/src/cmd/services/m3dbnode/main/main_index_test.go +++ b/src/cmd/services/m3dbnode/main/main_index_test.go @@ -111,7 +111,10 @@ func TestIndexEnabledServer(t *testing.T) { err = xconfig.LoadFile(&cfg, configFd.Name(), xconfig.Options{}) require.NoError(t, err) - syncCluster, err := cfg.DB.EnvironmentConfig.Services.SyncCluster() + envCfg, err := cfg.DB.DiscoveryConfig.EnvironmentConfig(hostID) + require.NoError(t, err) + + syncCluster, err := envCfg.Services.SyncCluster() require.NoError(t, err) configSvcClient, err := syncCluster.Service.NewClient(instrument.NewOptions(). SetLogger(zap.NewNop())) @@ -193,7 +196,7 @@ func TestIndexEnabledServer(t *testing.T) { // NB(r): Make sure client config points to the root config // service since we're going to instantiate the client configuration // just by itself. - cfg.DB.Client.EnvironmentConfig = &cfg.DB.EnvironmentConfig + cfg.DB.Client.EnvironmentConfig = &envCfg cli, err := cfg.DB.Client.NewClient(client.ConfigurationParameters{}) require.NoError(t, err) @@ -447,13 +450,14 @@ db: - capacity: 4096 size: 128 - config: - service: - env: {{.ServiceEnv}} - zone: {{.ServiceZone}} - service: {{.ServiceName}} - cacheDir: {{.ConfigServiceCacheDir}} - etcdClusters: - - zone: {{.ServiceZone}} - endpoints: {{.EtcdEndpoints}} + discovery: + config: + service: + env: {{.ServiceEnv}} + zone: {{.ServiceZone}} + service: {{.ServiceName}} + cacheDir: {{.ConfigServiceCacheDir}} + etcdClusters: + - zone: {{.ServiceZone}} + endpoints: {{.EtcdEndpoints}} ` diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go index 0bb417b2d7..ccfafbf771 100644 --- a/src/cmd/services/m3dbnode/main/main_test.go +++ b/src/cmd/services/m3dbnode/main/main_test.go @@ -103,7 +103,10 @@ func TestConfig(t *testing.T) { err = xconfig.LoadFile(&cfg, configFd.Name(), xconfig.Options{}) require.NoError(t, err) - syncCluster, err := cfg.DB.EnvironmentConfig.Services.SyncCluster() + envCfg, err := cfg.DB.DiscoveryConfig.EnvironmentConfig(hostID) + require.NoError(t, err) + + syncCluster, err := envCfg.Services.SyncCluster() require.NoError(t, err) configSvcClient, err := syncCluster.Service.NewClient(instrument.NewOptions(). SetLogger(zap.NewNop())) @@ -185,7 +188,7 @@ func TestConfig(t *testing.T) { // NB(r): Make sure client config points to the root config // service since we're going to instantiate the client configuration // just by itself. - cfg.DB.Client.EnvironmentConfig = &cfg.DB.EnvironmentConfig + cfg.DB.Client.EnvironmentConfig = &envCfg cli, err := cfg.DB.Client.NewClient(client.ConfigurationParameters{}) require.NoError(t, err) @@ -334,7 +337,10 @@ func TestEmbeddedConfig(t *testing.T) { err = xconfig.LoadFile(&cfg, configFd.Name(), xconfig.Options{}) require.NoError(t, err) - syncCluster, err := cfg.DB.EnvironmentConfig.Services.SyncCluster() + envCfg, err := cfg.DB.DiscoveryConfig.EnvironmentConfig(hostID) + require.NoError(t, err) + + syncCluster, err := envCfg.Services.SyncCluster() require.NoError(t, err) configSvcClient, err := syncCluster.Service.NewClient(instrument.NewOptions(). SetLogger(zap.NewNop())) @@ -395,7 +401,7 @@ func TestEmbeddedConfig(t *testing.T) { // NB(r): Make sure client config points to the root config // service since we're going to instantiate the client configuration // just by itself. - cfg.DB.Client.EnvironmentConfig = &cfg.DB.EnvironmentConfig + cfg.DB.Client.EnvironmentConfig = &envCfg cli, err := cfg.DB.Client.NewClient(client.ConfigurationParameters{}) require.NoError(t, err) @@ -613,40 +619,42 @@ db: ` kvConfigPortion = ` - config: - service: - env: {{.ServiceEnv}} - zone: {{.ServiceZone}} - service: {{.ServiceName}} - cacheDir: {{.ConfigServiceCacheDir}} - etcdClusters: - - zone: {{.ServiceZone}} - endpoints: {{.EtcdEndpoints}} + discovery: + config: + service: + env: {{.ServiceEnv}} + zone: {{.ServiceZone}} + service: {{.ServiceName}} + cacheDir: {{.ConfigServiceCacheDir}} + etcdClusters: + - zone: {{.ServiceZone}} + endpoints: {{.EtcdEndpoints}} ` embeddedKVConfigPortion = ` - config: - service: - env: {{.ServiceEnv}} - zone: {{.ServiceZone}} - service: {{.ServiceName}} - cacheDir: {{.ConfigServiceCacheDir}} - etcdClusters: - - zone: {{.ServiceZone}} - endpoints: - - {{.EtcdEndpoint}} - seedNodes: - rootDir: {{.EmbeddedKVDir}} - listenPeerUrls: - - {{.LPURL}} - listenClientUrls: - - {{.LCURL}} - initialAdvertisePeerUrls: - - {{.APURL}} - advertiseClientUrls: - - {{.ACURL}} - initialCluster: - - hostID: {{.InitialClusterHostID}} - endpoint: {{.InitialClusterEndpoint}} + discovery: + config: + service: + env: {{.ServiceEnv}} + zone: {{.ServiceZone}} + service: {{.ServiceName}} + cacheDir: {{.ConfigServiceCacheDir}} + etcdClusters: + - zone: {{.ServiceZone}} + endpoints: + - {{.EtcdEndpoint}} + seedNodes: + rootDir: {{.EmbeddedKVDir}} + listenPeerUrls: + - {{.LPURL}} + listenClientUrls: + - {{.LCURL}} + initialAdvertisePeerUrls: + - {{.APURL}} + advertiseClientUrls: + - {{.ACURL}} + initialCluster: + - hostID: {{.InitialClusterHostID}} + endpoint: {{.InitialClusterEndpoint}} ` ) diff --git a/src/cmd/services/m3query/config/testdata/config_test.yml b/src/cmd/services/m3query/config/testdata/config_test.yml index 68250c3f0d..dbe0cbb233 100644 --- a/src/cmd/services/m3query/config/testdata/config_test.yml +++ b/src/cmd/services/m3query/config/testdata/config_test.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: default @@ -33,23 +18,6 @@ clusters: initialCluster: - hostID: m3db_local endpoint: http://127.0.0.1:2380 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 limits: perQuery: diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml index 53764669f0..536499f031 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: aggregated @@ -33,8 +18,6 @@ clusters: - zone: embedded endpoints: - dbnode01:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority carbon: ingester: @@ -56,6 +39,3 @@ carbon: policies: - resolution: 5s retention: 10h - -tagOptions: - idScheme: quoted diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 193ec26aa4..28627c78f7 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -70,21 +70,22 @@ db: filesystem: filePathPrefix: /var/lib/m3db - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 - + discovery: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - 127.0.0.1:2379 + seedNodes: + initialCluster: + - hostID: m3db_local + endpoint: http://127.0.0.1:2380 + # un-comment the lines below to enable Jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ # for quick local setup (which this config will send data to). diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index 51d08e4812..da6caf8692 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -144,23 +144,24 @@ db: checkInterval: 1m # etcd configuration. - config: - service: - # KV environment, zone, and service from which to write/read KV data (placement - # and configuration). Leave these as the default values unless you know what - # you're doing. - env: default_env - zone: embedded - service: m3db - # Directory to store cached etcd data in. - cacheDir: /var/lib/m3kv - # Configuration to identify the etcd hosts this node should connect to. - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - # Should only be present if running an M3DB cluster with embedded etcd. - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 + discovery: + config: + service: + # KV environment, zone, and service from which to write/read KV data (placement + # and configuration). Leave these as the default values unless you know what + # you're doing. + env: default_env + zone: embedded + service: m3db + # Directory to store cached etcd data in. + cacheDir: /var/lib/m3kv + # Configuration to identify the etcd hosts this node should connect to. + etcdClusters: + - zone: embedded + endpoints: + - 127.0.0.1:2379 + # Should only be present if running an M3DB cluster with embedded etcd. + seedNodes: + initialCluster: + - hostID: m3db_local + endpoint: http://127.0.0.1:2380 diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index fe4de58051..4d6689a84c 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -1,40 +1,11 @@ coordinator: - listenAddress: 0.0.0.0:7201 - local: namespaces: - namespace: default type: unaggregated retention: 48h - logging: - level: info - - metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - - tagOptions: - # Configuration setting for generating metric IDs from tags. - idScheme: quoted - db: - logging: - level: info - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - hostID: resolver: hostname @@ -59,34 +30,3 @@ db: # endpoint: http://HOST2_STATIC_IP_ADDRESS:2380 # - hostID: host3 # endpoint: http://HOST3_STATIC_IP_ADDRESS:2380 - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index e76bb65f0e..3b49b3574f 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -1,100 +1,22 @@ coordinator: - listenAddress: 0.0.0.0:7201 - local: namespaces: - namespace: default type: unaggregated retention: 48h - logging: - level: info - - metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - - tagOptions: - # Configuration setting for generating metric IDs from tags. - idScheme: quoted - db: - logging: - level: info - - metrics: - prometheus: - handlerPath: /metrics - sanitization: prometheus - samplingRate: 1.0 - extended: detailed - - listenAddress: 0.0.0.0:9000 - clusterListenAddress: 0.0.0.0:9001 - httpNodeListenAddress: 0.0.0.0:9002 - httpClusterListenAddress: 0.0.0.0:9003 - debugListenAddress: 0.0.0.0:9004 - hostID: resolver: config value: m3db_local - client: - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - - gcPercentage: 100 - - writeNewSeriesAsync: true - writeNewSeriesBackoffDuration: 2ms - - cache: - series: - policy: lru - postingsList: - size: 262144 - - commitlog: - flushMaxBytes: 524288 - flushEvery: 1s - queue: - calculationType: fixed - size: 2097152 - - filesystem: - filePathPrefix: /var/lib/m3db - - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 + discovery: + type: m3db_single_node proto: - enabled: true - schema_registry: - # Need an entry for each configured namespace. - "default": - schemaFilePath: "/etc/m3dbnode/default_schema.proto" - messageName: "VehicleLocation" - - # un-comment the lines below to enable Jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ - # for quick local setup (which this config will send data to). - - # tracing: - # backend: jaeger + enabled: true + schema_registry: + # Need an entry for each configured namespace. + "default": + schemaFilePath: "/etc/m3dbnode/default_schema.proto" + messageName: "VehicleLocation" diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index b40dbaae7d..51019f02d4 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -10,20 +10,8 @@ db: resolver: config value: m3db_local - config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - 127.0.0.1:2379 - seedNodes: - initialCluster: - - hostID: m3db_local - endpoint: http://127.0.0.1:2380 + discovery: + type: m3db_single_node # un-comment the lines below to enable Jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ # for quick local setup (which this config will send data to). diff --git a/src/dbnode/discovery/config.go b/src/dbnode/discovery/config.go new file mode 100644 index 0000000000..8cb86318a1 --- /dev/null +++ b/src/dbnode/discovery/config.go @@ -0,0 +1,235 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package discovery provides discovery configuration. +package discovery + +import ( + "fmt" + + etcdclient "github.com/m3db/m3/src/cluster/client/etcd" + "github.com/m3db/m3/src/dbnode/environment" +) + +const ( + defaultEnvironment = "default_env" + defaultZone = "embedded" + defaultM3DBService = "m3db" + defaultM3AggregatorService = "m3aggregator" + defaultCacheDirectory = "/var/lib/m3kv" + defaultSingleNodeClusterEndpoint = "127.0.0.1:2379" + defaultSingleNodeClusterSeedEndpoint = "http://127.0.0.1:2380" +) + +var validDiscoveryConfigTypes = []ConfigurationType{ + ConfigType, + M3DBSingleNodeType, + M3DBClusterType, + M3AggregatorClusterType, +} + +// ConfigurationType defines the type of discovery configuration. +type ConfigurationType uint + +const ( + // ConfigType defines a generic definition for service discovery via etcd. + ConfigType ConfigurationType = iota + // M3DBSingleNodeType defines configuration for a single M3DB node via etcd. + M3DBSingleNodeType + // M3DBClusterType defines M3DB discovery via etcd. + M3DBClusterType + // M3AggregatorClusterType defines M3DB discovery via etcd. + M3AggregatorClusterType +) + +// UnmarshalYAML unmarshals an ConfigurationType into a valid type from string. +func (t *ConfigurationType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + + // If unspecified, use default mode. + if str == "" { + *t = ConfigType + + return nil + } + + for _, valid := range validDiscoveryConfigTypes { + if str == valid.String() { + *t = valid + + return nil + } + } + + return fmt.Errorf("invalid ConfigurationType '%s' valid types are: %s", + str, validDiscoveryConfigTypes) +} + +// String returns the discovery configuration type as a string. +func (t ConfigurationType) String() string { + switch t { + case ConfigType: + return "config" + case M3DBSingleNodeType: + return "m3db_single_node" + case M3DBClusterType: + return "m3db_cluster" + case M3AggregatorClusterType: + return "m3aggregator_cluster" + } + return "unknown" +} + +// Configuration defines how services are to be discovered. +type Configuration struct { + // Type defines the type of discovery configuration being used. + Type *ConfigurationType `yaml:"type"` + + // M3DBCluster defines M3DB discovery via etcd. + M3DBCluster *M3DBClusterDiscoveryConfiguration `yaml:"m3dbCluster"` + + // M3AggregatorCluster defines M3Aggregator discovery via etcd. + M3AggregatorCluster *M3AggregatorClusterDiscoveryConfiguration `yaml:"m3AggregatorCluster"` + + // Config defines a generic definition for service discovery via etcd. + Config *environment.Configuration `yaml:"config"` +} + +// M3DBClusterDiscoveryConfiguration defines discovery configuration for M3DB. +type M3DBClusterDiscoveryConfiguration struct { + Env string `yaml:"env" validate:"nonzero"` + Zone *string `yaml:"zone"` + Endpoints []string `yaml:"endpoints"` +} + +// M3AggregatorClusterDiscoveryConfiguration defines discovery configuration for M3Aggregator. +type M3AggregatorClusterDiscoveryConfiguration struct { + Env string `yaml:"env"` + Zone *string `yaml:"zone"` + Endpoints []string `yaml:"endpoints"` +} + +// EnvironmentConfig provides the environment configuration +// based on the type of discovery configuration set. +func (c *Configuration) EnvironmentConfig( + hostID string, +) (environment.Configuration, error) { + discoveryConfigType := ConfigType + if c.Type != nil { + discoveryConfigType = *c.Type + } + + switch discoveryConfigType { + case ConfigType: + return *c.Config, nil + case M3DBSingleNodeType: + return c.m3dbSingleNodeEnvConfig(hostID), nil + case M3DBClusterType: + return c.envConfig( + discoveryConfigType, + defaultM3DBService, + c.M3DBCluster.Zone, + c.M3DBCluster.Env, + c.M3DBCluster.Endpoints, + ) + case M3AggregatorClusterType: + return c.envConfig( + discoveryConfigType, + defaultM3AggregatorService, + c.M3AggregatorCluster.Zone, + c.M3AggregatorCluster.Env, + c.M3AggregatorCluster.Endpoints, + ) + } + + return environment.Configuration{}, fmt.Errorf("unrecognized discovery type: %d", c.Type) +} + +func (c *Configuration) m3dbSingleNodeEnvConfig( + hostID string, +) environment.Configuration { + return environment.Configuration{ + Services: []*environment.DynamicCluster{ + { + Service: &etcdclient.Configuration{ + Service: defaultM3DBService, + CacheDir: defaultCacheDirectory, + Zone: defaultZone, + Env: defaultEnvironment, + ETCDClusters: []etcdclient.ClusterConfig{ + { + Zone: defaultZone, + Endpoints: []string{defaultSingleNodeClusterEndpoint}, + }, + }, + }, + }, + }, + SeedNodes: &environment.SeedNodesConfig{ + InitialCluster: []environment.SeedNode{ + { + HostID: hostID, + Endpoint: defaultSingleNodeClusterSeedEndpoint, + }, + }, + }, + } +} + +func (c *Configuration) envConfig( + configType ConfigurationType, + service string, + zone *string, + env string, + endpoints []string, +) (environment.Configuration, error) { + if c == nil { + err := fmt.Errorf("discovery configuration required for type: %s", + configType.String()) + return environment.Configuration{}, err + } + + validZone := defaultZone + if zone != nil { + validZone = *zone + } + + return environment.Configuration{ + Services: []*environment.DynamicCluster{ + { + Service: &etcdclient.Configuration{ + Service: service, + CacheDir: defaultCacheDirectory, + Zone: validZone, + Env: env, + ETCDClusters: []etcdclient.ClusterConfig{ + { + Zone: validZone, + Endpoints: endpoints, + }, + }, + }, + }, + }, + }, nil +} diff --git a/src/dbnode/discovery/config_test.go b/src/dbnode/discovery/config_test.go new file mode 100644 index 0000000000..307769688c --- /dev/null +++ b/src/dbnode/discovery/config_test.go @@ -0,0 +1,166 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package discovery + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/m3db/m3/src/dbnode/environment" + "github.com/m3db/m3/src/x/config" +) + +func TestM3DBSingleNodeType(t *testing.T) { + in := ` +type: m3db_single_node +` + + hostID := "test_id" + envConfig := getEnvConfig(t, in, hostID) + + assert.Equal(t, 1, len(envConfig.Services)) + assert.Equal(t, 1, len(envConfig.SeedNodes.InitialCluster)) + + s := envConfig.Services[0].Service + assert.Equal(t, defaultM3DBService, s.Service) + assert.Equal(t, defaultEnvironment, s.Env) + assert.Equal(t, defaultZone, s.Zone) + assert.Equal(t, defaultCacheDirectory, s.CacheDir) + assert.Equal(t, 1, len(s.ETCDClusters)) + assert.Equal(t, defaultZone, s.ETCDClusters[0].Zone) + assert.Equal(t, 1, len(s.ETCDClusters[0].Endpoints)) + assert.Equal(t, defaultSingleNodeClusterEndpoint, s.ETCDClusters[0].Endpoints[0]) + + c := envConfig.SeedNodes.InitialCluster[0] + assert.Equal(t, defaultSingleNodeClusterSeedEndpoint, c.Endpoint) + assert.Equal(t, hostID, c.HostID) +} + +func TestM3DBClusterType(t *testing.T) { + in := ` +type: m3db_cluster +m3dbCluster: + env: a + zone: b + endpoints: + - end_1 + - end_2 +` + + envConfig := getEnvConfig(t, in, "") + validateClusterConfig(t, envConfig, defaultM3DBService) +} + +func TestM3AggregatorClusterType(t *testing.T) { + in := ` +type: m3aggregator_cluster +m3AggregatorCluster: + env: a + zone: b + endpoints: + - end_1 + - end_2 +` + + envConfig := getEnvConfig(t, in, "") + validateClusterConfig(t, envConfig, defaultM3AggregatorService) +} + +func TestConfigType(t *testing.T) { + in := ` +config: + service: + env: test_env + zone: test_zone + service: test_service + cacheDir: test/cache + etcdClusters: + - zone: test_zone_2 + endpoints: + - 127.0.0.1:2379 + seedNodes: + initialCluster: + - hostID: host_id + endpoint: http://127.0.0.1:2380 +` + + hostID := "test_id" + envConfig := getEnvConfig(t, in, hostID) + + assert.Equal(t, 1, len(envConfig.Services)) + assert.Equal(t, 1, len(envConfig.SeedNodes.InitialCluster)) + + s := envConfig.Services[0].Service + assert.Equal(t, "test_service", s.Service) + assert.Equal(t, "test_env", s.Env) + assert.Equal(t, "test_zone", s.Zone) + assert.Equal(t, "test/cache", s.CacheDir) + assert.Equal(t, 1, len(s.ETCDClusters)) + assert.Equal(t, "test_zone_2", s.ETCDClusters[0].Zone) + assert.Equal(t, 1, len(s.ETCDClusters[0].Endpoints)) + assert.Equal(t, "127.0.0.1:2379", s.ETCDClusters[0].Endpoints[0]) + + c := envConfig.SeedNodes.InitialCluster[0] + assert.Equal(t, "http://127.0.0.1:2380", c.Endpoint) + assert.Equal(t, "host_id", c.HostID) +} + +func getEnvConfig(t *testing.T, in string, hostID string) environment.Configuration { + fd, err := ioutil.TempFile("", "config.yaml") + assert.NoError(t, err) + defer func() { + assert.NoError(t, fd.Close()) + assert.NoError(t, os.Remove(fd.Name())) + }() + + _, err = fd.Write([]byte(in)) + assert.NoError(t, err) + + var cfg Configuration + err = config.LoadFile(&cfg, fd.Name(), config.Options{}) + assert.NoError(t, err) + + envConfig, err := cfg.EnvironmentConfig(hostID) + assert.NoError(t, err) + + return envConfig +} + +func validateClusterConfig(t *testing.T, + envConfig environment.Configuration, + expectedService string, +) { + assert.Equal(t, 1, len(envConfig.Services)) + assert.Nil(t, envConfig.SeedNodes) + s := envConfig.Services[0].Service + assert.Equal(t, expectedService, s.Service) + assert.Equal(t, "a", s.Env) + assert.Equal(t, "b", s.Zone) + assert.Equal(t, defaultCacheDirectory, s.CacheDir) + assert.Equal(t, 1, len(s.ETCDClusters)) + assert.Equal(t, "b", s.ETCDClusters[0].Zone) + assert.Equal(t, 2, len(s.ETCDClusters[0].Endpoints)) + assert.Equal(t, "end_1", s.ETCDClusters[0].Endpoints[0]) + assert.Equal(t, "end_2", s.ETCDClusters[0].Endpoints[1]) +} diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 114e836bb1..3e619835ca 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -268,17 +268,22 @@ func Run(runOpts RunOptions) { } // Presence of KV server config indicates embedded etcd cluster - if cfg.EnvironmentConfig.SeedNodes == nil { + envConfig, err := cfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + logger.Fatal("could not get env config from discovery config", zap.Error(err)) + } + + if envConfig.SeedNodes == nil { logger.Info("no seed nodes set, using dedicated etcd cluster") } else { // Default etcd client clusters if not set already - service, err := cfg.EnvironmentConfig.Services.SyncCluster() + service, err := envConfig.Services.SyncCluster() if err != nil { logger.Fatal("invalid cluster configuration", zap.Error(err)) } clusters := service.Service.ETCDClusters - seedNodes := cfg.EnvironmentConfig.SeedNodes.InitialCluster + seedNodes := envConfig.SeedNodes.InitialCluster if len(clusters) == 0 { endpoints, err := config.InitialClusterEndpoints(seedNodes) if err != nil { @@ -608,12 +613,12 @@ func Run(runOpts RunOptions) { opts = opts.SetPersistManager(pm) var ( - envCfg environment.ConfigureResults + envCfgResults environment.ConfigureResults ) - if len(cfg.EnvironmentConfig.Statics) == 0 { + if len(envConfig.Statics) == 0 { logger.Info("creating dynamic config service client with m3cluster") - envCfg, err = cfg.EnvironmentConfig.Configure(environment.ConfigurationParameters{ + envCfgResults, err = envConfig.Configure(environment.ConfigurationParameters{ InstrumentOpts: iopts, HashingSeed: cfg.Hashing.Seed, NewDirectoryMode: newDirectoryMode, @@ -625,7 +630,7 @@ func Run(runOpts RunOptions) { } else { logger.Info("creating static config service client with m3cluster") - envCfg, err = cfg.EnvironmentConfig.Configure(environment.ConfigurationParameters{ + envCfgResults, err = envConfig.Configure(environment.ConfigurationParameters{ InstrumentOpts: iopts, HostID: hostID, ForceColdWritesEnabled: runOpts.StorageOptions.ForceColdWritesEnabled, @@ -635,7 +640,7 @@ func Run(runOpts RunOptions) { } } - syncCfg, err := envCfg.SyncCluster() + syncCfg, err := envCfgResults.SyncCluster() if err != nil { logger.Fatal("invalid cluster config", zap.Error(err)) } @@ -704,11 +709,11 @@ func Run(runOpts RunOptions) { if err != nil { logger.Warn("could not create handler options for debug writer", zap.Error(err)) } else { - envCfg, err := cfg.EnvironmentConfig.Services.SyncCluster() - if err != nil || envCfg.Service == nil { + envCfgCluster, err := envConfig.Services.SyncCluster() + if err != nil || envCfgCluster.Service == nil { logger.Warn("could not get cluster config for debug writer", zap.Error(err), - zap.Bool("envCfgServiceIsNil", envCfg.Service == nil)) + zap.Bool("envCfgClusterServiceIsNil", envCfgCluster.Service == nil)) } else { debugWriter, err = xdebug.NewPlacementAndNamespaceZipWriterWithDefaultSources( cpuProfileDuration, @@ -718,8 +723,8 @@ func Run(runOpts RunOptions) { { ServiceName: handleroptions.M3DBServiceName, Defaults: []handleroptions.ServiceOptionsDefault{ - handleroptions.WithDefaultServiceEnvironment(envCfg.Service.Env), - handleroptions.WithDefaultServiceZone(envCfg.Service.Zone), + handleroptions.WithDefaultServiceEnvironment(envCfgCluster.Service.Env), + handleroptions.WithDefaultServiceZone(envCfgCluster.Service.Zone), }, }, }, diff --git a/src/query/config/m3coordinator-cluster-template.yml b/src/query/config/m3coordinator-cluster-template.yml index 79887d3388..bed8f8cae3 100644 --- a/src/query/config/m3coordinator-cluster-template.yml +++ b/src/query/config/m3coordinator-cluster-template.yml @@ -1,21 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - clusters: ## Fill-out the following and un-comment before using, and ## make sure indent by two spaces is applied. diff --git a/src/query/config/m3coordinator-local-etcd.yml b/src/query/config/m3coordinator-local-etcd.yml index 0dfefbd45f..1fb1940638 100644 --- a/src/query/config/m3coordinator-local-etcd.yml +++ b/src/query/config/m3coordinator-local-etcd.yml @@ -1,18 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: default @@ -29,8 +14,3 @@ clusters: - zone: embedded endpoints: - 127.0.0.1:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - -tagOptions: - idScheme: quoted diff --git a/src/query/config/m3query-dev-etcd.yml b/src/query/config/m3query-dev-etcd.yml index cdaf363ce9..90a62af0d9 100644 --- a/src/query/config/m3query-dev-etcd.yml +++ b/src/query/config/m3query-dev-etcd.yml @@ -1,21 +1,5 @@ # m3query configuration for local development setup. Mostly the same as m3query-local-etcd.yml, but using fewer # resources (threads primarily). - -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - clusters: - namespaces: - namespace: default @@ -32,23 +16,6 @@ clusters: - zone: embedded endpoints: - 127.0.0.1:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 readWorkerPoolPolicy: grow: false @@ -57,12 +24,3 @@ readWorkerPoolPolicy: writeWorkerPoolPolicy: grow: false size: 10 - -tagOptions: - idScheme: quoted - -# Uncomment this to enable local jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ -# for quick local setup (which this config will send data to). - -# tracing: -# backend: jaeger diff --git a/src/query/config/m3query-local-etcd.yml b/src/query/config/m3query-local-etcd.yml index f63b801a41..1fb1940638 100644 --- a/src/query/config/m3query-local-etcd.yml +++ b/src/query/config/m3query-local-etcd.yml @@ -1,21 +1,3 @@ -listenAddress: 0.0.0.0:7201 - -logging: - level: info - -metrics: - scope: - prefix: "coordinator" - prometheus: - handlerPath: /metrics - listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved - sanitization: prometheus - samplingRate: 1.0 - extended: none - -tagOptions: - idScheme: quoted - clusters: - namespaces: - namespace: default @@ -32,26 +14,3 @@ clusters: - zone: embedded endpoints: - 127.0.0.1:2379 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 - -# Uncomment this to enable local jaeger tracing. See https://www.jaegertracing.io/docs/1.9/getting-started/ -# for quick local setup (which this config will send data to). - -# tracing: -# backend: jaeger diff --git a/src/query/server/query.go b/src/query/server/query.go index c67327c669..0d44c0bbc4 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -436,7 +436,19 @@ func Run(runOpts RunOptions) { var serviceOptionDefaults []handleroptions.ServiceOptionsDefault if dbCfg := runOpts.DBConfig; dbCfg != nil { - cluster, err := dbCfg.EnvironmentConfig.Services.SyncCluster() + hostID, err := dbCfg.HostID.Resolve() + if err != nil { + logger.Fatal("could not resolve hostID", + zap.Error(err)) + } + + envCfg, err := dbCfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + logger.Fatal("could not get env config from discovery config", + zap.Error(err)) + } + + cluster, err := envCfg.Services.SyncCluster() if err != nil { logger.Fatal("could not resolve embedded db cluster info", zap.Error(err)) @@ -818,7 +830,20 @@ func initClusters( if dbCfg == nil { return nil, nil, nil, errors.New("environment config required when dynamically fetching namespaces") } - clusterStaticConfig.Client = client.Configuration{EnvironmentConfig: &dbCfg.EnvironmentConfig} + + hostID, err := dbCfg.HostID.Resolve() + if err != nil { + logger.Fatal("could not resolve hostID", + zap.Error(err)) + } + + envCfg, err := dbCfg.DiscoveryConfig.EnvironmentConfig(hostID) + if err != nil { + logger.Fatal("could not get env config from discovery config", + zap.Error(err)) + } + + clusterStaticConfig.Client = client.Configuration{EnvironmentConfig: &envCfg} } clustersCfg := m3.ClustersStaticConfiguration{clusterStaticConfig}