From d86e3ef2bb3419ac45094257b7e9dc7aeffaf004 Mon Sep 17 00:00:00 2001 From: Ryan Allen Date: Thu, 29 Oct 2020 07:50:32 -0400 Subject: [PATCH] [config] Remove deprecated configuration fields (#2771) --- CHANGELOG.md | 34 ++ config/m3db/clustered-etcd/generated.yaml | 9 +- config/m3db/clustered-etcd/m3dbnode.libsonnet | 6 - config/m3db/local-etcd/generated.yaml | 11 +- config/m3db/local-etcd/m3dbnode.libsonnet | 9 - kube/bundle.yaml | 11 +- kube/m3dbnode-configmap.yaml | 11 +- kube/terraform/main.tf | 2 +- scripts/comparator/m3query.yml | 4 +- scripts/development/m3_stack/m3collector.yml | 4 +- .../m3_stack/m3coordinator-aggregator.yml | 3 +- .../m3_stack/m3coordinator-standard.yml | 3 +- scripts/development/m3_stack/m3dbnode.yml | 7 +- scripts/development/m3_stack/start_m3.sh | 16 +- .../aggregator/m3coordinator.yml | 3 +- .../aggregator_legacy/m3coordinator.yml | 3 +- .../carbon/m3coordinator.yml | 3 +- .../cold_writes_simple/m3coordinator.yml | 3 +- scripts/docker-integration-tests/common.sh | 20 +- .../m3coordinator.yml | 3 +- .../coordinator_noop/m3coordinator.yml | 3 +- .../m3dbnode.yml | 10 +- .../m3coordinator-cluster-a.yml | 3 +- .../m3coordinator-cluster-b.yml | 3 +- .../m3dbnode-cluster-a.yml | 7 +- .../m3dbnode-cluster-b.yml | 7 +- .../prometheus/m3coordinator.yml | 3 +- .../m3coordinator01.yml | 3 +- .../m3coordinator02.yml | 3 +- .../query_fanout/m3coordinator-cluster-a.yml | 3 +- .../query_fanout/m3coordinator-cluster-b.yml | 3 +- .../query_fanout/m3coordinator-cluster-c.yml | 3 +- .../repair/m3coordinator.yml | 3 +- .../repair/m3dbnode.yml | 7 +- .../m3coordinator-cluster-a.yml | 3 +- .../m3coordinator-cluster-b.yml | 3 +- .../m3dbnode-cluster-a.yml | 7 +- .../m3dbnode-cluster-b.yml | 7 +- .../replication/m3coordinator-cluster-a.yml | 3 +- .../replication/m3coordinator-cluster-b.yml | 3 +- .../replication/m3dbnode-cluster-a.yml | 7 +- .../replication/m3dbnode-cluster-b.yml | 7 +- .../docker-integration-tests/simple/test.sh | 18 +- .../simple_v2_batch_apis/m3coordinator.yml | 3 +- scripts/m3dbnode_bootstrapped.sh | 4 +- .../manifests/m3coordinator-two.yaml | 4 +- site/content/docs/how_to/kubernetes.md | 6 +- site/content/docs/how_to/prometheus.md | 16 +- site/content/docs/how_to/query.md | 4 +- site/content/docs/integrations/prometheus.md | 4 +- .../docs/m3query/config/annotated_config.yaml | 4 +- .../operational_guide/fileset_migrations.md | 2 +- .../namespace_configuration.md | 8 +- .../placement_configuration.md | 2 +- src/cmd/services/m3collector/config/config.go | 3 +- src/cmd/services/m3dbnode/config/bootstrap.go | 228 +++++++------- .../m3dbnode/config/bootstrap_test.go | 81 ----- src/cmd/services/m3dbnode/config/config.go | 6 +- .../services/m3dbnode/config/config_mock.go | 129 -------- .../services/m3dbnode/config/config_test.go | 50 +-- src/cmd/services/m3dbnode/main/common_test.go | 2 +- .../services/m3dbnode/main/main_index_test.go | 9 +- src/cmd/services/m3dbnode/main/main_test.go | 9 +- src/cmd/services/m3query/config/config.go | 107 +------ .../services/m3query/config/config_test.go | 122 +------- .../m3query/config/testdata/config.yml | 11 +- .../m3query/config/testdata/config_test.yml | 9 +- .../resources/config/m3coordinator.yml | 3 +- .../harness/resources/config/m3dbnode.yml | 13 +- src/cmd/tools/m3ctl/namespaces/types.go | 2 +- src/collector/config/m3collector.yml | 4 +- src/collector/server/server.go | 11 +- src/dbnode/config/m3dbnode-all-config.yml | 16 +- .../config/m3dbnode-cluster-template.yml | 10 +- .../config/m3dbnode-local-etcd-proto.yml | 13 +- src/dbnode/config/m3dbnode-local-etcd.yml | 13 +- src/dbnode/generated/mocks/generate.go | 1 - src/dbnode/namespace/options.go | 3 +- src/dbnode/persist/fs/retriever_test.go | 3 +- src/dbnode/server/server.go | 93 +----- src/dbnode/topology/consistency_level.go | 4 +- src/query/api/v1/handler/database/common.go | 14 - .../database/config_bootstrappers_get.go | 92 ------ .../database/config_bootstrappers_get_test.go | 105 ------- .../database/config_bootstrappers_set.go | 113 ------- .../database/config_bootstrappers_set_test.go | 127 -------- src/query/api/v1/handler/graphite/render.go | 2 +- src/query/api/v1/handler/namespace/add.go | 4 - src/query/api/v1/handler/namespace/common.go | 3 - src/query/api/v1/handler/namespace/delete.go | 4 - src/query/api/v1/handler/namespace/get.go | 4 - src/query/api/v1/handler/placement/add.go | 4 - src/query/api/v1/handler/placement/common.go | 47 +-- src/query/api/v1/handler/placement/delete.go | 4 - .../api/v1/handler/placement/delete_all.go | 4 - src/query/api/v1/handler/placement/get.go | 4 - src/query/api/v1/handler/placement/init.go | 4 - src/query/api/v1/handler/prom/read_test.go | 1 - .../v1/handler/prometheus/native/common.go | 50 --- .../api/v1/handler/prometheus/native/read.go | 9 - .../handler/prometheus/native/read_common.go | 26 -- .../v1/handler/prometheus/native/read_test.go | 189 ----------- .../v1/handler/prometheus/remote/read_test.go | 9 +- src/query/api/v1/httpd/handler_test.go | 13 +- src/query/api/v1/options/handler.go | 19 -- src/query/block/accounted.go | 49 --- src/query/block/accounted_test.go | 46 --- src/query/block/column.go | 23 +- src/query/block/column_test.go | 3 +- .../config/m3coordinator-cluster-template.yml | 3 +- src/query/config/m3coordinator-local-etcd.yml | 3 +- src/query/config/m3query-dev-etcd.yml | 3 +- src/query/config/m3query-dev-remote.yml | 4 +- src/query/config/m3query-local-etcd.yml | 3 +- src/query/cost/cost.go | 208 ------------ src/query/cost/cost_mock.go | 236 -------------- src/query/cost/cost_prop_test.go | 77 ----- src/query/cost/cost_test.go | 242 -------------- src/query/executor/engine.go | 9 +- src/query/executor/engine_test.go | 14 +- src/query/executor/options.go | 12 - src/query/executor/types.go | 6 - src/query/functions/fetch_test.go | 3 +- src/query/functions/temporal/base_test.go | 6 +- src/query/generated/assets/openapi/assets.go | 77 +++-- src/query/generated/assets/openapi/spec.yml | 61 ---- src/query/generated/mocks/generate.go | 1 - src/query/graphite/storage/m3_wrapper.go | 11 - src/query/graphite/storage/m3_wrapper_test.go | 11 +- src/query/models/block_type.go | 2 - src/query/models/block_type_test.go | 2 - src/query/models/config.go | 5 +- src/query/models/config_test.go | 5 +- src/query/models/options.go | 2 +- src/query/models/options_test.go | 4 +- src/query/models/params.go | 12 - src/query/models/query_context.go | 23 +- src/query/models/tags_id_schemes.go | 16 - src/query/models/tags_test.go | 18 -- src/query/models/types.go | 10 - src/query/remote/client.go | 5 +- src/query/server/cost_reporters.go | 255 --------------- src/query/server/cost_reporters_test.go | 295 ------------------ src/query/server/query.go | 23 +- src/query/server/query_test.go | 137 +------- src/query/storage/fetch.go | 4 +- src/query/storage/m3/config.go | 20 +- src/query/storage/m3/storage.go | 7 - src/query/storage/m3/storage_test.go | 6 +- src/query/storage/prom_converter.go | 26 +- src/query/storage/prom_converter_test.go | 10 +- src/query/storage/types.go | 4 - src/x/config/config.go | 3 +- src/x/config/config_test.go | 37 +++ src/x/config/listenaddress/listenaddress.go | 126 -------- .../listenaddress/listenaddress_test.go | 149 --------- src/x/cost/enforcer.go | 176 ----------- src/x/cost/enforcer_test.go | 248 --------------- src/x/cost/limit_manager.go | 177 ----------- src/x/cost/limit_manager_test.go | 127 -------- src/x/cost/options.go | 167 ---------- src/x/cost/test/assert.go | 45 --- src/x/cost/tracker.go | 40 --- src/x/cost/tracker_test.go | 111 ------- src/x/cost/types.go | 85 ----- 165 files changed, 459 insertions(+), 5181 deletions(-) delete mode 100644 src/cmd/services/m3dbnode/config/bootstrap_test.go delete mode 100644 src/cmd/services/m3dbnode/config/config_mock.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_get.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_get_test.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_set.go delete mode 100644 src/query/api/v1/handler/database/config_bootstrappers_set_test.go delete mode 100644 src/query/block/accounted.go delete mode 100644 src/query/block/accounted_test.go delete mode 100644 src/query/cost/cost.go delete mode 100644 src/query/cost/cost_mock.go delete mode 100644 src/query/cost/cost_prop_test.go delete mode 100644 src/query/cost/cost_test.go delete mode 100644 src/query/server/cost_reporters.go delete mode 100644 src/query/server/cost_reporters_test.go delete mode 100644 src/x/config/listenaddress/listenaddress.go delete mode 100644 src/x/config/listenaddress/listenaddress_test.go delete mode 100644 src/x/cost/enforcer.go delete mode 100644 src/x/cost/enforcer_test.go delete mode 100644 src/x/cost/limit_manager.go delete mode 100644 src/x/cost/limit_manager_test.go delete mode 100644 src/x/cost/options.go delete mode 100644 src/x/cost/test/assert.go delete mode 100644 src/x/cost/tracker.go delete mode 100644 src/x/cost/tracker_test.go delete mode 100644 src/x/cost/types.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d66198616..3d5cf00b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +# 1.0.0 (PROVISIONAL - STILL WORK IN PROGRESS) + +## Backwards Incompatible Changes + +### Configuration +- **M3DB**: `db.bootstrap.bootstrappers` removed +- **M3Coordinator**: `cluster.namespaces.storageMetricsType` removed +- **M3Coordinator**: `tagOptions.tagOptions` no longer supports `legacy` type +- **M3Query**: `limits.perQuery.maxComputedDatapoints` removed +- **M3Query**: `limits.perQuery.maxFetchedDatapoints` removed +- **M3Query**: `limits.global.maxFetchedDatapoints` removed +- **M3Query**: `cache` removed +- **M3Query**: `listenAddress` changed to always be resolved as a string from config. Format changed from +``` +listenAddress: + config: "..." + value: "..." +``` +to +``` +listenAddress: "..." +``` + +### API +- **M3DB**: `/services/m3db/database/config/bootstrappers` dynamic bootstappers endpoint removed +- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace` in favor of stable preferred URL `/api/v1/services/m3db/namespace` +- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace/init` in favor of stable preferred URL `/api/v1/services/m3db/namespace/init` +- **M3Coordinator**: Removed deprecated URL `/api/v1/namespace/unagg` in favor of stable preferred URL `/api/v1/services/m3db/namespace/unagg` +- **M3Coordinator**: Removed deprecated URL `/api/v1/placement` in favor of stable preferred URL `/api/v1/services/m3db/placement` +- **M3Coordinator**: Removed deprecated URL `/api/v1/placement/init` in favor of stable preferred URL `/api/v1/services/m3db/placement/init` + +### Misc +- **M3Query**: Concept of data point limit enforcers removed in favor of the other remaining query limits (e.g. max series). This also removed metrics `cost_reporter_datapoints`, `cost_reporter_datapoints_counter`, and `cost_reporter_over_datapoints_limit`. + # 0.15.17 ## Features diff --git a/config/m3db/clustered-etcd/generated.yaml b/config/m3db/clustered-etcd/generated.yaml index ee47a4f7cd..646ee745a8 100644 --- a/config/m3db/clustered-etcd/generated.yaml +++ b/config/m3db/clustered-etcd/generated.yaml @@ -1,7 +1,5 @@ "coordinator": - "listenAddress": - "type": "config" - "value": "0.0.0.0:7201" + "listenAddress": "0.0.0.0:7201" "local": "namespaces": - "namespace": "default" @@ -22,11 +20,6 @@ "idScheme": "quoted" "db": "bootstrap": - "bootstrappers": - - "filesystem" - - "commitlog" - - "peers" - - "uninitialized_topology" "commitlog": "returnUnfulfilledForCorruptCommitLogFiles": false "cache": diff --git a/config/m3db/clustered-etcd/m3dbnode.libsonnet b/config/m3db/clustered-etcd/m3dbnode.libsonnet index 4e1833c33f..a3439439b5 100644 --- a/config/m3db/clustered-etcd/m3dbnode.libsonnet +++ b/config/m3db/clustered-etcd/m3dbnode.libsonnet @@ -104,12 +104,6 @@ function(cluster, coordinator={}, db={}) { "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", "bootstrap": { - "bootstrappers": [ - "filesystem", - "commitlog", - "peers", - "uninitialized_topology" - ], "commitlog": { "returnUnfulfilledForCorruptCommitLogFiles": false } diff --git a/config/m3db/local-etcd/generated.yaml b/config/m3db/local-etcd/generated.yaml index d82bacfc16..d98d832cee 100644 --- a/config/m3db/local-etcd/generated.yaml +++ b/config/m3db/local-etcd/generated.yaml @@ -1,9 +1,5 @@ "coordinator": - "limits": - "maxComputedDatapoints": 10000 - "listenAddress": - "type": "config" - "value": "0.0.0.0:7201" + "listenAddress": "0.0.0.0:7201" "local": "namespaces": - "namespace": "default" @@ -24,11 +20,6 @@ "idScheme": "quoted" "db": "bootstrap": - "bootstrappers": - - "filesystem" - - "commitlog" - - "peers" - - "uninitialized_topology" "commitlog": "returnUnfulfilledForCorruptCommitLogFiles": false "cache": diff --git a/config/m3db/local-etcd/m3dbnode.libsonnet b/config/m3db/local-etcd/m3dbnode.libsonnet index c99ced7d9f..a04ab6fb3a 100644 --- a/config/m3db/local-etcd/m3dbnode.libsonnet +++ b/config/m3db/local-etcd/m3dbnode.libsonnet @@ -28,9 +28,6 @@ function(coordinator={}, db={}) { "samplingRate": 1.0, "extended": "none" }, - "limits": { - "maxComputedDatapoints": 10000 - }, "tagOptions": { // Configuration setting for generating metric IDs from tags. "idScheme": "quoted" @@ -66,12 +63,6 @@ function(coordinator={}, db={}) { "writeNewSeriesLimitPerSecond": 1048576, "writeNewSeriesBackoffDuration": "2ms", "bootstrap": { - "bootstrappers": [ - "filesystem", - "commitlog", - "peers", - "uninitialized_topology" - ], "commitlog": { "returnUnfulfilledForCorruptCommitLogFiles": false } diff --git a/kube/bundle.yaml b/kube/bundle.yaml index c1a3d30f1e..348bf32c24 100644 --- a/kube/bundle.yaml +++ b/kube/bundle.yaml @@ -116,9 +116,7 @@ metadata: data: m3dbnode.yml: |+ coordinator: - listenAddress: - type: "config" - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: - namespace: default @@ -169,11 +167,6 @@ data: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -184,7 +177,7 @@ data: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/kube/m3dbnode-configmap.yaml b/kube/m3dbnode-configmap.yaml index 7706004656..4221fec07c 100644 --- a/kube/m3dbnode-configmap.yaml +++ b/kube/m3dbnode-configmap.yaml @@ -6,9 +6,7 @@ metadata: data: m3dbnode.yml: |+ coordinator: - listenAddress: - type: "config" - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: - namespace: default @@ -59,11 +57,6 @@ data: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -74,7 +67,7 @@ data: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/kube/terraform/main.tf b/kube/terraform/main.tf index 4003e3f637..324f91b853 100755 --- a/kube/terraform/main.tf +++ b/kube/terraform/main.tf @@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" { namespace = "m3db" } data { - m3dbnode.yml = "coordinator:\n listenAddress:\n type: \"config\"\n value: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n bootstrappers:\n - filesystem\n - commitlog\n - peers\n - uninitialized_topology\n fs:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n fs:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" + m3dbnode.yml = "coordinator:\n listenAddress: \"0.0.0.0:7201\"\n local:\n namespaces:\n - namespace: default\n type: unaggregated\n retention: 48h\n metrics:\n scope:\n prefix: \"coordinator\"\n prometheus:\n handlerPath: /metrics\n listenAddress: 0.0.0.0:7203\n sanitization: prometheus\n samplingRate: 1.0\n extended: none\n tagOptions:\n idScheme: quoted\n\ndb:\n logging:\n level: info\n\n metrics:\n prometheus:\n handlerPath: /metrics\n sanitization: prometheus\n samplingRate: 1.0\n extended: detailed\n\n listenAddress: 0.0.0.0:9000\n clusterListenAddress: 0.0.0.0:9001\n httpNodeListenAddress: 0.0.0.0:9002\n httpClusterListenAddress: 0.0.0.0:9003\n debugListenAddress: 0.0.0.0:9004\n\n hostID:\n resolver: hostname\n\n client:\n writeConsistencyLevel: majority\n readConsistencyLevel: unstrict_majority\n\n gcPercentage: 100\n\n writeNewSeriesAsync: true\n writeNewSeriesLimitPerSecond: 1048576\n writeNewSeriesBackoffDuration: 2ms\n\n bootstrap:\n filesystem:\n numProcessorsPerCPU: 0.125\n commitlog:\n returnUnfulfilledForCorruptCommitLogFiles: false\n\n commitlog:\n flushMaxBytes: 524288\n flushEvery: 1s\n queue:\n calculationType: fixed\n size: 2097152\n\n filesystem:\n filePathPrefix: /var/lib/m3db\n\n config:\n service:\n env: default_env\n zone: embedded\n service: m3db\n cacheDir: /var/lib/m3kv\n etcdClusters:\n - zone: embedded\n endpoints:\n - http://etcd-0.etcd:2379\n - http://etcd-1.etcd:2379\n - http://etcd-2.etcd:2379\n" } } diff --git a/scripts/comparator/m3query.yml b/scripts/comparator/m3query.yml index 2bd5f8a204..5e67225baa 100644 --- a/scripts/comparator/m3query.yml +++ b/scripts/comparator/m3query.yml @@ -1,6 +1,4 @@ -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 backend: grpc diff --git a/scripts/development/m3_stack/m3collector.yml b/scripts/development/m3_stack/m3collector.yml index ce940de31d..f3d4c15eca 100644 --- a/scripts/development/m3_stack/m3collector.yml +++ b/scripts/development/m3_stack/m3collector.yml @@ -1,6 +1,4 @@ -listenAddress: - type: config - value: 0.0.0.0:7206 +listenAddress: 0.0.0.0:7206 metrics: scope: diff --git a/scripts/development/m3_stack/m3coordinator-aggregator.yml b/scripts/development/m3_stack/m3coordinator-aggregator.yml index a9a4d40b32..b622668a4a 100644 --- a/scripts/development/m3_stack/m3coordinator-aggregator.yml +++ b/scripts/development/m3_stack/m3coordinator-aggregator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/development/m3_stack/m3coordinator-standard.yml b/scripts/development/m3_stack/m3coordinator-standard.yml index 348f16146c..16137c65b4 100644 --- a/scripts/development/m3_stack/m3coordinator-standard.yml +++ b/scripts/development/m3_stack/m3coordinator-standard.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/development/m3_stack/m3dbnode.yml b/scripts/development/m3_stack/m3dbnode.yml index 2b6ee4ac4a..54a55ba6f2 100644 --- a/scripts/development/m3_stack/m3dbnode.yml +++ b/scripts/development/m3_stack/m3dbnode.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - peers - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -60,7 +55,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/development/m3_stack/start_m3.sh b/scripts/development/m3_stack/start_m3.sh index f22aa8826a..88b6c24c3e 100755 --- a/scripts/development/m3_stack/start_m3.sh +++ b/scripts/development/m3_stack/start_m3.sh @@ -172,7 +172,7 @@ else fi echo "Initializing namespaces" -curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{ "name": "metrics_0_30m", "options": { "bootstrapEnabled": true, @@ -205,7 +205,7 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ } } }' -curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{ "name": "metrics_30s_24h", "options": { "bootstrapEnabled": true, @@ -244,8 +244,8 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{ echo "Done initializing namespaces" echo "Validating namespace" -[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.metrics_0_30m.indexOptions.enabled)" == true ] -[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.metrics_30s_24h.indexOptions.enabled)" == true ] +[ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_0_30m.indexOptions.enabled)" == true ] +[ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_30s_24h.indexOptions.enabled)" == true ] echo "Done validating namespace" echo "Waiting for namespaces to be ready" @@ -255,7 +255,7 @@ echo "Done waiting for namespaces to be ready" echo "Initializing topology" if [[ "$USE_MULTI_DB_NODES" = true ]] ; then - curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{ + curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 64, "replication_factor": 3, "instances": [ @@ -289,7 +289,7 @@ if [[ "$USE_MULTI_DB_NODES" = true ]] ; then ] }' else - curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{ + curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 64, "replication_factor": 1, "instances": [ @@ -307,12 +307,12 @@ else fi echo "Validating topology" -[ "$(curl -sSf localhost:7201/api/v1/placement | jq .placement.instances.m3db_seed.id)" == '"m3db_seed"' ] +[ "$(curl -sSf localhost:7201/api/v1/services/m3db/placement | jq .placement.instances.m3db_seed.id)" == '"m3db_seed"' ] echo "Done validating topology" echo "Waiting until shards are marked as available" ATTEMPTS=100 TIMEOUT=2 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | grep -c INITIALIZING)" -eq 0 ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | grep -c INITIALIZING)" -eq 0 ]' if [[ "$USE_AGGREGATOR" = true ]]; then echo "Initializing M3Coordinator topology" diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index 8c730ecfc0..35a42c248f 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7202" +listenAddress: 0.0.0.0:7202 logging: level: info diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index 475afe191f..db8feefcd7 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7202" +listenAddress: 0.0.0.0:7202 logging: level: info diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index ff075d84bc..ba69263bc6 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index 6c6ebd622c..64ae45cdca 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/common.sh b/scripts/docker-integration-tests/common.sh index df27a44042..ab54fd4c3a 100644 --- a/scripts/docker-integration-tests/common.sh +++ b/scripts/docker-integration-tests/common.sh @@ -53,7 +53,7 @@ function setup_single_m3db_node_long_namespaces { echo "Wait for API to be available" ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq ".namespaces | length")" == "0" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]' echo "Adding placement and agg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{ @@ -76,7 +76,7 @@ function setup_single_m3db_node_long_namespaces { echo "Wait until placement is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' wait_for_namespaces @@ -88,7 +88,7 @@ function setup_single_m3db_node_long_namespaces { echo "Wait until agg2d namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]' echo "Wait until bootstrapped" @@ -106,7 +106,7 @@ function setup_single_m3db_node { echo "Wait for API to be available" ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq ".namespaces | length")" == "0" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]' echo "Adding placement and agg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{ @@ -129,7 +129,7 @@ function setup_single_m3db_node { echo "Wait until placement is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]' wait_for_namespaces @@ -150,7 +150,7 @@ function setup_two_m3db_nodes { echo "Wait for API to be available" ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq ".namespaces | length")" == "0" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]' echo "Adding placement and agg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{ @@ -181,7 +181,7 @@ function setup_two_m3db_nodes { echo "Wait until placement is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]' wait_for_namespaces @@ -197,7 +197,7 @@ function wait_for_namespaces { echo "Wait until agg namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' echo "Adding unagg namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{ @@ -207,7 +207,7 @@ function wait_for_namespaces { echo "Wait until unagg namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' echo "Adding coldWritesRepairAndNoIndex namespace" curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{ @@ -233,6 +233,6 @@ function wait_for_namespaces { echo "Wait until coldWritesRepairAndNoIndex namespace is init'd" ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/namespace | jq .registry.namespaces.coldWritesRepairAndNoIndex.coldWritesEnabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.coldWritesRepairAndNoIndex.coldWritesEnabled)" == true ]' } diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 5cf1fa9db2..1ad54297c1 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml index e35b204202..c3c08c0104 100644 --- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml index 6c793f8330..a0689a4f27 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 logging: level: info @@ -50,11 +49,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -69,7 +63,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml index e31c37acdc..1a165ab6b9 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml index b13af81125..0f434a0865 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index f724192e96..0d1a4c4c67 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -60,11 +60,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -81,7 +76,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index fa32e79853..ba15ee5ebc 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -60,7 +55,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index 6dbb0970d5..f9d46f31b3 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml index 2a559916c2..83d61c1b74 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml index 9a2d14308f..8d15b050d8 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml index 39879fc86d..a3ef780416 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml index 4d1dbcfce0..2464d98685 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml index adbf3ca63d..034036b6ef 100644 --- a/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml +++ b/scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index 6c6ebd622c..64ae45cdca 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 6bdae716a7..ed221416b3 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -60,7 +55,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml index e31c37acdc..1a165ab6b9 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml index b13af81125..0f434a0865 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml index 002f7dd1cf..01613fb5f8 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -60,7 +55,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml index 59dcc55b6b..29edf3d35a 100644 --- a/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -60,7 +55,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml index e31c37acdc..1a165ab6b9 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml index b13af81125..0f434a0865 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index aca38c3af9..ee44d87669 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -60,7 +55,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index 558e511304..cdd084aa3d 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -39,11 +39,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test. - bootstrappers: - - filesystem - - commitlog - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -60,7 +55,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/scripts/docker-integration-tests/simple/test.sh b/scripts/docker-integration-tests/simple/test.sh index 118eab927e..d85be4fa92 100755 --- a/scripts/docker-integration-tests/simple/test.sh +++ b/scripts/docker-integration-tests/simple/test.sh @@ -37,7 +37,7 @@ ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \ 'curl -vvvsSf 0.0.0.0:7201/health' echo "Adding namespace" -curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/services/m3db/namespace -d '{ "name": "agg", "options": { "bootstrapEnabled": true, @@ -63,9 +63,9 @@ curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ echo "Sleep until namespace is init'd" ATTEMPTS=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]' -curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ +curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/services/m3db/namespace -d '{ "name": "unagg", "options": { "bootstrapEnabled": true, @@ -91,10 +91,10 @@ curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{ echo "Sleep until namespace is init'd" ATTEMPTS=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]' echo "Placement initialization" -curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/placement/init -d '{ +curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 4, "replication_factor": 1, "instances": [ @@ -112,7 +112,7 @@ curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/placement/init -d '{ echo "Sleep until placement is init'd" ATTEMPTS=4 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | jq .placement.instances.m3db_local.id)" == \"m3db_local\" ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | jq .placement.instances.m3db_local.id)" == \"m3db_local\" ]' echo "Sleep until bootstrapped" ATTEMPTS=7 TIMEOUT=2 retry_with_backoff \ @@ -120,7 +120,7 @@ ATTEMPTS=7 TIMEOUT=2 retry_with_backoff \ echo "Waiting until shards are marked as available" ATTEMPTS=10 TIMEOUT=1 retry_with_backoff \ - '[ "$(curl -sSf 0.0.0.0:7201/api/v1/placement | grep -c INITIALIZING)" -eq 0 ]' + '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | grep -c INITIALIZING)" -eq 0 ]' echo "Write data" curl -vvvsS -X POST 0.0.0.0:9003/writetagged -d '{ @@ -163,7 +163,7 @@ else fi echo "Deleting placement" -curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/placement +curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/services/m3db/placement echo "Deleting namespace" -curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/namespace/unagg +curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/services/m3db/namespace/unagg diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 1f9333bdec..68f407d984 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/scripts/m3dbnode_bootstrapped.sh b/scripts/m3dbnode_bootstrapped.sh index 85647ebde8..8a5e22657d 100755 --- a/scripts/m3dbnode_bootstrapped.sh +++ b/scripts/m3dbnode_bootstrapped.sh @@ -12,8 +12,8 @@ set -o pipefail COORDINATOR_PORT=${COORDINATOR_PORT:-7201} DBNODE_PORT=${DBNODE_PORT:-9002} -COORD_PLACEMENT_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/placement" -COORD_NAMESPACE_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/namespace" +COORD_PLACEMENT_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/services/m3db/placement" +COORD_NAMESPACE_ENDPOINT="http://localhost:${COORDINATOR_PORT}/api/v1/services/m3db/namespace" DBNODE_ENDPOINT="http://localhost:${DBNODE_PORT}/health" HOSTNAME=${HOSTNAME:-$(hostname)} diff --git a/scripts/vagrant/provision/manifests/m3coordinator-two.yaml b/scripts/vagrant/provision/manifests/m3coordinator-two.yaml index c837a2ab43..d12e18489e 100644 --- a/scripts/vagrant/provision/manifests/m3coordinator-two.yaml +++ b/scripts/vagrant/provision/manifests/m3coordinator-two.yaml @@ -7,9 +7,7 @@ metadata: name: m3coordinator-dedicated-test-cluster data: coordinator.yaml: |+ - listenAddress: - type: "config" - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 metrics: scope: prefix: "coordinator" diff --git a/site/content/docs/how_to/kubernetes.md b/site/content/docs/how_to/kubernetes.md index 863192a398..ed98cfe66f 100644 --- a/site/content/docs/how_to/kubernetes.md +++ b/site/content/docs/how_to/kubernetes.md @@ -95,7 +95,7 @@ Forwarding from [::1]:7201 -> 7201 ```shell # Create an initial cluster topology -curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{ +curl -sSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{ "num_shards": 1024, "replication_factor": 3, "instances": [ @@ -132,7 +132,7 @@ curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{ ```shell # Create a namespace to hold your metrics -curl -X POST localhost:7201/api/v1/namespace -d '{ +curl -X POST localhost:7201/api/v1/services/m3db/namespace -d '{ "name": "default", "options": { "bootstrapEnabled": true, @@ -262,7 +262,7 @@ Forwarding from [::1]:7201 -> 7201 ``` ```shell -curl -sSf -X POST localhost:7201/api/v1/placement -d '{ +curl -sSf -X POST localhost:7201/api/v1/services/m3db/placement -d '{ "instances": [ { "id": "m3dbnode-3", diff --git a/site/content/docs/how_to/prometheus.md b/site/content/docs/how_to/prometheus.md index 6e4853f96c..bc2e11cbb6 100644 --- a/site/content/docs/how_to/prometheus.md +++ b/site/content/docs/how_to/prometheus.md @@ -29,9 +29,8 @@ Start by downloading the config template. Update the namespaces and the client s You'll need to specify the static IPs or hostnames of your M3DB seed nodes, and the name and retention values of the namespace you set up. You can leave the namespace storage metrics type as unaggregated since it's required by default to have a cluster that receives all Prometheus metrics unaggregated. In the future you might also want to aggregate and downsample metrics for longer retention, and you can come back and update the config once you've setup those clusters. You can read more about our aggregation functionality here. It should look something like: -listenAddress: - type: "config" - value: "0.0.0.0:7201" +```yaml +listenAddress: 0.0.0.0:7201 logging: level: info @@ -86,32 +85,41 @@ clusters: jitter: true backgroundHealthCheckFailLimit: 4 backgroundHealthCheckFailThrottleFactor: 0.5 +``` Now start the process up: -m3coordinator -f +`m3coordinator -f ` Or, use the docker container: +``` docker pull quay.io/m3db/m3coordinator:latest docker run -p 7201:7201 --name m3coordinator -v :/etc/m3coordinator/m3coordinator.yml quay.io/m3db/m3coordinator:latest +``` ### Prometheus configuration Add to your Prometheus configuration the m3coordinator sidecar remote read/write endpoints, something like: +``` remote_read: - url: "http://localhost:7201/api/v1/prom/remote/read" # To test reading even when local Prometheus has the data read_recent: true remote_write: - url: "http://localhost:7201/api/v1/prom/remote/write" +``` Also, we recommend adding M3DB and M3Coordinator/M3Query to your list of jobs under scrape_configs so that you can monitor them using Prometheus. With this scraping setup, you can also use our pre-configured M3DB Grafana dashboard. +``` - job_name: 'm3db' static_configs: - targets: [':7203', ':7203', ':7203'] - job_name: 'm3coordinator' static_configs: - targets: [':7203'] +``` NOTE: If you are running M3DB with embedded M3Coordinator, you should only have one job. We recommend just calling this job m3. For example: +``` - job_name: 'm3' static_configs: - targets: [':7203'] +``` diff --git a/site/content/docs/how_to/query.md b/site/content/docs/how_to/query.md index 94a491ab4b..3f66a90bb2 100644 --- a/site/content/docs/how_to/query.md +++ b/site/content/docs/how_to/query.md @@ -58,7 +58,9 @@ If you run Statsite, m3agg, or some other aggregation tier, you will want to set ## ID generation -The default generation scheme for IDs, `legacy`, is unfortunately prone to collisions, but remains the default for backwards compatibility reasons. It is suggested to set the ID generation scheme to one of either `quoted` or `prepend_meta`. `quoted` generation scheme yields the most human-readable IDs, whereas `prepend_meta` is better for more compact IDs, or if tags are expected to contain non-ASCII characters. To set the ID generation scheme, add the following to your m3coordinator configuration yaml file: +As of 1.0, the default generation scheme for IDs is `quoted`. If you are using the deprecated `legacy` scheme, please contact the M3 project contributors on the open source Slack channel. + +The `quoted` generation scheme yields the most human-readable IDs, whereas `prepend_meta` is better for more compact IDs, or if tags are expected to contain non-ASCII characters. To set the ID generation scheme, add the following to your m3coordinator configuration yaml file: ```yaml tagOptions: diff --git a/site/content/docs/integrations/prometheus.md b/site/content/docs/integrations/prometheus.md index eb74b5dcee..512a3e9ae5 100644 --- a/site/content/docs/integrations/prometheus.md +++ b/site/content/docs/integrations/prometheus.md @@ -16,9 +16,7 @@ You'll need to specify the static IPs or hostnames of your M3DB seed nodes, and It should look something like: ```yaml -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/site/content/docs/m3query/config/annotated_config.yaml b/site/content/docs/m3query/config/annotated_config.yaml index 4c7080847e..3b5da983d2 100644 --- a/site/content/docs/m3query/config/annotated_config.yaml +++ b/site/content/docs/m3query/config/annotated_config.yaml @@ -1,6 +1,4 @@ -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/site/content/docs/operational_guide/fileset_migrations.md b/site/content/docs/operational_guide/fileset_migrations.md index b099725aff..95c40c7ba2 100644 --- a/site/content/docs/operational_guide/fileset_migrations.md +++ b/site/content/docs/operational_guide/fileset_migrations.md @@ -16,7 +16,7 @@ Migrations are enabled by setting the following fields in the M3 configuration ( ```yaml db: bootstrap: - fs: + filesystem: migration: targetMigrationVersion: "1.1" # Optional. Defaults to the number of available CPUs / 2. diff --git a/site/content/docs/operational_guide/namespace_configuration.md b/site/content/docs/operational_guide/namespace_configuration.md index 9f9f41202e..ac711d1f4f 100644 --- a/site/content/docs/operational_guide/namespace_configuration.md +++ b/site/content/docs/operational_guide/namespace_configuration.md @@ -42,7 +42,7 @@ The "advanced" API allows you to configure every aspect of the namespace that yo Adding a namespace is a simple as using the `POST` `api/v1/namespace` API on an M3Coordinator instance. ```shell -curl -X POST :/api/v1/namespace -d '{ +curl -X POST :/api/v1/services/m3db/namespace -d '{ "name": "default_unaggregated", "options": { "bootstrapEnabled": true, @@ -71,9 +71,9 @@ Adding a namespace does not require restarting M3DB, but will require modifying ### Deleting a Namespace -Deleting a namespace is a simple as using the `DELETE` `/api/v1/namespace` API on an M3Coordinator instance. +Deleting a namespace is a simple as using the `DELETE` `/api/v1/services/m3db/namespace` API on an M3Coordinator instance. -`curl -X DELETE :/api/v1/namespace/` +`curl -X DELETE :/api/v1/services/m3db/namespace/` Note that deleting a namespace will not have any effect on the M3DB nodes until they are all restarted. In addition, the namespace will need to be removed from the M3Coordinator configuration and then the M3Coordinator node will need to be restarted. @@ -85,7 +85,7 @@ Also, be very careful not to restart the M3DB nodes after deleting the namespace ### Viewing a Namespace -In order to view a namespace and its attributes, use the `GET` `/api/v1/namespace` API on a M3Coordinator instance. +In order to view a namespace and its attributes, use the `GET` `/api/v1/services/m3db/namespace` API on a M3Coordinator instance. Additionally, for readability/debugging purposes, you can add the `debug=true` parameter to the URL to view block sizes, buffer sizes, etc. in duration format as opposed to nanoseconds (default). diff --git a/site/content/docs/operational_guide/placement_configuration.md b/site/content/docs/operational_guide/placement_configuration.md index b56e829739..079f615dc0 100644 --- a/site/content/docs/operational_guide/placement_configuration.md +++ b/site/content/docs/operational_guide/placement_configuration.md @@ -265,7 +265,7 @@ If the placement for `M3DB` needs to be recreated, the `/api/v1/services/m3db/pl Please note, a placement already needs to exist to use this endpoint. If no placement exists, use the `Placement Initialization` endpoint described above. Also, as mentioned above, this endpoint creates an entirely new placement therefore complete placement information needs to be passed into the body of the request. The recommended way to this -is to get the existing placement using `/api/v1/placement` and modify that (as the `placement` field) along +is to get the existing placement using `/api/v1/services/m3db/placement` and modify that (as the `placement` field) along with two additional fields -- `version` and `confirm`. Please see below for a full example: ```shell diff --git a/src/cmd/services/m3collector/config/config.go b/src/cmd/services/m3collector/config/config.go index 722024e508..2d607524ad 100644 --- a/src/cmd/services/m3collector/config/config.go +++ b/src/cmd/services/m3collector/config/config.go @@ -26,7 +26,6 @@ import ( "github.com/m3db/m3/src/metrics/matcher" "github.com/m3db/m3/src/metrics/matcher/cache" "github.com/m3db/m3/src/x/clock" - "github.com/m3db/m3/src/x/config/listenaddress" "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/pool" @@ -37,7 +36,7 @@ import ( type Configuration struct { Metrics instrument.MetricsConfiguration `yaml:"metrics"` Logging zap.Config `yaml:"logging"` - ListenAddress listenaddress.Configuration `yaml:"listenAddress" validate:"nonzero"` + ListenAddress string `yaml:"listenAddress" validate:"nonzero"` Etcd etcdclient.Configuration `yaml:"etcd"` Reporter ReporterConfiguration `yaml:"reporter"` } diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index f1a1545616..108b9dc824 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -21,6 +21,7 @@ package config import ( + "errors" "fmt" "github.com/m3db/m3/src/dbnode/client" @@ -43,16 +44,101 @@ import ( var ( // defaultNumProcessorsPerCPU is the default number of processors per CPU. defaultNumProcessorsPerCPU = 0.125 + + // default order in which bootstrappers are run + // (run in ascending order of precedence). + defaultOrderedBootstrappers = []string{ + // Filesystem bootstrapping must be first. + bfs.FileSystemBootstrapperName, + // Peers and commitlog must come before the uninitialized topology bootrapping. + commitlog.CommitLogBootstrapperName, + peers.PeersBootstrapperName, + uninitialized.UninitializedTopologyBootstrapperName, + } + + // bootstrapper order where peers is prefered over commitlog. + preferPeersOrderedBootstrappers = []string{ + // Filesystem bootstrapping must be first. + bfs.FileSystemBootstrapperName, + // Prefer peers over commitlog. + peers.PeersBootstrapperName, + commitlog.CommitLogBootstrapperName, + uninitialized.UninitializedTopologyBootstrapperName, + } + + // bootstrapper order where commitlog is excluded. + excludeCommitLogOrderedBootstrappers = []string{ + // Filesystem bootstrapping must be first. + bfs.FileSystemBootstrapperName, + // Commitlog excluded. + peers.PeersBootstrapperName, + uninitialized.UninitializedTopologyBootstrapperName, + } + + validBootstrapModes = []BootstrapMode{ + DefaultBootstrapMode, + PreferPeersBootstrapMode, + ExcludeCommitLogBootstrapMode, + } + + errReadBootstrapModeInvalid = errors.New("bootstrap mode invalid") +) + +// BootstrapMode defines the mode in which bootstrappers are run. +type BootstrapMode uint + +const ( + // DefaultBootstrapMode executes bootstrappers in default order. + DefaultBootstrapMode BootstrapMode = iota + // PreferPeersBootstrapMode executes peers before commitlog bootstrapper. + PreferPeersBootstrapMode + // ExcludeCommitLogBootstrapMode executes all default bootstrappers except commitlog. + ExcludeCommitLogBootstrapMode ) +// UnmarshalYAML unmarshals an BootstrapMode into a valid type from string. +func (m *BootstrapMode) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + + // If unspecified, use default mode. + if str == "" { + *m = DefaultBootstrapMode + return nil + } + + for _, valid := range validBootstrapModes { + if str == valid.String() { + *m = valid + return nil + } + } + return fmt.Errorf("invalid BootstrapMode '%s' valid types are: %s", + str, validBootstrapModes) +} + +// String returns the bootstrap mode as a string +func (m BootstrapMode) String() string { + switch m { + case DefaultBootstrapMode: + return "default" + case PreferPeersBootstrapMode: + return "prefer_peers" + case ExcludeCommitLogBootstrapMode: + return "exclude_commitlog" + } + return "unknown" +} + // BootstrapConfiguration specifies the config for bootstrappers. type BootstrapConfiguration struct { - // Bootstrappers is the list of bootstrappers, ordered by precedence in - // descending order. - Bootstrappers []string `yaml:"bootstrappers" validate:"nonzero"` + // BootstrapMode defines the mode in which bootstrappers are run. + BootstrapMode *BootstrapMode `yaml:"mode"` // Filesystem bootstrapper configuration. - Filesystem *BootstrapFilesystemConfiguration `yaml:"fs"` + Filesystem *BootstrapFilesystemConfiguration `yaml:"filesystem"` // Commitlog bootstrapper configuration. Commitlog *BootstrapCommitlogConfiguration `yaml:"commitlog"` @@ -156,31 +242,14 @@ func newDefaultBootstrapPeersConfiguration() BootstrapPeersConfiguration { } } -// BootstrapConfigurationValidator can be used to validate the option sets -// that the bootstrap configuration builds. -// Useful for tests and perhaps verifying same options set across multiple -// bootstrappers. -type BootstrapConfigurationValidator interface { - ValidateBootstrappersOrder(names []string) error - ValidateFilesystemBootstrapperOptions(opts bfs.Options) error - ValidateCommitLogBootstrapperOptions(opts commitlog.Options) error - ValidatePeersBootstrapperOptions(opts peers.Options) error - ValidateUninitializedBootstrapperOptions(opts uninitialized.Options) error -} - // New creates a bootstrap process based on the bootstrap configuration. func (bsc BootstrapConfiguration) New( - validator BootstrapConfigurationValidator, rsOpts result.Options, opts storage.Options, topoMapProvider topology.MapProvider, origin topology.Host, adminClient client.AdminClient, ) (bootstrap.ProcessProvider, error) { - if err := validator.ValidateBootstrappersOrder(bsc.Bootstrappers); err != nil { - return nil, err - } - idxOpts := opts.IndexOptions() compactor, err := compaction.NewCompactor(idxOpts.DocumentArrayPool(), index.DocumentArrayPoolCapacity, @@ -199,12 +268,14 @@ func (bsc BootstrapConfiguration) New( } var ( - bs bootstrap.BootstrapperProvider - fsOpts = opts.CommitLogOptions().FilesystemOptions() + bs bootstrap.BootstrapperProvider + fsOpts = opts.CommitLogOptions().FilesystemOptions() + orderedBootstrappers = bsc.orderedBootstrappers() ) // Start from the end of the list because the bootstrappers are ordered by precedence in descending order. - for i := len(bsc.Bootstrappers) - 1; i >= 0; i-- { - switch bsc.Bootstrappers[i] { + // I.e. each bootstrapper wraps the preceding bootstrapper, and so the outer-most bootstrapper is run first. + for i := len(orderedBootstrappers) - 1; i >= 0; i-- { + switch orderedBootstrappers[i] { case bootstrapper.NoOpAllBootstrapperName: bs = bootstrapper.NewNoOpAllBootstrapperProvider() case bootstrapper.NoOpNoneBootstrapperName: @@ -225,7 +296,7 @@ func (bsc BootstrapConfiguration) New( if v := bsc.IndexSegmentConcurrency; v != nil { fsbOpts = fsbOpts.SetIndexSegmentConcurrency(*v) } - if err := validator.ValidateFilesystemBootstrapperOptions(fsbOpts); err != nil { + if err := fsbOpts.Validate(); err != nil { return nil, err } bs, err = bfs.NewFileSystemBootstrapperProvider(fsbOpts, bs) @@ -239,7 +310,7 @@ func (bsc BootstrapConfiguration) New( SetCommitLogOptions(opts.CommitLogOptions()). SetRuntimeOptionsManager(opts.RuntimeOptionsManager()). SetReturnUnfulfilledForCorruptCommitLogFiles(cCfg.ReturnUnfulfilledForCorruptCommitLogFiles) - if err := validator.ValidateCommitLogBootstrapperOptions(cOpts); err != nil { + if err := cOpts.Validate(); err != nil { return nil, err } inspection, err := fs.InspectFilesystem(fsOpts) @@ -267,7 +338,7 @@ func (bsc BootstrapConfiguration) New( if v := bsc.IndexSegmentConcurrency; v != nil { pOpts = pOpts.SetIndexSegmentConcurrency(*v) } - if err := validator.ValidatePeersBootstrapperOptions(pOpts); err != nil { + if err := pOpts.Validate(); err != nil { return nil, err } bs, err = peers.NewPeersBootstrapperProvider(pOpts, bs) @@ -278,12 +349,12 @@ func (bsc BootstrapConfiguration) New( uOpts := uninitialized.NewOptions(). SetResultOptions(rsOpts). SetInstrumentOptions(opts.InstrumentOptions()) - if err := validator.ValidateUninitializedBootstrapperOptions(uOpts); err != nil { + if err := uOpts.Validate(); err != nil { return nil, err } bs = uninitialized.NewUninitializedTopologyBootstrapperProvider(uOpts, bs) default: - return nil, fmt.Errorf("unknown bootstrapper: %s", bsc.Bootstrappers[i]) + return nil, fmt.Errorf("unknown bootstrapper: %s", orderedBootstrappers[i]) } } @@ -317,93 +388,16 @@ func (bsc BootstrapConfiguration) peersConfig() BootstrapPeersConfiguration { return newDefaultBootstrapPeersConfiguration() } -type bootstrapConfigurationValidator struct { -} - -// NewBootstrapConfigurationValidator returns a new bootstrap configuration -// validator that validates certain options configured by the bootstrap -// configuration. -func NewBootstrapConfigurationValidator() BootstrapConfigurationValidator { - return bootstrapConfigurationValidator{} -} - -func (v bootstrapConfigurationValidator) ValidateBootstrappersOrder(names []string) error { - dataFetchingBootstrappers := []string{ - bfs.FileSystemBootstrapperName, - peers.PeersBootstrapperName, - commitlog.CommitLogBootstrapperName, - } - - precedingBootstrappersAllowedByBootstrapper := map[string][]string{ - bootstrapper.NoOpAllBootstrapperName: dataFetchingBootstrappers, - bootstrapper.NoOpNoneBootstrapperName: dataFetchingBootstrappers, - bfs.FileSystemBootstrapperName: []string{ - // Filesystem bootstrapper must always appear first - }, - peers.PeersBootstrapperName: []string{ - // Peers must always appear after filesystem - bfs.FileSystemBootstrapperName, - // Peers may appear before OR after commitlog - commitlog.CommitLogBootstrapperName, - }, - commitlog.CommitLogBootstrapperName: []string{ - // Commit log bootstrapper may appear after filesystem or peers - bfs.FileSystemBootstrapperName, - peers.PeersBootstrapperName, - }, - uninitialized.UninitializedTopologyBootstrapperName: []string{ - // Unintialized bootstrapper may appear after filesystem or peers or commitlog - bfs.FileSystemBootstrapperName, - commitlog.CommitLogBootstrapperName, - peers.PeersBootstrapperName, - }, - } - - validated := make(map[string]struct{}) - for _, name := range names { - precedingAllowed, ok := precedingBootstrappersAllowedByBootstrapper[name] - if !ok { - return fmt.Errorf("unknown bootstrapper: %v", name) +func (bsc BootstrapConfiguration) orderedBootstrappers() []string { + if bsc.BootstrapMode != nil { + switch *bsc.BootstrapMode { + case DefaultBootstrapMode: + return defaultOrderedBootstrappers + case PreferPeersBootstrapMode: + return preferPeersOrderedBootstrappers + case ExcludeCommitLogBootstrapMode: + return excludeCommitLogOrderedBootstrappers } - - allowed := make(map[string]struct{}) - for _, elem := range precedingAllowed { - allowed[elem] = struct{}{} - } - - for existing := range validated { - if _, ok := allowed[existing]; !ok { - return fmt.Errorf("bootstrapper %s cannot appear after %s: ", - name, existing) - } - } - - validated[name] = struct{}{} } - - return nil -} - -func (v bootstrapConfigurationValidator) ValidateFilesystemBootstrapperOptions( - opts bfs.Options, -) error { - return opts.Validate() -} - -func (v bootstrapConfigurationValidator) ValidateCommitLogBootstrapperOptions( - opts commitlog.Options, -) error { - return opts.Validate() -} - -func (v bootstrapConfigurationValidator) ValidatePeersBootstrapperOptions( - opts peers.Options, -) error { - return opts.Validate() -} - -func (v bootstrapConfigurationValidator) ValidateUninitializedBootstrapperOptions( - opts uninitialized.Options, -) error { - return opts.Validate() + return defaultOrderedBootstrappers } diff --git a/src/cmd/services/m3dbnode/config/bootstrap_test.go b/src/cmd/services/m3dbnode/config/bootstrap_test.go deleted file mode 100644 index d9d332cd1f..0000000000 --- a/src/cmd/services/m3dbnode/config/bootstrap_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package config - -import ( - "fmt" - "testing" - - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/commitlog" - bfs "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/fs" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/peers" - - "github.com/stretchr/testify/require" -) - -var ( - peersBs = peers.PeersBootstrapperName - fsBs = bfs.FileSystemBootstrapperName - commitLogBs = commitlog.CommitLogBootstrapperName - noOpAllBs = bootstrapper.NoOpAllBootstrapperName - noOpNoneBs = bootstrapper.NoOpNoneBootstrapperName -) - -func TestValidatorValidateBootstrappersOrder(t *testing.T) { - tests := []struct { - valid bool - bootstrappers []string - }{ - {true, []string{fsBs, peersBs, commitLogBs, noOpNoneBs}}, - {true, []string{fsBs, commitLogBs, noOpNoneBs}}, - {true, []string{commitLogBs, noOpNoneBs}}, - {true, []string{fsBs, noOpNoneBs}}, - {true, []string{fsBs}}, - {true, []string{fsBs, peersBs}}, - {true, []string{peersBs}}, - {true, []string{peersBs, commitLogBs}}, - {true, []string{fsBs, commitLogBs}}, - {true, []string{noOpNoneBs}}, - {true, []string{noOpAllBs}}, - // Do not allow peers to appear before FS - {false, []string{peersBs, fsBs, commitLogBs, noOpNoneBs}}, - // Do not allow a non-data fetching bootstrapper twice - {false, []string{commitLogBs, noOpAllBs, noOpNoneBs}}, - // Do not allow multiple bootstrappers to appear - {false, []string{commitLogBs, commitLogBs, noOpNoneBs}}, - // Do not allow unknown bootstrappers - {false, []string{"foo"}}, - } - - for _, tt := range tests { - name := fmt.Sprintf("%v", tt.bootstrappers) - t.Run(name, func(t *testing.T) { - validator := NewBootstrapConfigurationValidator() - err := validator.ValidateBootstrappersOrder(tt.bootstrappers) - if tt.valid { - require.NoError(t, err) - } else { - require.Error(t, err) - } - }) - } -} diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index fb3f98862e..e0ebed20da 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -126,7 +126,7 @@ type DBConfiguration struct { Cache CacheConfigurations `yaml:"cache"` // The filesystem configuration for the node. - Filesystem FilesystemConfiguration `yaml:"fs"` + Filesystem FilesystemConfiguration `yaml:"filesystem"` // The commit log policy for the node. CommitLog CommitLogPolicy `yaml:"commitlog"` @@ -301,10 +301,6 @@ type CommitLogPolicy struct { // works in most cases because the default size of the QueueChannel should be large // enough for almost all workloads assuming a reasonable batch size is used. QueueChannel *CommitLogQueuePolicy `yaml:"queueChannel"` - - // Deprecated. Left in struct to keep old YAMLs parseable. - // TODO(V1): remove - DeprecatedBlockSize *time.Duration `yaml:"blockSize"` } // CalculationType is a type of configuration parameter. diff --git a/src/cmd/services/m3dbnode/config/config_mock.go b/src/cmd/services/m3dbnode/config/config_mock.go deleted file mode 100644 index b0e94f8b4d..0000000000 --- a/src/cmd/services/m3dbnode/config/config_mock.go +++ /dev/null @@ -1,129 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/m3db/m3/src/cmd/services/m3dbnode/config (interfaces: BootstrapConfigurationValidator) - -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package config is a generated GoMock package. -package config - -import ( - "reflect" - - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/commitlog" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/fs" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/peers" - "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/uninitialized" - - "github.com/golang/mock/gomock" -) - -// MockBootstrapConfigurationValidator is a mock of BootstrapConfigurationValidator interface -type MockBootstrapConfigurationValidator struct { - ctrl *gomock.Controller - recorder *MockBootstrapConfigurationValidatorMockRecorder -} - -// MockBootstrapConfigurationValidatorMockRecorder is the mock recorder for MockBootstrapConfigurationValidator -type MockBootstrapConfigurationValidatorMockRecorder struct { - mock *MockBootstrapConfigurationValidator -} - -// NewMockBootstrapConfigurationValidator creates a new mock instance -func NewMockBootstrapConfigurationValidator(ctrl *gomock.Controller) *MockBootstrapConfigurationValidator { - mock := &MockBootstrapConfigurationValidator{ctrl: ctrl} - mock.recorder = &MockBootstrapConfigurationValidatorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockBootstrapConfigurationValidator) EXPECT() *MockBootstrapConfigurationValidatorMockRecorder { - return m.recorder -} - -// ValidateBootstrappersOrder mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateBootstrappersOrder(arg0 []string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateBootstrappersOrder", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateBootstrappersOrder indicates an expected call of ValidateBootstrappersOrder -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateBootstrappersOrder(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateBootstrappersOrder", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateBootstrappersOrder), arg0) -} - -// ValidateCommitLogBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateCommitLogBootstrapperOptions(arg0 commitlog.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateCommitLogBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateCommitLogBootstrapperOptions indicates an expected call of ValidateCommitLogBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateCommitLogBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateCommitLogBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateCommitLogBootstrapperOptions), arg0) -} - -// ValidateFilesystemBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateFilesystemBootstrapperOptions(arg0 fs.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateFilesystemBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateFilesystemBootstrapperOptions indicates an expected call of ValidateFilesystemBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateFilesystemBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateFilesystemBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateFilesystemBootstrapperOptions), arg0) -} - -// ValidatePeersBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidatePeersBootstrapperOptions(arg0 peers.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidatePeersBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidatePeersBootstrapperOptions indicates an expected call of ValidatePeersBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidatePeersBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidatePeersBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidatePeersBootstrapperOptions), arg0) -} - -// ValidateUninitializedBootstrapperOptions mocks base method -func (m *MockBootstrapConfigurationValidator) ValidateUninitializedBootstrapperOptions(arg0 uninitialized.Options) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateUninitializedBootstrapperOptions", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateUninitializedBootstrapperOptions indicates an expected call of ValidateUninitializedBootstrapperOptions -func (mr *MockBootstrapConfigurationValidatorMockRecorder) ValidateUninitializedBootstrapperOptions(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateUninitializedBootstrapperOptions", reflect.TypeOf((*MockBootstrapConfigurationValidator)(nil).ValidateUninitializedBootstrapperOptions), arg0) -} diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index ee76910bef..10c82b710b 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -99,11 +99,7 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - peers - - noop-all - fs: + filesystem: numProcessorsPerCPU: 0.42 commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -115,7 +111,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db writeBufferSize: 65536 dataReadBufferSize: 65536 @@ -417,11 +413,8 @@ func TestConfiguration(t *testing.T) { writeNewSeriesBackoffDuration: 2ms tick: null bootstrap: - bootstrappers: - - filesystem - - peers - - noop-all - fs: + mode: null + filesystem: numProcessorsPerCPU: 0.42 migration: null commitlog: @@ -437,7 +430,7 @@ func TestConfiguration(t *testing.T) { cacheRegexp: false cacheTerms: false regexp: null - fs: + filesystem: filePathPrefix: /var/lib/m3db writeBufferSize: 65536 dataReadBufferSize: 65536 @@ -458,7 +451,6 @@ func TestConfiguration(t *testing.T) { calculationType: fixed size: 2097152 queueChannel: null - blockSize: null repair: enabled: false throttle: 2m0s @@ -942,10 +934,6 @@ db: httpNodeListenAddress: 0.0.0.0:9002 httpClusterListenAddress: 0.0.0.0:9003 - bootstrap: - bootstrappers: - - noop-all - commitlog: flushMaxBytes: 524288 flushEvery: 1s @@ -1010,11 +998,6 @@ db: httpClusterListenAddress: 0.0.0.0:9003 bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: ` + notDefaultStr + ` @@ -1040,25 +1023,16 @@ db: require.NoError(t, err) require.NotNil(t, cfg.DB) - validator := NewMockBootstrapConfigurationValidator(ctrl) - validator.EXPECT().ValidateBootstrappersOrder(gomock.Any()).Return(nil).AnyTimes() - validator.EXPECT().ValidateFilesystemBootstrapperOptions(gomock.Any()).Return(nil) - validator.EXPECT().ValidatePeersBootstrapperOptions(gomock.Any()).Return(nil) - validator.EXPECT().ValidateUninitializedBootstrapperOptions(gomock.Any()).Return(nil) - validator.EXPECT(). - ValidateCommitLogBootstrapperOptions(gomock.Any()). - DoAndReturn(func(opts commitlog.Options) error { - actual := opts.ReturnUnfulfilledForCorruptCommitLogFiles() - expected := notDefault - require.Equal(t, expected, actual) - return nil - }) - mapProvider := topology.NewMockMapProvider(ctrl) origin := topology.NewMockHost(ctrl) adminClient := client.NewMockAdminClient(ctrl) - _, err = cfg.DB.Bootstrap.New(validator, - result.NewOptions(), storage.DefaultTestOptions(), mapProvider, origin, adminClient) + _, err = cfg.DB.Bootstrap.New( + result.NewOptions(), + storage.DefaultTestOptions(), + mapProvider, + origin, + adminClient, + ) require.NoError(t, err) } diff --git a/src/cmd/services/m3dbnode/main/common_test.go b/src/cmd/services/m3dbnode/main/common_test.go index 9766515e7f..c1f744f9e7 100644 --- a/src/cmd/services/m3dbnode/main/common_test.go +++ b/src/cmd/services/m3dbnode/main/common_test.go @@ -35,8 +35,8 @@ import ( "github.com/m3db/m3/src/cluster/shard" "github.com/m3db/m3/src/dbnode/client" - "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/namespace" + "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/x/ident" "github.com/gogo/protobuf/proto" diff --git a/src/cmd/services/m3dbnode/main/main_index_test.go b/src/cmd/services/m3dbnode/main/main_index_test.go index b5022112af..a4cba46dbf 100644 --- a/src/cmd/services/m3dbnode/main/main_index_test.go +++ b/src/cmd/services/m3dbnode/main/main_index_test.go @@ -350,13 +350,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology - commitlog: flushMaxBytes: 524288 flushEvery: 1s @@ -364,7 +357,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: {{.DataDir}} writeBufferSize: 65536 dataReadBufferSize: 65536 diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go index 690f639d24..d8068c6e29 100644 --- a/src/cmd/services/m3dbnode/main/main_test.go +++ b/src/cmd/services/m3dbnode/main/main_test.go @@ -506,13 +506,6 @@ db: writeNewSeriesLimitPerSecond: 1048576 writeNewSeriesBackoffDuration: 2ms - bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology - commitlog: flushMaxBytes: 524288 flushEvery: 1s @@ -520,7 +513,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: {{.DataDir}} writeBufferSize: 65536 dataReadBufferSize: 65536 diff --git a/src/cmd/services/m3query/config/config.go b/src/cmd/services/m3query/config/config.go index 63290041bb..3d1a0adfcf 100644 --- a/src/cmd/services/m3query/config/config.go +++ b/src/cmd/services/m3query/config/config.go @@ -22,7 +22,6 @@ package config import ( "errors" - "fmt" "time" etcdclient "github.com/m3db/m3/src/cluster/client/etcd" @@ -38,10 +37,7 @@ import ( "github.com/m3db/m3/src/query/storage/m3/consolidators" "github.com/m3db/m3/src/query/storage/m3/storagemetadata" xconfig "github.com/m3db/m3/src/x/config" - "github.com/m3db/m3/src/x/config/listenaddress" - "github.com/m3db/m3/src/x/cost" "github.com/m3db/m3/src/x/debug/config" - xdocs "github.com/m3db/m3/src/x/docs" "github.com/m3db/m3/src/x/instrument" xlog "github.com/m3db/m3/src/x/log" "github.com/m3db/m3/src/x/opentracing" @@ -62,9 +58,6 @@ const ( NoopEtcdStorageType BackendStorageType = "noop-etcd" defaultCarbonIngesterListenAddress = "0.0.0.0:7204" - errNoIDGenerationScheme = "error: a recent breaking change means that an ID " + - "generation scheme is required in coordinator configuration settings. " + - "More information is available here: %s" defaultQueryTimeout = 30 * time.Second @@ -117,7 +110,7 @@ type Configuration struct { ClusterManagement *ClusterManagementConfiguration `yaml:"clusterManagement"` // ListenAddress is the server listen address. - ListenAddress *listenaddress.Configuration `yaml:"listenAddress" validate:"nonzero"` + ListenAddress string `yaml:"listenAddress" validate:"nonzero"` // Filter is the read/write/complete tags filter configuration. Filter FilterConfiguration `yaml:"filter"` @@ -167,15 +160,6 @@ type Configuration struct { // StoreMetricsType controls if metrics type is stored or not. StoreMetricsType *bool `yaml:"storeMetricsType"` - // Cache configurations. - // - // Deprecated: cache configurations are no longer supported. Remove from file - // when we can make breaking changes. - // (If/when removed it will make existing configurations with the cache - // stanza not able to startup the binary since we parse YAML in strict mode - // by default). - DeprecatedCache CacheConfiguration `yaml:"cache"` - // MultiProcess is the multi-process configuration. MultiProcess MultiProcessConfiguration `yaml:"multiProcess"` @@ -217,17 +201,6 @@ type FilterConfiguration struct { CompleteTags Filter `yaml:"completeTags"` } -// CacheConfiguration contains the cache configurations. -type CacheConfiguration struct { - // Deprecated: remove from config. - DeprecatedQueryConversion *DeprecatedQueryConversionCacheConfiguration `yaml:"queryConversion"` -} - -// DeprecatedQueryConversionCacheConfiguration is deprecated: remove from config. -type DeprecatedQueryConversionCacheConfiguration struct { - Size *int `yaml:"size"` -} - // ResultOptions are the result options for query. type ResultOptions struct { // KeepNans keeps NaNs before returning query results. @@ -324,41 +297,6 @@ func (c PrometheusQueryConfiguration) MaxSamplesPerQueryOrDefault() int { type LimitsConfiguration struct { // PerQuery configures limits which apply to each query individually. PerQuery PerQueryLimitsConfiguration `yaml:"perQuery"` - - // Global configures limits which apply across all queries running on this - // instance. - Global GlobalLimitsConfiguration `yaml:"global"` - - // deprecated: use PerQuery.MaxComputedDatapoints instead. - DeprecatedMaxComputedDatapoints int `yaml:"maxComputedDatapoints"` -} - -// MaxComputedDatapoints is a getter providing backwards compatibility between -// LimitsConfiguration.DeprecatedMaxComputedDatapoints and -// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints. See -// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints for a comment on -// the semantics. -func (lc LimitsConfiguration) MaxComputedDatapoints() int { - if lc.PerQuery.PrivateMaxComputedDatapoints != 0 { - return lc.PerQuery.PrivateMaxComputedDatapoints - } - - return lc.DeprecatedMaxComputedDatapoints -} - -// GlobalLimitsConfiguration represents limits on resource usage across a query -// instance. Zero or negative values imply no limit. -type GlobalLimitsConfiguration struct { - // MaxFetchedDatapoints limits the max number of datapoints allowed to be - // used by all queries at any point in time, this is applied at the query - // service after the result has been returned by a storage node. - MaxFetchedDatapoints int `yaml:"maxFetchedDatapoints"` -} - -// AsLimitManagerOptions converts this configuration to -// cost.LimitManagerOptions for MaxFetchedDatapoints. -func (l *GlobalLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions { - return toLimitManagerOptions(l.MaxFetchedDatapoints) } // PerQueryLimitsConfiguration represents limits on resource usage within a @@ -376,26 +314,6 @@ type PerQueryLimitsConfiguration struct { // RequireExhaustive results in an error if the query exceeds any limit. RequireExhaustive *bool `yaml:"requireExhaustive"` - - // MaxFetchedDatapoints limits the max number of datapoints allowed to be - // used by a given query, this is applied at the query service after the - // result has been returned by a storage node. - MaxFetchedDatapoints int `yaml:"maxFetchedDatapoints"` - - // PrivateMaxComputedDatapoints limits the number of datapoints that can be - // returned by a query. It's determined purely - // from the size of the time range and the step size (end - start / step). - // - // N.B.: the hacky "Private" prefix is to indicate that callers should use - // LimitsConfiguration.MaxComputedDatapoints() instead of accessing - // this field directly. - PrivateMaxComputedDatapoints int `yaml:"maxComputedDatapoints"` -} - -// AsLimitManagerOptions converts this configuration to -// cost.LimitManagerOptions for MaxFetchedDatapoints. -func (l *PerQueryLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions { - return toLimitManagerOptions(l.MaxFetchedDatapoints) } // AsFetchOptionsBuilderLimitsOptions converts this configuration to @@ -423,13 +341,6 @@ func (l *PerQueryLimitsConfiguration) AsFetchOptionsBuilderLimitsOptions() handl } } -func toLimitManagerOptions(limit int) cost.LimitManagerOptions { - return cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{ - Threshold: cost.Cost(limit), - Enabled: limit > 0, - }) -} - // IngestConfiguration is the configuration for ingestion server. type IngestConfiguration struct { // Ingester is the configuration for storage based ingester. @@ -489,12 +400,9 @@ type CarbonConfiguration struct { // CarbonIngesterConfiguration is the configuration struct for carbon ingestion. type CarbonIngesterConfiguration struct { - // Deprecated: simply use the logger debug level, this has been deprecated - // in favor of setting the log level to debug. - DeprecatedDebug bool `yaml:"debug"` - ListenAddress string `yaml:"listenAddress"` - MaxConcurrency int `yaml:"maxConcurrency"` - Rules []CarbonIngesterRuleConfiguration `yaml:"rules"` + ListenAddress string `yaml:"listenAddress"` + MaxConcurrency int `yaml:"maxConcurrency"` + Rules []CarbonIngesterRuleConfiguration `yaml:"rules"` } // LookbackDurationOrDefault validates the LookbackDuration @@ -680,7 +588,7 @@ type TagOptionsConfiguration struct { // If not provided, defaults to `le`. BucketName string `yaml:"bucketName"` - // Scheme determines the default ID generation scheme. Defaults to TypeLegacy. + // Scheme determines the default ID generation scheme. Defaults to TypeQuoted. Scheme models.IDSchemeType `yaml:"idScheme"` // Filters are optional tag filters, removing all series with tags @@ -719,9 +627,8 @@ func TagOptionsFromConfig(cfg TagOptionsConfiguration) (models.TagOptions, error } if cfg.Scheme == models.TypeDefault { - // If no config has been set, error. - docLink := xdocs.Path("how_to/query#migration") - return nil, fmt.Errorf(errNoIDGenerationScheme, docLink) + // Default to quoted if unspecified. + cfg.Scheme = models.TypeQuoted } opts = opts.SetIDSchemeType(cfg.Scheme) diff --git a/src/cmd/services/m3query/config/config_test.go b/src/cmd/services/m3query/config/config_test.go index 0143abf2fc..acb787da8c 100644 --- a/src/cmd/services/m3query/config/config_test.go +++ b/src/cmd/services/m3query/config/config_test.go @@ -26,8 +26,6 @@ import ( "github.com/m3db/m3/src/query/models" xconfig "github.com/m3db/m3/src/x/config" - "github.com/m3db/m3/src/x/cost" - xdocs "github.com/m3db/m3/src/x/docs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,16 +35,18 @@ import ( const testConfigFile = "./testdata/config.yml" -func TestTagOptionsFromEmptyConfigErrors(t *testing.T) { +func TestDefaultTagOptionsFromEmptyConfig(t *testing.T) { cfg := TagOptionsConfiguration{} opts, err := TagOptionsFromConfig(cfg) - require.Error(t, err) - require.Nil(t, opts) + require.NoError(t, err) + require.Equal(t, models.TypeQuoted, opts.IDSchemeType()) } func TestTagOptionsFromConfigWithIDGenerationScheme(t *testing.T) { - schemes := []models.IDSchemeType{models.TypeLegacy, - models.TypePrependMeta, models.TypeQuoted} + schemes := []models.IDSchemeType{ + models.TypePrependMeta, + models.TypeQuoted, + } for _, scheme := range schemes { cfg := TagOptionsConfiguration{ Scheme: scheme, @@ -64,7 +64,7 @@ func TestTagOptionsFromConfig(t *testing.T) { name := "foobar" cfg := TagOptionsConfiguration{ MetricName: name, - Scheme: models.TypeLegacy, + Scheme: models.TypeQuoted, Filters: []TagFilter{ {Name: "foo", Values: []string{".", "abc"}}, {Name: "bar", Values: []string{".*"}}, @@ -86,103 +86,17 @@ func TestTagOptionsFromConfig(t *testing.T) { } } -func TestLimitsConfigurationAsLimitManagerOptions(t *testing.T) { - cases := []struct { - input interface { - AsLimitManagerOptions() cost.LimitManagerOptions - } - expectedDefault int - }{{ - input: &PerQueryLimitsConfiguration{ - MaxFetchedDatapoints: 5, - }, - expectedDefault: 5, - }, { - input: &GlobalLimitsConfiguration{ - MaxFetchedDatapoints: 6, - }, - expectedDefault: 6, - }} - - for _, tc := range cases { - t.Run(fmt.Sprintf("type_%T", tc.input), func(t *testing.T) { - res := tc.input.AsLimitManagerOptions() - assert.Equal(t, cost.Limit{ - Threshold: cost.Cost(tc.expectedDefault), - Enabled: true, - }, res.DefaultLimit()) - }) - } -} - -func TestLimitsConfigurationMaxComputedDatapoints(t *testing.T) { - t.Run("uses PerQuery value if provided", func(t *testing.T) { - lc := &LimitsConfiguration{ - DeprecatedMaxComputedDatapoints: 6, - PerQuery: PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: 5, - }, - } - - assert.Equal(t, 5, lc.MaxComputedDatapoints()) - }) - - t.Run("uses deprecated value if PerQuery not provided", func(t *testing.T) { - lc := &LimitsConfiguration{ - DeprecatedMaxComputedDatapoints: 6, - } - - assert.Equal(t, 6, lc.MaxComputedDatapoints()) - }) -} - -func TestToLimitManagerOptions(t *testing.T) { - cases := []struct { - name string - input int - expectedLimit cost.Limit - }{{ - name: "negative is disabled", - input: -5, - expectedLimit: cost.Limit{ - Threshold: cost.Cost(-5), - Enabled: false, - }, - }, { - name: "zero is disabled", - input: 0, - expectedLimit: cost.Limit{ - Threshold: cost.Cost(0), - Enabled: false, - }, - }, { - name: "positive is enabled", - input: 5, - expectedLimit: cost.Limit{ - Threshold: cost.Cost(5), - Enabled: true, - }, - }} - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expectedLimit, toLimitManagerOptions(tc.input).DefaultLimit()) - }) - } -} - func TestConfigLoading(t *testing.T) { var cfg Configuration require.NoError(t, xconfig.LoadFile(&cfg, testConfigFile, xconfig.Options{})) + var requireExhaustive bool + requireExhaustive = true assert.Equal(t, &LimitsConfiguration{ - DeprecatedMaxComputedDatapoints: 10555, PerQuery: PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: 12000, - MaxFetchedDatapoints: 11000, - }, - Global: GlobalLimitsConfiguration{ - MaxFetchedDatapoints: 13000, + MaxFetchedSeries: 12000, + MaxFetchedDocs: 11000, + RequireExhaustive: &requireExhaustive, }, }, &cfg.Limits) // TODO: assert on more fields here. @@ -217,7 +131,7 @@ func TestConfigValidation(t *testing.T) { cfg := baseCfg(t) cfg.Limits = LimitsConfiguration{ PerQuery: PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: tc.limit, + MaxFetchedSeries: tc.limit, }} assert.NoError(t, validator.Validate(cfg)) @@ -229,11 +143,8 @@ func TestDefaultTagOptionsConfigErrors(t *testing.T) { var cfg TagOptionsConfiguration require.NoError(t, yaml.Unmarshal([]byte(""), &cfg)) opts, err := TagOptionsFromConfig(cfg) - - docLink := xdocs.Path("how_to/query#migration") - expectedError := fmt.Sprintf(errNoIDGenerationScheme, docLink) - require.EqualError(t, err, expectedError) - require.Nil(t, opts) + require.NoError(t, err) + require.Equal(t, models.TypeQuoted, opts.IDSchemeType()) } func TestGraphiteIDGenerationSchemeIsInvalid(t *testing.T) { @@ -246,7 +157,6 @@ func TestTagOptionsConfigWithTagGenerationScheme(t *testing.T) { schemeStr string scheme models.IDSchemeType }{ - {"legacy", models.TypeLegacy}, {"prepend_meta", models.TypePrependMeta}, {"quoted", models.TypeQuoted}, } diff --git a/src/cmd/services/m3query/config/testdata/config.yml b/src/cmd/services/m3query/config/testdata/config.yml index 0e27dfe751..5ee0e2ff7b 100644 --- a/src/cmd/services/m3query/config/testdata/config.yml +++ b/src/cmd/services/m3query/config/testdata/config.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info @@ -53,9 +52,7 @@ clusters: backgroundHealthCheckFailThrottleFactor: 0.5 limits: - maxComputedDatapoints: 10555 perQuery: - maxComputedDatapoints: 12000 - maxFetchedDatapoints: 11000 - global: - maxFetchedDatapoints: 13000 + maxFetchedSeries: 12000 + maxFetchedDocs: 11000 + requireExhaustive: true diff --git a/src/cmd/services/m3query/config/testdata/config_test.yml b/src/cmd/services/m3query/config/testdata/config_test.yml index f8e982ed1a..68250c3f0d 100644 --- a/src/cmd/services/m3query/config/testdata/config_test.yml +++ b/src/cmd/services/m3query/config/testdata/config_test.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info @@ -54,7 +53,5 @@ clusters: limits: perQuery: - maxComputedDatapoints: 12000 - maxFetchedDatapoints: 11000 - global: - maxFetchedDatapoints: 13000 + maxFetchedSeries: 12000 + maxFetchedDocs: 11000 diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml index 25da9de16f..53764669f0 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3coordinator.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml index 6970916644..9c2b6f1dec 100644 --- a/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml +++ b/src/cmd/tools/dtest/docker/harness/resources/config/m3dbnode.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: @@ -21,9 +20,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted @@ -60,11 +56,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -81,7 +72,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/src/cmd/tools/m3ctl/namespaces/types.go b/src/cmd/tools/m3ctl/namespaces/types.go index 8046270391..a4f44f6c5d 100644 --- a/src/cmd/tools/m3ctl/namespaces/types.go +++ b/src/cmd/tools/m3ctl/namespaces/types.go @@ -22,7 +22,7 @@ package namespaces const ( // DefaultPath is the default url path for the namespace api calls - DefaultPath = "/api/v1/namespace" + DefaultPath = "/api/v1/services/m3db/namespace" // DebugQS this is the query string to activate debug output in api responses DebugQS = "debug=true" ) diff --git a/src/collector/config/m3collector.yml b/src/collector/config/m3collector.yml index 15bf496756..b555eeccd7 100644 --- a/src/collector/config/m3collector.yml +++ b/src/collector/config/m3collector.yml @@ -1,6 +1,4 @@ -listenAddress: - type: config - value: 0.0.0.0:7206 +listenAddress: 0.0.0.0:7206 metrics: scope: diff --git a/src/collector/server/server.go b/src/collector/server/server.go index e28578a424..c8b1d53189 100644 --- a/src/collector/server/server.go +++ b/src/collector/server/server.go @@ -136,12 +136,7 @@ func Run(runOpts RunOptions) { logger.Fatal("unable to register routes", zap.Error(err)) } - listenAddress, err := cfg.ListenAddress.Resolve() - if err != nil { - logger.Fatal("unable to resolve listen address", zap.Error(err)) - } - - srv := &http.Server{Addr: listenAddress, Handler: handler.Router()} + srv := &http.Server{Addr: cfg.ListenAddress, Handler: handler.Router()} defer func() { logger.Info("closing server") if err := srv.Shutdown(ctx); err != nil { @@ -150,10 +145,10 @@ func Run(runOpts RunOptions) { }() go func() { - logger.Info("starting server", zap.String("address", listenAddress)) + logger.Info("starting server", zap.String("address", cfg.ListenAddress)) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Fatal("server error while listening", - zap.String("address", listenAddress), zap.Error(err)) + zap.String("address", cfg.ListenAddress), zap.Error(err)) } }() diff --git a/src/dbnode/config/m3dbnode-all-config.yml b/src/dbnode/config/m3dbnode-all-config.yml index 75d3e55255..bc0ff7c742 100644 --- a/src/dbnode/config/m3dbnode-all-config.yml +++ b/src/dbnode/config/m3dbnode-all-config.yml @@ -1,8 +1,7 @@ # Include this field if you want to enable an embedded M3Coordinator instance. coordinator: # Address for M3Coordinator to listen for traffic. - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 # All configured M3DB namespaces must be listed in this config if running an # embedded M3Coordinator instance. @@ -30,9 +29,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted @@ -108,14 +104,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - # Order in which to run the bootstrappers. Don't change these values unless - # you know what you're doing as non-standard configurations can cause data - # loss or make recovery from disaster scenarios difficult. - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -135,7 +123,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: # Directory to store M3DB data in. filePathPrefix: /var/lib/m3db # Various fixed-sized buffers used for M3DB I/O. diff --git a/src/dbnode/config/m3dbnode-cluster-template.yml b/src/dbnode/config/m3dbnode-cluster-template.yml index 23b5c9b223..085b9ee694 100644 --- a/src/dbnode/config/m3dbnode-cluster-template.yml +++ b/src/dbnode/config/m3dbnode-cluster-template.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: @@ -78,11 +77,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -99,5 +93,5 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db diff --git a/src/dbnode/config/m3dbnode-local-etcd-proto.yml b/src/dbnode/config/m3dbnode-local-etcd-proto.yml index 7cae28ed10..82f877c9b6 100644 --- a/src/dbnode/config/m3dbnode-local-etcd-proto.yml +++ b/src/dbnode/config/m3dbnode-local-etcd-proto.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: @@ -21,9 +20,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted @@ -60,11 +56,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -81,7 +72,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 6970916644..9c2b6f1dec 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -1,6 +1,5 @@ coordinator: - listenAddress: - value: "0.0.0.0:7201" + listenAddress: 0.0.0.0:7201 local: namespaces: @@ -21,9 +20,6 @@ coordinator: samplingRate: 1.0 extended: none - limits: - maxComputedDatapoints: 10000 - tagOptions: # Configuration setting for generating metric IDs from tags. idScheme: quoted @@ -60,11 +56,6 @@ db: writeNewSeriesBackoffDuration: 2ms bootstrap: - bootstrappers: - - filesystem - - commitlog - - peers - - uninitialized_topology commitlog: returnUnfulfilledForCorruptCommitLogFiles: false @@ -81,7 +72,7 @@ db: calculationType: fixed size: 2097152 - fs: + filesystem: filePathPrefix: /var/lib/m3db config: diff --git a/src/dbnode/generated/mocks/generate.go b/src/dbnode/generated/mocks/generate.go index 1313816ff6..8da07ecdc5 100644 --- a/src/dbnode/generated/mocks/generate.go +++ b/src/dbnode/generated/mocks/generate.go @@ -26,7 +26,6 @@ //go:generate sh -c "mockgen -package=digest -destination=$GOPATH/src/$PACKAGE/src/dbnode/digest/digest_mock.go $PACKAGE/src/dbnode/digest ReaderWithDigest" //go:generate sh -c "mockgen -package=series $PACKAGE/src/dbnode/storage/series DatabaseSeries,QueryableBlockRetriever | genclean -pkg $PACKAGE/src/dbnode/storage/series -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/series_mock.go" //go:generate sh -c "mockgen -package=lookup $PACKAGE/src/dbnode/storage/series/lookup OnReleaseReadWriteRef,IndexWriter | genclean -pkg $PACKAGE/src/dbnode/storage/series/lookup -out $GOPATH/src/$PACKAGE/src/dbnode/storage/series/lookup/lookup_mock.go" -//go:generate sh -c "mockgen -package=config $PACKAGE/src/cmd/services/m3dbnode/config BootstrapConfigurationValidator | genclean -pkg $PACKAGE/src/cmd/services/m3dbnode/config -out $GOPATH/src/$PACKAGE/src/cmd/services/m3dbnode/config/config_mock.go" // mockgen rules for generating mocks for unexported interfaces (file mode) //go:generate sh -c "mockgen -package=encoding -destination=$GOPATH/src/$PACKAGE/src/dbnode/encoding/encoding_mock.go -source=$GOPATH/src/$PACKAGE/src/dbnode/encoding/types.go" diff --git a/src/dbnode/namespace/options.go b/src/dbnode/namespace/options.go index b7088e3f6f..fd84acdeb2 100644 --- a/src/dbnode/namespace/options.go +++ b/src/dbnode/namespace/options.go @@ -48,7 +48,8 @@ const ( // Namespace with cold writes disabled by default. defaultColdWritesEnabled = false - // Namespace does not cache retrieved blocks by default. + // Namespace does not cache retrieved blocks by default since this is only + // useful specifically for usage patterns tending towards heavy historical reads. defaultCacheBlocksOnRetrieve = false ) diff --git a/src/dbnode/persist/fs/retriever_test.go b/src/dbnode/persist/fs/retriever_test.go index f176e53323..824e012c96 100644 --- a/src/dbnode/persist/fs/retriever_test.go +++ b/src/dbnode/persist/fs/retriever_test.go @@ -268,7 +268,8 @@ func testBlockRetrieverHighConcurrentSeeks(t *testing.T, shouldCacheShardIndices return block.LeaseState{Volume: 0}, nil }).AnyTimes() - seekerMgr.blockRetrieverOpts = seekerMgr.blockRetrieverOpts.SetBlockLeaseManager(mockBlockLeaseManager) + seekerMgr.blockRetrieverOpts = seekerMgr.blockRetrieverOpts. + SetBlockLeaseManager(mockBlockLeaseManager) // Generate data. for _, shard := range shards { diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 152e81eb6d..2adc9a52d1 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -39,7 +39,6 @@ import ( "github.com/m3db/m3/src/cluster/client/etcd" "github.com/m3db/m3/src/cluster/generated/proto/commonpb" "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/cluster/kv/util" "github.com/m3db/m3/src/cmd/services/m3dbnode/config" queryconfig "github.com/m3db/m3/src/cmd/services/m3query/config" "github.com/m3db/m3/src/dbnode/client" @@ -855,52 +854,13 @@ func Run(runOpts RunOptions) { // recent as the one that triggered the bootstrap, if not newer. // See GitHub issue #1013 for more details. topoMapProvider := newTopoMapProvider(topo) - bs, err := cfg.Bootstrap.New(config.NewBootstrapConfigurationValidator(), - rsOpts, opts, topoMapProvider, origin, m3dbClient) + bs, err := cfg.Bootstrap.New( + rsOpts, opts, topoMapProvider, origin, m3dbClient, + ) if err != nil { logger.Fatal("could not create bootstrap process", zap.Error(err)) } - opts = opts.SetBootstrapProcessProvider(bs) - timeout := bootstrapConfigInitTimeout - - bsGauge := instrument.NewStringListEmitter(scope, "bootstrappers") - if err := bsGauge.Start(cfg.Bootstrap.Bootstrappers); err != nil { - logger.Error("unable to start emitting bootstrap gauge", - zap.Strings("bootstrappers", cfg.Bootstrap.Bootstrappers), - zap.Error(err), - ) - } - defer func() { - if err := bsGauge.Close(); err != nil { - logger.Error("stop emitting bootstrap gauge failed", zap.Error(err)) - } - }() - - kvWatchBootstrappers(syncCfg.KVStore, logger, timeout, cfg.Bootstrap.Bootstrappers, - func(bootstrappers []string) { - if len(bootstrappers) == 0 { - logger.Error("updated bootstrapper list is empty") - return - } - - cfg.Bootstrap.Bootstrappers = bootstrappers - updated, err := cfg.Bootstrap.New(config.NewBootstrapConfigurationValidator(), - rsOpts, opts, topoMapProvider, origin, m3dbClient) - if err != nil { - logger.Error("updated bootstrapper list failed", zap.Error(err)) - return - } - - bs.SetBootstrapperProvider(updated.BootstrapperProvider()) - - if err := bsGauge.UpdateStringList(bootstrappers); err != nil { - logger.Error("unable to update bootstrap gauge with new bootstrappers", - zap.Strings("bootstrappers", bootstrappers), - zap.Error(err), - ) - } - }) // Start the cluster services now that the M3DB client is available. tchannelthriftClusterClose, err := ttcluster.NewServer(m3dbClient, @@ -1315,53 +1275,6 @@ func setEncodersPerBlockLimitOnChange( return runtimeOptsMgr.Update(newRuntimeOpts) } -// this function will block for at most waitTimeout to try to get an initial value -// before we kick off the bootstrap -func kvWatchBootstrappers( - kv kv.Store, - logger *zap.Logger, - waitTimeout time.Duration, - defaultBootstrappers []string, - onUpdate func(bootstrappers []string), -) { - vw, err := kv.Watch(kvconfig.BootstrapperKey) - if err != nil { - logger.Fatal("could not watch value for key with KV", - zap.String("key", kvconfig.BootstrapperKey)) - } - - initializedCh := make(chan struct{}) - - var initialized bool - go func() { - opts := util.NewOptions().SetLogger(logger) - - for range vw.C() { - v, err := util.StringArrayFromValue(vw.Get(), - kvconfig.BootstrapperKey, defaultBootstrappers, opts) - if err != nil { - logger.Error("error converting KV update to string array", - zap.String("key", kvconfig.BootstrapperKey), - zap.Error(err), - ) - continue - } - - onUpdate(v) - - if !initialized { - initialized = true - close(initializedCh) - } - } - }() - - select { - case <-time.After(waitTimeout): - case <-initializedCh: - } -} - func withEncodingAndPoolingOptions( cfg config.DBConfiguration, logger *zap.Logger, diff --git a/src/dbnode/topology/consistency_level.go b/src/dbnode/topology/consistency_level.go index 64dd50b367..88c20fbe13 100644 --- a/src/dbnode/topology/consistency_level.go +++ b/src/dbnode/topology/consistency_level.go @@ -192,16 +192,14 @@ func (l *ConnectConsistencyLevel) UnmarshalYAML(unmarshal func(interface{}) erro if str == "" { return errClusterConnectConsistencyLevelUnspecified } - strs := make([]string, 0, len(validConnectConsistencyLevels)) for _, valid := range validConnectConsistencyLevels { if str == valid.String() { *l = valid return nil } - strs = append(strs, "'"+valid.String()+"'") } return fmt.Errorf("invalid ConnectConsistencyLevel '%s' valid types are: %s", - str, strings.Join(strs, ", ")) + str, validConnectConsistencyLevels) } // ReadConsistencyLevel is the consistency level for reading from a cluster diff --git a/src/query/api/v1/handler/database/common.go b/src/query/api/v1/handler/database/common.go index ee2d654fdb..f7960909b9 100644 --- a/src/query/api/v1/handler/database/common.go +++ b/src/query/api/v1/handler/database/common.go @@ -64,24 +64,10 @@ func RegisterRoutes( wrapped(createHandler).ServeHTTP). Methods(CreateHTTPMethod) - r.HandleFunc(ConfigGetBootstrappersURL, wrapped( - NewConfigGetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigGetBootstrappersHTTPMethod) - r.HandleFunc(ConfigSetBootstrappersURL, wrapped( - NewConfigSetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigSetBootstrappersHTTPMethod) - // Register the same handler under two different endpoints. This just makes explaining things in // our documentation easier so we can separate out concepts, but share the underlying code. r.HandleFunc(CreateURL, wrapped(createHandler).ServeHTTP).Methods(CreateHTTPMethod) r.HandleFunc(CreateNamespaceURL, wrapped(createHandler).ServeHTTP).Methods(CreateNamespaceHTTPMethod) - r.HandleFunc(ConfigGetBootstrappersURL, wrapped( - NewConfigGetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigGetBootstrappersHTTPMethod) - r.HandleFunc(ConfigSetBootstrappersURL, wrapped( - NewConfigSetBootstrappersHandler(client, instrumentOpts)).ServeHTTP). - Methods(ConfigSetBootstrappersHTTPMethod) - return nil } diff --git a/src/query/api/v1/handler/database/config_bootstrappers_get.go b/src/query/api/v1/handler/database/config_bootstrappers_get.go deleted file mode 100644 index 6cd3690660..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_get.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "net/http" - - clusterclient "github.com/m3db/m3/src/cluster/client" - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/query/api/v1/handler" - "github.com/m3db/m3/src/query/util/logging" - "github.com/m3db/m3/src/x/instrument" - xhttp "github.com/m3db/m3/src/x/net/http" - - "go.uber.org/zap" -) - -const ( - // ConfigGetBootstrappersURL is the url for the database create handler. - ConfigGetBootstrappersURL = handler.RoutePrefixV1 + "/database/config/bootstrappers" - - // ConfigGetBootstrappersHTTPMethod is the HTTP method used with this resource. - ConfigGetBootstrappersHTTPMethod = http.MethodGet -) - -type configGetBootstrappersHandler struct { - client clusterclient.Client - instrumentOpts instrument.Options -} - -// NewConfigGetBootstrappersHandler returns a new instance of a database create handler. -func NewConfigGetBootstrappersHandler( - client clusterclient.Client, - instrumentOpts instrument.Options, -) http.Handler { - return &configGetBootstrappersHandler{ - client: client, - instrumentOpts: instrumentOpts, - } -} - -func (h *configGetBootstrappersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - logger := logging.WithContext(ctx, h.instrumentOpts) - - store, err := h.client.KV() - if err != nil { - logger.Error("unable to get kv store", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - value, err := store.Get(kvconfig.BootstrapperKey) - if err != nil && err == kv.ErrNotFound { - xhttp.WriteError(w, xhttp.NewError(err, http.StatusNotFound)) - return - } - if err != nil { - logger.Error("unable to get kv key", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - array := new(commonpb.StringArrayProto) - if err := value.Unmarshal(array); err != nil { - logger.Error("unable to unmarshal kv key", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - xhttp.WriteProtoMsgJSONResponse(w, array, logger) -} diff --git a/src/query/api/v1/handler/database/config_bootstrappers_get_test.go b/src/query/api/v1/handler/database/config_bootstrappers_get_test.go deleted file mode 100644 index df46f40f98..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_get_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/x/instrument" - xjson "github.com/m3db/m3/src/x/json" - xtest "github.com/m3db/m3/src/x/test" - - "github.com/gogo/protobuf/proto" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConfigGetBootstrappersHandler(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, mockStore, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigGetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - value := kv.NewMockValue(ctrl) - value.EXPECT(). - Unmarshal(gomock.Any()). - Return(nil). - Do(func(v proto.Message) { - array, ok := v.(*commonpb.StringArrayProto) - require.True(t, ok) - array.Values = []string{"filesystem", "commitlog", "peers", "uninitialized_topology"} - }) - - mockStore.EXPECT(). - Get(kvconfig.BootstrapperKey). - Return(value, nil) - - req := httptest.NewRequest("GET", "/database/config/bootstrappers", nil) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - body, err := ioutil.ReadAll(resp.Body) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - expectedResp := xjson.Map{ - "values": xjson.Array{"filesystem", "commitlog", "peers", "uninitialized_topology"}, - } - - expected := xtest.MustPrettyJSONMap(t, expectedResp) - actual := xtest.MustPrettyJSONString(t, string(body)) - - assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) -} - -func TestConfigGetBootstrappersHandlerNotFound(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, mockStore, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigGetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - mockStore.EXPECT(). - Get(kvconfig.BootstrapperKey). - Return(nil, kv.ErrNotFound) - - req := httptest.NewRequest("GET", "/database/config/bootstrappers", nil) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(t, http.StatusNotFound, resp.StatusCode) -} diff --git a/src/query/api/v1/handler/database/config_bootstrappers_set.go b/src/query/api/v1/handler/database/config_bootstrappers_set.go deleted file mode 100644 index fb98853c98..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_set.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "fmt" - "net/http" - - clusterclient "github.com/m3db/m3/src/cluster/client" - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - dbconfig "github.com/m3db/m3/src/cmd/services/m3dbnode/config" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/query/api/v1/handler" - "github.com/m3db/m3/src/query/util/logging" - xerrors "github.com/m3db/m3/src/x/errors" - "github.com/m3db/m3/src/x/instrument" - xhttp "github.com/m3db/m3/src/x/net/http" - - "github.com/gogo/protobuf/jsonpb" - "go.uber.org/zap" -) - -const ( - // ConfigSetBootstrappersURL is the url for the database create handler. - ConfigSetBootstrappersURL = handler.RoutePrefixV1 + "/database/config/bootstrappers" - - // ConfigSetBootstrappersHTTPMethod is the HTTP method used with this resource. - ConfigSetBootstrappersHTTPMethod = http.MethodPost -) - -type configSetBootstrappersHandler struct { - client clusterclient.Client - instrumentOpts instrument.Options -} - -// NewConfigSetBootstrappersHandler returns a new instance of a database create handler. -func NewConfigSetBootstrappersHandler( - client clusterclient.Client, - instrumentOpts instrument.Options, -) http.Handler { - return &configSetBootstrappersHandler{ - client: client, - instrumentOpts: instrumentOpts, - } -} - -func (h *configSetBootstrappersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - logger := logging.WithContext(ctx, h.instrumentOpts) - - value, rErr := h.parseRequest(r) - if rErr != nil { - logger.Error("unable to parse request", zap.Error(rErr)) - xhttp.WriteError(w, rErr) - return - } - - store, err := h.client.KV() - if err != nil { - logger.Error("unable to get kv store", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - if _, err := store.Set(kvconfig.BootstrapperKey, value); err != nil { - logger.Error("unable to set kv key", zap.Error(err)) - xhttp.WriteError(w, err) - return - } - - xhttp.WriteProtoMsgJSONResponse(w, value, logger) -} - -func (h *configSetBootstrappersHandler) parseRequest( - r *http.Request, -) (*commonpb.StringArrayProto, error) { - array := new(commonpb.StringArrayProto) - - defer r.Body.Close() - - if err := jsonpb.Unmarshal(r.Body, array); err != nil { - return nil, xerrors.NewInvalidParamsError(err) - } - - if len(array.Values) == 0 { - return nil, xerrors.NewInvalidParamsError(fmt.Errorf("no values")) - } - - validator := dbconfig.NewBootstrapConfigurationValidator() - if err := validator.ValidateBootstrappersOrder(array.Values); err != nil { - return nil, xerrors.NewInvalidParamsError(err) - } - - return array, nil -} diff --git a/src/query/api/v1/handler/database/config_bootstrappers_set_test.go b/src/query/api/v1/handler/database/config_bootstrappers_set_test.go deleted file mode 100644 index 8ae3a3fe8e..0000000000 --- a/src/query/api/v1/handler/database/config_bootstrappers_set_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package database - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/dbnode/kvconfig" - "github.com/m3db/m3/src/x/instrument" - xjson "github.com/m3db/m3/src/x/json" - xtest "github.com/m3db/m3/src/x/test" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConfigSetBootstrappersHandler(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, mockStore, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigSetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - jsonInput := xjson.Map{ - "values": xjson.Array{"filesystem", "commitlog", "peers", "uninitialized_topology"}, - } - - mockStore.EXPECT(). - Set(kvconfig.BootstrapperKey, gomock.Any()). - Return(int(1), nil). - Do(func(key string, value *commonpb.StringArrayProto) { - assert.Equal(t, []string{ - "filesystem", "commitlog", "peers", "uninitialized_topology", - }, value.Values) - }) - - req := httptest.NewRequest("POST", "/database/config/bootstrappers", - xjson.MustNewTestReader(t, jsonInput)) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - body, err := ioutil.ReadAll(resp.Body) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, resp.StatusCode) - - expectedResp := xjson.Map{ - "values": xjson.Array{"filesystem", "commitlog", "peers", "uninitialized_topology"}, - } - - expected := xtest.MustPrettyJSONMap(t, expectedResp) - actual := xtest.MustPrettyJSONString(t, string(body)) - - assert.Equal(t, expected, actual, xtest.Diff(expected, actual)) -} - -func TestConfigSetBootstrappersHandlerNoValues(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, _, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigSetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - jsonInput := xjson.Map{ - "values": xjson.Array{}, - } - - req := httptest.NewRequest("POST", "/database/config/bootstrappers", - xjson.MustNewTestReader(t, jsonInput)) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) -} - -func TestConfigSetBootstrappersHandlerInvalidValue(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockClient, _, _ := SetupDatabaseTest(t, ctrl) - handler := NewConfigSetBootstrappersHandler(mockClient, - instrument.NewOptions()) - w := httptest.NewRecorder() - - jsonInput := xjson.Map{ - "values": xjson.Array{"filesystem", "foo"}, - } - - req := httptest.NewRequest("POST", "/database/config/bootstrappers", - xjson.MustNewTestReader(t, jsonInput)) - require.NotNil(t, req) - - handler.ServeHTTP(w, req) - - resp := w.Result() - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) -} diff --git a/src/query/api/v1/handler/graphite/render.go b/src/query/api/v1/handler/graphite/render.go index 485e4d93e8..f5117aff51 100644 --- a/src/query/api/v1/handler/graphite/render.go +++ b/src/query/api/v1/handler/graphite/render.go @@ -68,7 +68,7 @@ type respError struct { // NewRenderHandler returns a new render handler around the given storage. func NewRenderHandler(opts options.HandlerOptions) http.Handler { wrappedStore := graphite.NewM3WrappedStorage(opts.Storage(), - opts.Enforcer(), opts.M3DBOptions(), opts.InstrumentOpts(), opts.GraphiteStorageOptions()) + opts.M3DBOptions(), opts.InstrumentOpts(), opts.GraphiteStorageOptions()) return &renderHandler{ engine: native.NewEngine(wrappedStore), queryContextOpts: opts.QueryContextOptions(), diff --git a/src/query/api/v1/handler/namespace/add.go b/src/query/api/v1/handler/namespace/add.go index e62527a74c..85ebdf108c 100644 --- a/src/query/api/v1/handler/namespace/add.go +++ b/src/query/api/v1/handler/namespace/add.go @@ -43,10 +43,6 @@ import ( ) var ( - // DeprecatedM3DBAddURL is the old url for the namespace add handler, maintained - // for backwards compatibility. - DeprecatedM3DBAddURL = path.Join(handler.RoutePrefixV1, NamespacePathName) - // M3DBAddURL is the url for the M3DB namespace add handler. M3DBAddURL = path.Join(handler.RoutePrefixV1, M3DBServiceNamespacePathName) diff --git a/src/query/api/v1/handler/namespace/common.go b/src/query/api/v1/handler/namespace/common.go index 969e8c0151..4e95e121ef 100644 --- a/src/query/api/v1/handler/namespace/common.go +++ b/src/query/api/v1/handler/namespace/common.go @@ -130,13 +130,11 @@ func RegisterRoutes( // Get M3DB namespaces. getHandler := wrapped( applyMiddleware(NewGetHandler(client, instrumentOpts).ServeHTTP, defaults)) - r.HandleFunc(DeprecatedM3DBGetURL, getHandler.ServeHTTP).Methods(GetHTTPMethod) r.HandleFunc(M3DBGetURL, getHandler.ServeHTTP).Methods(GetHTTPMethod) // Add M3DB namespaces. addHandler := wrapped( applyMiddleware(NewAddHandler(client, instrumentOpts).ServeHTTP, defaults)) - r.HandleFunc(DeprecatedM3DBAddURL, addHandler.ServeHTTP).Methods(AddHTTPMethod) r.HandleFunc(M3DBAddURL, addHandler.ServeHTTP).Methods(AddHTTPMethod) // Update M3DB namespaces. @@ -147,7 +145,6 @@ func RegisterRoutes( // Delete M3DB namespaces. deleteHandler := wrapped( applyMiddleware(NewDeleteHandler(client, instrumentOpts).ServeHTTP, defaults)) - r.HandleFunc(DeprecatedM3DBDeleteURL, deleteHandler.ServeHTTP).Methods(DeleteHTTPMethod) r.HandleFunc(M3DBDeleteURL, deleteHandler.ServeHTTP).Methods(DeleteHTTPMethod) // Deploy M3DB schemas. diff --git a/src/query/api/v1/handler/namespace/delete.go b/src/query/api/v1/handler/namespace/delete.go index f0b10b10d9..c974c13bd5 100644 --- a/src/query/api/v1/handler/namespace/delete.go +++ b/src/query/api/v1/handler/namespace/delete.go @@ -49,10 +49,6 @@ const ( ) var ( - // DeprecatedM3DBDeleteURL is the deprecated url for the M3DB namespace delete handler. - // Maintained for backwards compatibility. - DeprecatedM3DBDeleteURL = fmt.Sprintf("%s/namespace/{%s}", handler.RoutePrefixV1, namespaceIDVar) - // M3DBDeleteURL is the url for the M3DB namespace delete handler. M3DBDeleteURL = path.Join( handler.RoutePrefixV1, diff --git a/src/query/api/v1/handler/namespace/get.go b/src/query/api/v1/handler/namespace/get.go index 1576d5d518..ce156be2fb 100644 --- a/src/query/api/v1/handler/namespace/get.go +++ b/src/query/api/v1/handler/namespace/get.go @@ -43,10 +43,6 @@ import ( ) var ( - // DeprecatedM3DBGetURL is the deprecated url for the namespace get handler (with the GET method). - // Maintained for backwards compatibility. - DeprecatedM3DBGetURL = path.Join(handler.RoutePrefixV1, NamespacePathName) - // M3DBGetURL is the url for the namespace get handler (with the GET method). M3DBGetURL = path.Join(handler.RoutePrefixV1, M3DBServiceNamespacePathName) diff --git a/src/query/api/v1/handler/placement/add.go b/src/query/api/v1/handler/placement/add.go index 74195fa714..0cc7a581cd 100644 --- a/src/query/api/v1/handler/placement/add.go +++ b/src/query/api/v1/handler/placement/add.go @@ -43,10 +43,6 @@ const ( ) var ( - // DeprecatedM3DBAddURL is the old url for the placement add handler, maintained for - // backwards compatibility. - DeprecatedM3DBAddURL = path.Join(handler.RoutePrefixV1, PlacementPathName) - // M3DBAddURL is the url for the placement add handler (with the POST method) // for the M3DB service. M3DBAddURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName) diff --git a/src/query/api/v1/handler/placement/common.go b/src/query/api/v1/handler/placement/common.go index f7ae91c858..66d732442c 100644 --- a/src/query/api/v1/handler/placement/common.go +++ b/src/query/api/v1/handler/placement/common.go @@ -228,55 +228,45 @@ func RegisterRoutes( ) { // Init var ( - initHandler = NewInitHandler(opts) - deprecatedInitFn = applyDeprecatedMiddleware(initHandler.ServeHTTP, defaults, opts.instrumentOptions) - initFn = applyMiddleware(initHandler.ServeHTTP, defaults, opts.instrumentOptions) + initHandler = NewInitHandler(opts) + initFn = applyMiddleware(initHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBInitURL, deprecatedInitFn).Methods(InitHTTPMethod) r.HandleFunc(M3DBInitURL, initFn).Methods(InitHTTPMethod) r.HandleFunc(M3AggInitURL, initFn).Methods(InitHTTPMethod) r.HandleFunc(M3CoordinatorInitURL, initFn).Methods(InitHTTPMethod) // Get var ( - getHandler = NewGetHandler(opts) - deprecatedGetFn = applyDeprecatedMiddleware(getHandler.ServeHTTP, defaults, opts.instrumentOptions) - getFn = applyMiddleware(getHandler.ServeHTTP, defaults, opts.instrumentOptions) + getHandler = NewGetHandler(opts) + getFn = applyMiddleware(getHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBGetURL, deprecatedGetFn).Methods(GetHTTPMethod) r.HandleFunc(M3DBGetURL, getFn).Methods(GetHTTPMethod) r.HandleFunc(M3AggGetURL, getFn).Methods(GetHTTPMethod) r.HandleFunc(M3CoordinatorGetURL, getFn).Methods(GetHTTPMethod) // Delete all var ( - deleteAllHandler = NewDeleteAllHandler(opts) - deprecatedDeleteAllFn = applyDeprecatedMiddleware(deleteAllHandler.ServeHTTP, defaults, opts.instrumentOptions) - deleteAllFn = applyMiddleware(deleteAllHandler.ServeHTTP, defaults, opts.instrumentOptions) + deleteAllHandler = NewDeleteAllHandler(opts) + deleteAllFn = applyMiddleware(deleteAllHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBDeleteAllURL, deprecatedDeleteAllFn).Methods(DeleteAllHTTPMethod) r.HandleFunc(M3DBDeleteAllURL, deleteAllFn).Methods(DeleteAllHTTPMethod) r.HandleFunc(M3AggDeleteAllURL, deleteAllFn).Methods(DeleteAllHTTPMethod) r.HandleFunc(M3CoordinatorDeleteAllURL, deleteAllFn).Methods(DeleteAllHTTPMethod) // Add var ( - addHandler = NewAddHandler(opts) - deprecatedAddFn = applyDeprecatedMiddleware(addHandler.ServeHTTP, defaults, opts.instrumentOptions) - addFn = applyMiddleware(addHandler.ServeHTTP, defaults, opts.instrumentOptions) + addHandler = NewAddHandler(opts) + addFn = applyMiddleware(addHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBAddURL, deprecatedAddFn).Methods(AddHTTPMethod) r.HandleFunc(M3DBAddURL, addFn).Methods(AddHTTPMethod) r.HandleFunc(M3AggAddURL, addFn).Methods(AddHTTPMethod) r.HandleFunc(M3CoordinatorAddURL, addFn).Methods(AddHTTPMethod) // Delete var ( - deleteHandler = NewDeleteHandler(opts) - deprecatedDeleteFn = applyDeprecatedMiddleware(deleteHandler.ServeHTTP, defaults, opts.instrumentOptions) - deleteFn = applyMiddleware(deleteHandler.ServeHTTP, defaults, opts.instrumentOptions) + deleteHandler = NewDeleteHandler(opts) + deleteFn = applyMiddleware(deleteHandler.ServeHTTP, defaults, opts.instrumentOptions) ) - r.HandleFunc(DeprecatedM3DBDeleteURL, deprecatedDeleteFn).Methods(DeleteHTTPMethod) r.HandleFunc(M3DBDeleteURL, deleteFn).Methods(DeleteHTTPMethod) r.HandleFunc(M3AggDeleteURL, deleteFn).Methods(DeleteHTTPMethod) r.HandleFunc(M3CoordinatorDeleteURL, deleteFn).Methods(DeleteHTTPMethod) @@ -403,23 +393,6 @@ func applyMiddleware( ).ServeHTTP } -func applyDeprecatedMiddleware( - f func(svc handleroptions.ServiceNameAndDefaults, w http.ResponseWriter, r *http.Request), - defaults []handleroptions.ServiceOptionsDefault, - instrumentOpts instrument.Options, -) func(w http.ResponseWriter, r *http.Request) { - return logging.WithResponseTimeAndPanicErrorLoggingFunc( - func(w http.ResponseWriter, r *http.Request) { - svc := handleroptions.ServiceNameAndDefaults{ - ServiceName: handleroptions.M3DBServiceName, - Defaults: defaults, - } - f(svc, w, r) - }, - instrumentOpts, - ).ServeHTTP -} - func parseServiceMiddleware( next func(svc handleroptions.ServiceNameAndDefaults, w http.ResponseWriter, r *http.Request), defaults []handleroptions.ServiceOptionsDefault, diff --git a/src/query/api/v1/handler/placement/delete.go b/src/query/api/v1/handler/placement/delete.go index 73e81baad5..140514679f 100644 --- a/src/query/api/v1/handler/placement/delete.go +++ b/src/query/api/v1/handler/placement/delete.go @@ -50,10 +50,6 @@ const ( var ( placementIDPath = fmt.Sprintf("{%s}", placementIDVar) - // DeprecatedM3DBDeleteURL is the old url for the placement delete handler, maintained - // for backwards compatibility. - DeprecatedM3DBDeleteURL = path.Join(handler.RoutePrefixV1, PlacementPathName, placementIDPath) - // M3DBDeleteURL is the url for the placement delete handler for the M3DB service. M3DBDeleteURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName, placementIDPath) diff --git a/src/query/api/v1/handler/placement/delete_all.go b/src/query/api/v1/handler/placement/delete_all.go index 7cc07581eb..a5cf100223 100644 --- a/src/query/api/v1/handler/placement/delete_all.go +++ b/src/query/api/v1/handler/placement/delete_all.go @@ -41,10 +41,6 @@ const ( ) var ( - // DeprecatedM3DBDeleteAllURL is the old url for the handler to delete all placements, maintained - // for backwards compatibility. - DeprecatedM3DBDeleteAllURL = path.Join(handler.RoutePrefixV1, PlacementPathName) - // M3DBDeleteAllURL is the url for the handler to delete all placements (with the DELETE method) // for the M3DB service. M3DBDeleteAllURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName) diff --git a/src/query/api/v1/handler/placement/get.go b/src/query/api/v1/handler/placement/get.go index caaa53416e..0b453f2fa2 100644 --- a/src/query/api/v1/handler/placement/get.go +++ b/src/query/api/v1/handler/placement/get.go @@ -46,10 +46,6 @@ const ( ) var ( - // DeprecatedM3DBGetURL is the old url for the placement get handler, maintained for - // backwards compatibility. - DeprecatedM3DBGetURL = path.Join(handler.RoutePrefixV1, PlacementPathName) - // M3DBGetURL is the url for the placement get handler (with the GET method) // for the M3DB service. M3DBGetURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName) diff --git a/src/query/api/v1/handler/placement/init.go b/src/query/api/v1/handler/placement/init.go index 79fdea7266..00a85e56b8 100644 --- a/src/query/api/v1/handler/placement/init.go +++ b/src/query/api/v1/handler/placement/init.go @@ -43,10 +43,6 @@ const ( ) var ( - // DeprecatedM3DBInitURL is the old url for the placement init handler, maintained for backwards - // compatibility. (with the POST method). - DeprecatedM3DBInitURL = path.Join(handler.RoutePrefixV1, PlacementPathName, initPathName) - // M3DBInitURL is the url for the placement init handler, (with the POST method). M3DBInitURL = path.Join(handler.RoutePrefixV1, M3DBServicePlacementPathName, initPathName) diff --git a/src/query/api/v1/handler/prom/read_test.go b/src/query/api/v1/handler/prom/read_test.go index 392a664b5d..0d9b521aae 100644 --- a/src/query/api/v1/handler/prom/read_test.go +++ b/src/query/api/v1/handler/prom/read_test.go @@ -64,7 +64,6 @@ func setupTest(t *testing.T) testHandlers { instrumentOpts := instrument.NewOptions() engineOpts := executor.NewEngineOptions(). SetLookbackDuration(time.Minute). - SetGlobalEnforcer(nil). SetInstrumentOptions(instrumentOpts) engine := executor.NewEngine(engineOpts) hOpts := options.EmptyHandlerOptions(). diff --git a/src/query/api/v1/handler/prometheus/native/common.go b/src/query/api/v1/handler/prometheus/native/common.go index dd5867fa58..59ec8630e0 100644 --- a/src/query/api/v1/handler/prometheus/native/common.go +++ b/src/query/api/v1/handler/prometheus/native/common.go @@ -26,7 +26,6 @@ import ( "math" "net/http" "strconv" - "strings" "time" "github.com/m3db/m3/src/query/api/v1/handler/prometheus" @@ -41,7 +40,6 @@ import ( "github.com/m3db/m3/src/query/util" "github.com/m3db/m3/src/query/util/json" xerrors "github.com/m3db/m3/src/x/errors" - "github.com/m3db/m3/src/x/headers" ) const ( @@ -170,10 +168,6 @@ func parseParams( params.IncludeEnd = !excludeEnd } - if strings.ToLower(r.Header.Get(headers.RenderFormat)) == "m3ql" { - params.FormatType = models.FormatM3QL - } - params.LookbackDuration = engineOpts.LookbackDuration() if v := fetchOpts.LookbackDuration; v != nil { params.LookbackDuration = *v @@ -436,47 +430,3 @@ func renderResultsInstantaneousJSON( jw.EndObject() jw.Close() } - -func renderM3QLResultsJSON( - w io.Writer, - series []*ts.Series, - params models.RequestParams, -) { - jw := json.NewWriter(w) - jw.BeginArray() - - for _, s := range series { - jw.BeginObject() - jw.BeginObjectField("target") - jw.WriteString(string(s.Name())) - - jw.BeginObjectField("tags") - jw.BeginObject() - - for _, tag := range s.Tags.Tags { - jw.BeginObjectField(string(tag.Name)) - jw.WriteString(string(tag.Value)) - } - - jw.EndObject() - - jw.BeginObjectField("datapoints") - jw.BeginArray() - for i := 0; i < s.Len(); i++ { - dp := s.Values().DatapointAt(i) - jw.BeginArray() - jw.WriteFloat64(dp.Value) - jw.WriteInt(int(dp.Timestamp.Unix())) - jw.EndArray() - } - jw.EndArray() - - jw.BeginObjectField("step_size_ms") - jw.WriteInt(int(params.Step.Seconds() * 1000)) - - jw.EndObject() - } - - jw.EndArray() - jw.Close() -} diff --git a/src/query/api/v1/handler/prometheus/native/read.go b/src/query/api/v1/handler/prometheus/native/read.go index c4235e554e..9db53457b1 100644 --- a/src/query/api/v1/handler/prometheus/native/read.go +++ b/src/query/api/v1/handler/prometheus/native/read.go @@ -27,7 +27,6 @@ import ( "github.com/m3db/m3/src/query/api/v1/handler" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" "github.com/m3db/m3/src/query/api/v1/options" - "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/util/logging" xhttp "github.com/m3db/m3/src/x/net/http" xopentracing "github.com/m3db/m3/src/x/opentracing" @@ -93,9 +92,6 @@ func newHandler(opts options.HandlerOptions, instant bool) http.Handler { opts: opts, instant: instant, } - - maxDatapoints := opts.Config().Limits.MaxComputedDatapoints() - h.promReadMetrics.maxDatapoints.Update(float64(maxDatapoints)) return h } @@ -150,11 +146,6 @@ func (h *promReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if parsedOptions.Params.FormatType == models.FormatM3QL { - renderM3QLResultsJSON(w, result.Series, parsedOptions.Params) - return - } - err = RenderResultsJSON(w, result, RenderResultsOptions{ Start: parsedOptions.Params.Start, End: parsedOptions.Params.End, diff --git a/src/query/api/v1/handler/prometheus/native/read_common.go b/src/query/api/v1/handler/prometheus/native/read_common.go index 57ac37137f..44cb14662a 100644 --- a/src/query/api/v1/handler/prometheus/native/read_common.go +++ b/src/query/api/v1/handler/prometheus/native/read_common.go @@ -22,7 +22,6 @@ package native import ( "context" - "fmt" "math" "net/http" @@ -47,7 +46,6 @@ type promReadMetrics struct { fetchErrorsServer tally.Counter fetchErrorsClient tally.Counter fetchTimerSuccess tally.Timer - maxDatapoints tally.Gauge } func newPromReadMetrics(scope tally.Scope) promReadMetrics { @@ -58,7 +56,6 @@ func newPromReadMetrics(scope tally.Scope) promReadMetrics { fetchErrorsClient: scope.Tagged(map[string]string{"code": "4XX"}). Counter("fetch.errors"), fetchTimerSuccess: scope.Timer("fetch.success.latency"), - maxDatapoints: scope.Gauge("max_datapoints"), } } @@ -131,11 +128,6 @@ func parseRequest( return ParsedOptions{}, err } - maxPoints := opts.Config().Limits.MaxComputedDatapoints() - if err := validateRequest(params, maxPoints); err != nil { - return ParsedOptions{}, err - } - return ParsedOptions{ QueryOpts: queryOpts, FetchOpts: fetchOpts, @@ -143,24 +135,6 @@ func parseRequest( }, nil } -func validateRequest(params models.RequestParams, maxPoints int) error { - // Impose a rough limit on the number of returned time series. - // This is intended to prevent things like querying from the beginning of - // time with a 1s step size. - numSteps := int(params.End.Sub(params.Start) / params.Step) - if maxPoints > 0 && numSteps > maxPoints { - return fmt.Errorf( - "querying from %v to %v with step size %v would result in too many "+ - "datapoints (end - start / step > %d). Either decrease the query "+ - "resolution (?step=XX), decrease the time window, or increase "+ - "the limit (`limits.maxComputedDatapoints`)", - params.Start, params.End, params.Step, maxPoints, - ) - } - - return nil -} - // ParsedOptions are parsed options for the query. type ParsedOptions struct { QueryOpts *executor.QueryOptions diff --git a/src/query/api/v1/handler/prometheus/native/read_test.go b/src/query/api/v1/handler/prometheus/native/read_test.go index 10cb2e5e56..5ae88efae2 100644 --- a/src/query/api/v1/handler/prometheus/native/read_test.go +++ b/src/query/api/v1/handler/prometheus/native/read_test.go @@ -22,10 +22,7 @@ package native import ( "context" - "encoding/json" - "io/ioutil" "net/http" - "net/http/httptest" "net/url" "testing" "time" @@ -171,64 +168,6 @@ func testPromReadHandlerRead( } } -type M3QLResp []struct { - Target string `json:"target"` - Tags map[string]string `json:"tags"` - Datapoints [][]float64 `json:"datapoints"` - StepSizeMs int `json:"step_size_ms"` -} - -func TestM3PromReadHandlerRead(t *testing.T) { - testM3PromReadHandlerRead(t, block.NewResultMetadata(), "") - testM3PromReadHandlerRead(t, buildWarningMeta("foo", "bar"), "foo_bar") - testM3PromReadHandlerRead(t, block.ResultMetadata{Exhaustive: false}, - headers.LimitHeaderSeriesLimitApplied) -} - -func testM3PromReadHandlerRead( - t *testing.T, - resultMeta block.ResultMetadata, - ex string, -) { - values, bounds := test.GenerateValuesAndBounds(nil, nil) - - setup := newTestSetup(timeoutOpts, nil) - promRead := setup.Handlers.read - - seriesMeta := test.NewSeriesMeta("dummy", len(values)) - meta := block.Metadata{ - Bounds: bounds, - Tags: models.NewTags(0, models.NewTagOptions()), - ResultMetadata: resultMeta, - } - - b := test.NewBlockFromValuesWithMetaAndSeriesMeta(meta, seriesMeta, values) - setup.Storage.SetFetchBlocksResult(block.Result{Blocks: []block.Block{b}}, nil) - - req, _ := http.NewRequest("GET", PromReadURL, nil) - req.Header.Add(headers.RenderFormat, "m3ql") - req.URL.RawQuery = defaultParams().Encode() - - recorder := httptest.NewRecorder() - promRead.ServeHTTP(recorder, req) - - header := recorder.Header().Get(headers.LimitHeader) - assert.Equal(t, ex, header) - - var m3qlResp M3QLResp - require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &m3qlResp)) - - assert.Len(t, m3qlResp, 2) - assert.Equal(t, "dummy0", m3qlResp[0].Target) - assert.Equal(t, map[string]string{"__name__": "dummy0", "dummy0": "dummy0"}, - m3qlResp[0].Tags) - assert.Equal(t, 10000, m3qlResp[0].StepSizeMs) - assert.Equal(t, "dummy1", m3qlResp[1].Target) - assert.Equal(t, map[string]string{"__name__": "dummy1", "dummy1": "dummy1"}, - m3qlResp[1].Tags) - assert.Equal(t, 10000, m3qlResp[1].StepSizeMs) -} - func newReadRequest(t *testing.T, params url.Values) *http.Request { req, err := http.NewRequest("GET", PromReadURL, nil) require.NoError(t, err) @@ -260,7 +199,6 @@ func newTestSetup( engineOpts := executor.NewEngineOptions(). SetStore(mockStorage). SetLookbackDuration(time.Minute). - SetGlobalEnforcer(nil). SetInstrumentOptions(instrumentOpts) engine := executor.NewEngine(engineOpts) if mockEngine != nil { @@ -301,133 +239,6 @@ func newTestSetup( } } -func TestPromReadHandlerServeHTTPMaxComputedDatapoints(t *testing.T) { - setup := newTestSetup(timeoutOpts, nil) - opts := setup.Handlers.read.opts - setup.Handlers.read.opts = opts.SetConfig(config.Configuration{ - Limits: config.LimitsConfiguration{ - PerQuery: config.PerQueryLimitsConfiguration{ - PrivateMaxComputedDatapoints: 3599, - }, - }, - }) - - params := defaultParams() - params.Set(startParam, time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC). - Format(time.RFC3339Nano)) - params.Set(endParam, time.Date(2018, 1, 1, 1, 0, 0, 0, time.UTC). - Format(time.RFC3339Nano)) - params.Set(handleroptions.StepParam, (time.Second).String()) - req := newReadRequest(t, params) - - recorder := httptest.NewRecorder() - setup.Handlers.read.ServeHTTP(recorder, req) - resp := recorder.Result() - - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - - d, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - // not a public struct in xhttp, but it's small. - var errResp struct { - Error string `json:"error"` - } - require.NoError(t, json.Unmarshal(d, &errResp)) - - expected := "querying from 2018-01-01 00:00:00 +0000 UTC to 2018-01-01 01:00:00 +0000 UTC with step size 1s " + - "would result in too many datapoints (end - start / step > 3599). Either decrease the query resolution " + - "(?step=XX), decrease the time window, or increase the limit (`limits.maxComputedDatapoints`)" - assert.Equal(t, expected, errResp.Error) -} - -func TestPromReadHandler_validateRequest(t *testing.T) { - dt := func(year int, month time.Month, day, hour int) time.Time { - return time.Date(year, month, day, hour, 0, 0, 0, time.UTC) - } - - cases := []struct { - name string - params models.RequestParams - max int - errorExpected bool - }{{ - name: "under limit", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 3601, - errorExpected: false, - }, { - name: "at limit", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 3600, - errorExpected: false, - }, { - name: "over limit", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 3599, - errorExpected: true, - }, { - name: "large query, limit disabled (0)", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 0, - errorExpected: false, - }, { - name: "large query, limit disabled (negative)", - params: models.RequestParams{ - Step: time.Second, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: -50, - errorExpected: false, - }, { - name: "uneven step over limit", - params: models.RequestParams{ - Step: 34 * time.Minute, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 11), - }, - max: 1, - errorExpected: true, - }, { - name: "uneven step under limit", - params: models.RequestParams{ - Step: 34 * time.Minute, - Start: dt(2018, 1, 1, 0), - End: dt(2018, 1, 1, 1), - }, - max: 2, - errorExpected: false}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := validateRequest(tc.params, tc.max) - if tc.errorExpected { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - type cancelWatcher struct { delay time.Duration } diff --git a/src/query/api/v1/handler/prometheus/remote/read_test.go b/src/query/api/v1/handler/prometheus/remote/read_test.go index aebaebcda2..c8c2504211 100644 --- a/src/query/api/v1/handler/prometheus/remote/read_test.go +++ b/src/query/api/v1/handler/prometheus/remote/read_test.go @@ -41,7 +41,6 @@ import ( "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" "github.com/m3db/m3/src/query/api/v1/options" "github.com/m3db/m3/src/query/block" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" @@ -132,13 +131,11 @@ func TestParseExpr(t *testing.T) { func newEngine( s storage.Storage, lookbackDuration time.Duration, - enforcer qcost.ChainedEnforcer, instrumentOpts instrument.Options, ) executor.Engine { engineOpts := executor.NewEngineOptions(). SetStore(s). SetLookbackDuration(lookbackDuration). - SetGlobalEnforcer(enforcer). SetInstrumentOptions(instrumentOpts) return executor.NewEngine(engineOpts) @@ -167,7 +164,7 @@ func readHandler(store storage.Storage, }, } iOpts := instrument.NewOptions() - engine := newEngine(store, defaultLookbackDuration, nil, iOpts) + engine := newEngine(store, defaultLookbackDuration, iOpts) opts := options.EmptyHandlerOptions(). SetEngine(engine). SetInstrumentOpts(iOpts). @@ -185,7 +182,7 @@ func TestPromReadParsing(t *testing.T) { SeriesLimit: 100, }, } - engine := newEngine(storage, defaultLookbackDuration, nil, + engine := newEngine(storage, defaultLookbackDuration, instrument.NewOptions()) opts := options.EmptyHandlerOptions(). @@ -297,7 +294,7 @@ func TestReadErrorMetricsCount(t *testing.T) { SeriesLimit: 100, }, } - engine := newEngine(storage, defaultLookbackDuration, nil, + engine := newEngine(storage, defaultLookbackDuration, instrument.NewOptions()) opts := options.EmptyHandlerOptions(). SetEngine(engine). diff --git a/src/query/api/v1/httpd/handler_test.go b/src/query/api/v1/httpd/handler_test.go index 0a23f88c27..f0e64b3c65 100644 --- a/src/query/api/v1/httpd/handler_test.go +++ b/src/query/api/v1/httpd/handler_test.go @@ -37,7 +37,6 @@ import ( "github.com/m3db/m3/src/query/api/v1/handler/prometheus/native" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/remote" "github.com/m3db/m3/src/query/api/v1/options" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor" graphite "github.com/m3db/m3/src/query/graphite/storage" "github.com/m3db/m3/src/query/models" @@ -75,13 +74,11 @@ func makeTagOptions() models.TagOptions { func newEngine( s storage.Storage, lookbackDuration time.Duration, - enforcer qcost.ChainedEnforcer, instrumentOpts instrument.Options, ) executor.Engine { engineOpts := executor.NewEngineOptions(). SetStore(s). SetLookbackDuration(lookbackDuration). - SetGlobalEnforcer(enforcer). SetInstrumentOptions(instrumentOpts) return executor.NewEngine(engineOpts) @@ -93,7 +90,7 @@ func setupHandler( ) (*Handler, error) { instrumentOpts := instrument.NewOptions() downsamplerAndWriter := ingest.NewDownsamplerAndWriter(store, nil, testWorkerPool, instrument.NewOptions()) - engine := newEngine(store, time.Minute, nil, instrumentOpts) + engine := newEngine(store, time.Minute, instrumentOpts) opts, err := options.NewHandlerOptions( downsamplerAndWriter, makeTagOptions(), @@ -103,7 +100,6 @@ func setupHandler( nil, config.Configuration{LookbackDuration: &defaultLookbackDuration}, nil, - nil, handleroptions.NewFetchOptionsBuilder(handleroptions.FetchOptionsBuilderOptions{}), models.QueryContextOptions{}, instrumentOpts, @@ -130,7 +126,7 @@ func TestHandlerFetchTimeout(t *testing.T) { fourMin := 4 * time.Minute dbconfig := &dbconfig.DBConfiguration{Client: client.Configuration{FetchTimeout: &fourMin}} - engine := newEngine(storage, time.Minute, nil, instrument.NewOptions()) + engine := newEngine(storage, time.Minute, instrument.NewOptions()) cfg := config.Configuration{LookbackDuration: &defaultLookbackDuration} opts, err := options.NewHandlerOptions( downsamplerAndWriter, @@ -141,7 +137,6 @@ func TestHandlerFetchTimeout(t *testing.T) { nil, cfg, dbconfig, - nil, handleroptions.NewFetchOptionsBuilder(handleroptions.FetchOptionsBuilderOptions{}), models.QueryContextOptions{}, instrument.NewOptions(), @@ -406,10 +401,10 @@ func TestCustomRoutes(t *testing.T) { store, _ := m3.NewStorageAndSession(t, ctrl) instrumentOpts := instrument.NewOptions() downsamplerAndWriter := ingest.NewDownsamplerAndWriter(store, nil, testWorkerPool, instrument.NewOptions()) - engine := newEngine(store, time.Minute, nil, instrumentOpts) + engine := newEngine(store, time.Minute, instrumentOpts) opts, err := options.NewHandlerOptions( downsamplerAndWriter, makeTagOptions().SetMetricName([]byte("z")), engine, nil, nil, nil, - config.Configuration{LookbackDuration: &defaultLookbackDuration}, nil, nil, + config.Configuration{LookbackDuration: &defaultLookbackDuration}, nil, handleroptions.NewFetchOptionsBuilder(handleroptions.FetchOptionsBuilderOptions{}), models.QueryContextOptions{}, instrumentOpts, defaultCPUProfileduration, defaultPlacementServices, svcDefaultOptions, NewQueryRouter(), NewQueryRouter(), diff --git a/src/query/api/v1/options/handler.go b/src/query/api/v1/options/handler.go index 4ab364b969..15eab3f99f 100644 --- a/src/query/api/v1/options/handler.go +++ b/src/query/api/v1/options/handler.go @@ -32,7 +32,6 @@ import ( "github.com/m3db/m3/src/cmd/services/m3query/config" "github.com/m3db/m3/src/query/api/v1/handler/prometheus" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor" graphite "github.com/m3db/m3/src/query/graphite/storage" "github.com/m3db/m3/src/query/models" @@ -147,11 +146,6 @@ type HandlerOptions interface { // SetTimeoutOpts sets the timeout options. SetTimeoutOpts(t *prometheus.TimeoutOpts) HandlerOptions - // Enforcer returns the enforcer. - Enforcer() cost.ChainedEnforcer - // SetEnforcer sets the enforcer. - SetEnforcer(e cost.ChainedEnforcer) HandlerOptions - // FetchOptionsBuilder returns the fetch options builder. FetchOptionsBuilder() handleroptions.FetchOptionsBuilder // SetFetchOptionsBuilder sets the fetch options builder. @@ -235,7 +229,6 @@ type handlerOptions struct { createdAt time.Time tagOptions models.TagOptions timeoutOpts *prometheus.TimeoutOpts - enforcer cost.ChainedEnforcer fetchOptionsBuilder handleroptions.FetchOptionsBuilder queryContextOptions models.QueryContextOptions instrumentOpts instrument.Options @@ -269,7 +262,6 @@ func NewHandlerOptions( clusterClient clusterclient.Client, cfg config.Configuration, embeddedDbCfg *dbconfig.DBConfiguration, - enforcer cost.ChainedEnforcer, fetchOptionsBuilder handleroptions.FetchOptionsBuilder, queryContextOptions models.QueryContextOptions, instrumentOpts instrument.Options, @@ -305,7 +297,6 @@ func NewHandlerOptions( embeddedDbCfg: embeddedDbCfg, createdAt: time.Now(), tagOptions: tagOptions, - enforcer: enforcer, fetchOptionsBuilder: fetchOptionsBuilder, queryContextOptions: queryContextOptions, instrumentOpts: instrumentOpts, @@ -431,16 +422,6 @@ func (o *handlerOptions) SetTimeoutOpts(t *prometheus.TimeoutOpts) HandlerOption return &opts } -func (o *handlerOptions) Enforcer() cost.ChainedEnforcer { - return o.enforcer -} - -func (o *handlerOptions) SetEnforcer(e cost.ChainedEnforcer) HandlerOptions { - opts := *o - opts.enforcer = e - return &opts -} - func (o *handlerOptions) FetchOptionsBuilder() handleroptions.FetchOptionsBuilder { return o.fetchOptionsBuilder } diff --git a/src/query/block/accounted.go b/src/query/block/accounted.go deleted file mode 100644 index 3dc2b45785..0000000000 --- a/src/query/block/accounted.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package block - -import "github.com/m3db/m3/src/query/cost" - -// AccountedBlock is a wrapper for a block which enforces limits on the number -// of datapoints used by the block. -type AccountedBlock struct { - Block - - enforcer cost.ChainedEnforcer -} - -// NewAccountedBlock wraps the given block and enforces datapoint limits. -func NewAccountedBlock( - wrapped Block, - enforcer cost.ChainedEnforcer, -) *AccountedBlock { - return &AccountedBlock{ - Block: wrapped, - enforcer: enforcer, - } -} - -// Close closes the block, and marks the number of datapoints used -// by this block as finished. -func (ab *AccountedBlock) Close() error { - ab.enforcer.Close() - return ab.Block.Close() -} diff --git a/src/query/block/accounted_test.go b/src/query/block/accounted_test.go deleted file mode 100644 index b5858ef210..0000000000 --- a/src/query/block/accounted_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package block - -import ( - "testing" - - "github.com/m3db/m3/src/query/cost" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" -) - -func TestAccountedBlock_Close(t *testing.T) { - ctrl := gomock.NewController(t) - - wrapped := NewMockBlock(ctrl) - wrapped.EXPECT().Close() - - mockEnforcer := cost.NewMockChainedEnforcer(ctrl) - mockEnforcer.EXPECT().Close() - - block := NewAccountedBlock(wrapped, mockEnforcer) - - wrapped.EXPECT().Info().Return(NewBlockInfo(BlockM3TSZCompressed)) - assert.Equal(t, BlockM3TSZCompressed, block.Info().Type()) - assert.NotPanics(t, func() { block.Close() }) -} diff --git a/src/query/block/column.go b/src/query/block/column.go index 86f406a566..dfcde95b91 100644 --- a/src/query/block/column.go +++ b/src/query/block/column.go @@ -25,9 +25,7 @@ import ( "fmt" "time" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" - xcost "github.com/m3db/m3/src/x/cost" "github.com/uber-go/tally" ) @@ -39,7 +37,6 @@ type column struct { // ColumnBlockBuilder builds a block optimized for column iteration. type ColumnBlockBuilder struct { block *columnBlock - enforcer cost.ChainedEnforcer blockDatapoints tally.Counter } @@ -174,7 +171,6 @@ func NewColumnBlockBuilder( meta Metadata, seriesMeta []SeriesMeta) Builder { return ColumnBlockBuilder{ - enforcer: queryCtx.Enforcer.Child(cost.BlockLevel), blockDatapoints: queryCtx.Scope.Tagged( map[string]string{"type": "generated"}).Counter("datapoints"), block: &columnBlock{ @@ -192,11 +188,6 @@ func (cb ColumnBlockBuilder) AppendValue(idx int, value float64) error { return fmt.Errorf("idx out of range for append: %d", idx) } - r := cb.enforcer.Add(1) - if r.Error != nil { - return r.Error - } - cb.blockDatapoints.Inc(1) columns[idx].Values = append(columns[idx].Values, value) @@ -210,11 +201,6 @@ func (cb ColumnBlockBuilder) AppendValues(idx int, values []float64) error { return fmt.Errorf("idx out of range for append: %d", idx) } - r := cb.enforcer.Add(xcost.Cost(len(values))) - if r.Error != nil { - return r.Error - } - cb.blockDatapoints.Inc(int64(len(values))) columns[idx].Values = append(columns[idx].Values, values...) return nil @@ -269,22 +255,17 @@ func (cb ColumnBlockBuilder) SetRow( cb.block.columns[i].Values[idx] = v } - r := cb.enforcer.Add(xcost.Cost(len(values))) - if r.Error != nil { - return r.Error - } - cb.block.seriesMeta[idx] = meta return nil } // Build builds the block. func (cb ColumnBlockBuilder) Build() Block { - return NewAccountedBlock(cb.block, cb.enforcer) + return cb.block } // BuildAsType builds the block, forcing it to the given BlockType. func (cb ColumnBlockBuilder) BuildAsType(blockType BlockType) Block { cb.block.blockType = blockType - return NewAccountedBlock(cb.block, cb.enforcer) + return cb.block } diff --git a/src/query/block/column_test.go b/src/query/block/column_test.go index ee1c9392bc..b341792c4c 100644 --- a/src/query/block/column_test.go +++ b/src/query/block/column_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/stretchr/testify/assert" @@ -36,7 +35,7 @@ import ( func makeTestQueryContext() *models.QueryContext { return models.NewQueryContext(context.Background(), - tally.NoopScope, cost.NoopChainedEnforcer(), + tally.NoopScope, models.QueryContextOptions{}) } diff --git a/src/query/config/m3coordinator-cluster-template.yml b/src/query/config/m3coordinator-cluster-template.yml index dd6167eb34..79887d3388 100644 --- a/src/query/config/m3coordinator-cluster-template.yml +++ b/src/query/config/m3coordinator-cluster-template.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/config/m3coordinator-local-etcd.yml b/src/query/config/m3coordinator-local-etcd.yml index 3c4afc7137..0dfefbd45f 100644 --- a/src/query/config/m3coordinator-local-etcd.yml +++ b/src/query/config/m3coordinator-local-etcd.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/config/m3query-dev-etcd.yml b/src/query/config/m3query-dev-etcd.yml index 09c1278e6b..cdaf363ce9 100644 --- a/src/query/config/m3query-dev-etcd.yml +++ b/src/query/config/m3query-dev-etcd.yml @@ -1,8 +1,7 @@ # m3query configuration for local development setup. Mostly the same as m3query-local-etcd.yml, but using fewer # resources (threads primarily). -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/config/m3query-dev-remote.yml b/src/query/config/m3query-dev-remote.yml index 49972a8340..560830e9fd 100644 --- a/src/query/config/m3query-dev-remote.yml +++ b/src/query/config/m3query-dev-remote.yml @@ -1,6 +1,4 @@ -listenAddress: - type: "config" - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 backend: grpc diff --git a/src/query/config/m3query-local-etcd.yml b/src/query/config/m3query-local-etcd.yml index bc97a8000d..f63b801a41 100644 --- a/src/query/config/m3query-local-etcd.yml +++ b/src/query/config/m3query-local-etcd.yml @@ -1,5 +1,4 @@ -listenAddress: - value: "0.0.0.0:7201" +listenAddress: 0.0.0.0:7201 logging: level: info diff --git a/src/query/cost/cost.go b/src/query/cost/cost.go deleted file mode 100644 index 30e7fdafcb..0000000000 --- a/src/query/cost/cost.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "errors" - "fmt" - - "github.com/m3db/m3/src/x/cost" -) - -const ( - // BlockLevel identifies per-block enforcers. - BlockLevel = "block" - // QueryLevel identifies per-query enforcers. - QueryLevel = "query" - // GlobalLevel identifies global enforcers. - GlobalLevel = "global" -) - -// ChainedEnforcer is a cost.Enforcer implementation which tracks resource usage implements cost.Enforcer to enforce -// limits on multiple resources at once, linked together in a tree. -type ChainedEnforcer interface { - cost.Enforcer - - // Child creates a new ChainedEnforcer which rolls up to this one. - Child(resourceName string) ChainedEnforcer - - // Close indicates that all resources have been returned for this - // ChainedEnforcer. It should inform all parent enforcers that the - // resources have been freed. - Close() -} - -type noopChainedReporter struct{} - -func (noopChainedReporter) ReportCost(_ cost.Cost) {} -func (noopChainedReporter) ReportCurrent(_ cost.Cost) {} -func (noopChainedReporter) ReportOverLimit(_ bool) {} -func (noopChainedReporter) OnChildClose(_ cost.Cost) {} -func (noopChainedReporter) OnClose(_ cost.Cost) {} - -var noopChainedReporterInstance = noopChainedReporter{} - -// ChainedReporter is a listener for chainedEnforcer methods, which listens to Close events in addition to -// events used by cost.EnforcerReporter. -type ChainedReporter interface { - cost.EnforcerReporter - - // OnChildClose is called whenever a child of this reporter's chainedEnforcer is released. - OnChildClose(currentCost cost.Cost) - - // OnClose is called whenever this reporter's chainedEnforcer is released. - OnClose(currentCost cost.Cost) -} - -// chainedEnforcer is the actual implementation of ChainedEnforcer. -type chainedEnforcer struct { - resourceName string - local cost.Enforcer - parent *chainedEnforcer - models []cost.Enforcer - reporter ChainedReporter -} - -var noopChainedEnforcer = mustNoopChainedEnforcer() - -func mustNoopChainedEnforcer() ChainedEnforcer { - rtn, err := NewChainedEnforcer("", []cost.Enforcer{cost.NoopEnforcer()}) - if err != nil { - panic(err.Error()) - } - - return rtn -} - -// NoopChainedEnforcer returns a chainedEnforcer which enforces no limits and does no reporting. -func NoopChainedEnforcer() ChainedEnforcer { - return noopChainedEnforcer -} - -// NewChainedEnforcer constructs a chainedEnforcer which creates children using the provided models. -// models[0] enforces this instance; models[1] enforces the first level of children, and so on. -func NewChainedEnforcer(rootResourceName string, models []cost.Enforcer) (ChainedEnforcer, error) { - if len(models) == 0 { - return nil, errors.New("must provide at least one Enforcer instance for a chainedEnforcer") - } - - local := models[0] - - return &chainedEnforcer{ - resourceName: rootResourceName, - parent: nil, // root has nil parent - local: local, - models: models[1:], - reporter: upcastReporterOrNoop(local.Reporter()), - }, nil -} - -func upcastReporterOrNoop(r cost.EnforcerReporter) ChainedReporter { - if r, ok := r.(ChainedReporter); ok { - return r - } - - return noopChainedReporterInstance -} - -// Add adds the given cost both to this enforcer and any parents, working recursively until the root is reached. -// The most local error is preferred. -func (ce *chainedEnforcer) Add(c cost.Cost) cost.Report { - if ce.parent == nil { - return ce.wrapLocalResult(ce.local.Add(c)) - } - - localR := ce.local.Add(c) - globalR := ce.parent.Add(c) - - // check our local limit first - if localR.Error != nil { - return ce.wrapLocalResult(localR) - } - - // check the global limit - if globalR.Error != nil { - return globalR - } - - return localR -} - -func (ce *chainedEnforcer) wrapLocalResult(localR cost.Report) cost.Report { - if localR.Error != nil { - return cost.Report{ - Cost: localR.Cost, - Error: fmt.Errorf("exceeded %s limit: %s", ce.resourceName, localR.Error.Error()), - } - } - return localR -} - -// Child creates a new chainedEnforcer whose resource consumption rolls up into this instance. -func (ce *chainedEnforcer) Child(resourceName string) ChainedEnforcer { - // no more models; just return a noop default. TODO: this could be a panic case? Technically speaking it's - // misconfiguration. - if len(ce.models) == 0 { - return NoopChainedEnforcer() - } - - newLocal := ce.models[0] - return &chainedEnforcer{ - resourceName: resourceName, - parent: ce, - // make sure to clone the local enforcer, so that we're using an - // independent instance with the same configuration. - local: newLocal.Clone(), - models: ce.models[1:], - reporter: upcastReporterOrNoop(newLocal.Reporter()), - } -} - -// Clone on a chainedEnforcer is a noop--TODO: implement? -func (ce *chainedEnforcer) Clone() cost.Enforcer { - return ce -} - -// State returns the local state of this enforcer (ignoring anything further up the chain). -func (ce *chainedEnforcer) State() (cost.Report, cost.Limit) { - return ce.local.State() -} - -// Close releases all resources tracked by this enforcer back to the global enforcer -func (ce *chainedEnforcer) Close() { - r, _ := ce.local.State() - ce.reporter.OnClose(r.Cost) - - if ce.parent != nil { - parentR, _ := ce.parent.State() - ce.parent.reporter.OnChildClose(parentR.Cost) - } - - ce.Add(-r.Cost) -} - -func (ce *chainedEnforcer) Limit() cost.Limit { - return ce.local.Limit() -} - -func (ce *chainedEnforcer) Reporter() cost.EnforcerReporter { - return ce.local.Reporter() -} diff --git a/src/query/cost/cost_mock.go b/src/query/cost/cost_mock.go deleted file mode 100644 index 85abd1379b..0000000000 --- a/src/query/cost/cost_mock.go +++ /dev/null @@ -1,236 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/m3db/m3/src/query/cost (interfaces: ChainedEnforcer,ChainedReporter) - -// Copyright (c) 2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package cost is a generated GoMock package. -package cost - -import ( - "reflect" - - cost0 "github.com/m3db/m3/src/x/cost" - - "github.com/golang/mock/gomock" -) - -// MockChainedEnforcer is a mock of ChainedEnforcer interface -type MockChainedEnforcer struct { - ctrl *gomock.Controller - recorder *MockChainedEnforcerMockRecorder -} - -// MockChainedEnforcerMockRecorder is the mock recorder for MockChainedEnforcer -type MockChainedEnforcerMockRecorder struct { - mock *MockChainedEnforcer -} - -// NewMockChainedEnforcer creates a new mock instance -func NewMockChainedEnforcer(ctrl *gomock.Controller) *MockChainedEnforcer { - mock := &MockChainedEnforcer{ctrl: ctrl} - mock.recorder = &MockChainedEnforcerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockChainedEnforcer) EXPECT() *MockChainedEnforcerMockRecorder { - return m.recorder -} - -// Add mocks base method -func (m *MockChainedEnforcer) Add(arg0 cost0.Cost) cost0.Report { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Add", arg0) - ret0, _ := ret[0].(cost0.Report) - return ret0 -} - -// Add indicates an expected call of Add -func (mr *MockChainedEnforcerMockRecorder) Add(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockChainedEnforcer)(nil).Add), arg0) -} - -// Child mocks base method -func (m *MockChainedEnforcer) Child(arg0 string) ChainedEnforcer { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Child", arg0) - ret0, _ := ret[0].(ChainedEnforcer) - return ret0 -} - -// Child indicates an expected call of Child -func (mr *MockChainedEnforcerMockRecorder) Child(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Child", reflect.TypeOf((*MockChainedEnforcer)(nil).Child), arg0) -} - -// Clone mocks base method -func (m *MockChainedEnforcer) Clone() cost0.Enforcer { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Clone") - ret0, _ := ret[0].(cost0.Enforcer) - return ret0 -} - -// Clone indicates an expected call of Clone -func (mr *MockChainedEnforcerMockRecorder) Clone() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clone", reflect.TypeOf((*MockChainedEnforcer)(nil).Clone)) -} - -// Close mocks base method -func (m *MockChainedEnforcer) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close -func (mr *MockChainedEnforcerMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockChainedEnforcer)(nil).Close)) -} - -// Limit mocks base method -func (m *MockChainedEnforcer) Limit() cost0.Limit { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Limit") - ret0, _ := ret[0].(cost0.Limit) - return ret0 -} - -// Limit indicates an expected call of Limit -func (mr *MockChainedEnforcerMockRecorder) Limit() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Limit", reflect.TypeOf((*MockChainedEnforcer)(nil).Limit)) -} - -// Reporter mocks base method -func (m *MockChainedEnforcer) Reporter() cost0.EnforcerReporter { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Reporter") - ret0, _ := ret[0].(cost0.EnforcerReporter) - return ret0 -} - -// Reporter indicates an expected call of Reporter -func (mr *MockChainedEnforcerMockRecorder) Reporter() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reporter", reflect.TypeOf((*MockChainedEnforcer)(nil).Reporter)) -} - -// State mocks base method -func (m *MockChainedEnforcer) State() (cost0.Report, cost0.Limit) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "State") - ret0, _ := ret[0].(cost0.Report) - ret1, _ := ret[1].(cost0.Limit) - return ret0, ret1 -} - -// State indicates an expected call of State -func (mr *MockChainedEnforcerMockRecorder) State() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "State", reflect.TypeOf((*MockChainedEnforcer)(nil).State)) -} - -// MockChainedReporter is a mock of ChainedReporter interface -type MockChainedReporter struct { - ctrl *gomock.Controller - recorder *MockChainedReporterMockRecorder -} - -// MockChainedReporterMockRecorder is the mock recorder for MockChainedReporter -type MockChainedReporterMockRecorder struct { - mock *MockChainedReporter -} - -// NewMockChainedReporter creates a new mock instance -func NewMockChainedReporter(ctrl *gomock.Controller) *MockChainedReporter { - mock := &MockChainedReporter{ctrl: ctrl} - mock.recorder = &MockChainedReporterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockChainedReporter) EXPECT() *MockChainedReporterMockRecorder { - return m.recorder -} - -// OnChildClose mocks base method -func (m *MockChainedReporter) OnChildClose(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "OnChildClose", arg0) -} - -// OnChildClose indicates an expected call of OnChildClose -func (mr *MockChainedReporterMockRecorder) OnChildClose(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnChildClose", reflect.TypeOf((*MockChainedReporter)(nil).OnChildClose), arg0) -} - -// OnClose mocks base method -func (m *MockChainedReporter) OnClose(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "OnClose", arg0) -} - -// OnClose indicates an expected call of OnClose -func (mr *MockChainedReporterMockRecorder) OnClose(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnClose", reflect.TypeOf((*MockChainedReporter)(nil).OnClose), arg0) -} - -// ReportCost mocks base method -func (m *MockChainedReporter) ReportCost(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ReportCost", arg0) -} - -// ReportCost indicates an expected call of ReportCost -func (mr *MockChainedReporterMockRecorder) ReportCost(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportCost", reflect.TypeOf((*MockChainedReporter)(nil).ReportCost), arg0) -} - -// ReportCurrent mocks base method -func (m *MockChainedReporter) ReportCurrent(arg0 cost0.Cost) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ReportCurrent", arg0) -} - -// ReportCurrent indicates an expected call of ReportCurrent -func (mr *MockChainedReporterMockRecorder) ReportCurrent(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportCurrent", reflect.TypeOf((*MockChainedReporter)(nil).ReportCurrent), arg0) -} - -// ReportOverLimit mocks base method -func (m *MockChainedReporter) ReportOverLimit(arg0 bool) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ReportOverLimit", arg0) -} - -// ReportOverLimit indicates an expected call of ReportOverLimit -func (mr *MockChainedReporterMockRecorder) ReportOverLimit(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportOverLimit", reflect.TypeOf((*MockChainedReporter)(nil).ReportOverLimit), arg0) -} diff --git a/src/query/cost/cost_prop_test.go b/src/query/cost/cost_prop_test.go deleted file mode 100644 index 5bcc726b04..0000000000 --- a/src/query/cost/cost_prop_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "math" - "sync" - "testing" - - "github.com/m3db/m3/src/x/cost" - - "github.com/leanovate/gopter" - "github.com/leanovate/gopter/gen" - "github.com/leanovate/gopter/prop" - "github.com/stretchr/testify/require" -) - -func TestPropertyPerQueryEnforcerAlwaysEndsUpZero(t *testing.T) { - testParams := gopter.DefaultTestParameters() - testParams.MinSuccessfulTests = 1000 - props := gopter.NewProperties(testParams) - - globalEndsUpZero := func(costs []float64, perQueryThreshold, globalThreshold float64) bool { - pqfIFace, err := NewChainedEnforcer( - "", - []cost.Enforcer{newTestEnforcer(cost.Limit{Threshold: cost.Cost(globalThreshold), Enabled: true}), - newTestEnforcer(cost.Limit{Threshold: cost.Cost(perQueryThreshold), Enabled: true})}) - require.NoError(t, err) - pqf := pqfIFace.(*chainedEnforcer) - wg := sync.WaitGroup{} - for _, c := range costs { - wg.Add(1) - go func(c float64) { - defer wg.Done() - - perQuery := pqf.Child("query") - defer perQuery.Close() - perQuery.Add(cost.Cost(c)) - }(c) - } - - wg.Wait() - r, _ := pqf.local.State() - - // do delta comparison to deal with floating point errors. TODO: cost could potentially be an int - const tolerance = 0.000001 - return math.Abs(float64(r.Cost)) < tolerance - } - - props.Property("global enforcer >= 0", - prop.ForAll( - globalEndsUpZero, - gen.SliceOf(gen.Float64Range(0, 10000)), - gen.Float64Range(0.0, 10000), - gen.Float64Range(0.0, 10000), - )) - - props.TestingRun(t) -} diff --git a/src/query/cost/cost_test.go b/src/query/cost/cost_test.go deleted file mode 100644 index a554f45d98..0000000000 --- a/src/query/cost/cost_test.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "math" - "testing" - - "github.com/m3db/m3/src/x/cost" - "github.com/m3db/m3/src/x/cost/test" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestChainedEnforcer_Child(t *testing.T) { - t.Run("creates independent local enforcers with shared global enforcer", func(t *testing.T) { - globalEnforcer := newTestEnforcer(cost.Limit{Threshold: 10.0, Enabled: true}) - localEnforcer := newTestEnforcer(cost.Limit{Threshold: 5.0, Enabled: true}) - - pef, err := NewChainedEnforcer("", []cost.Enforcer{globalEnforcer, localEnforcer}) - require.NoError(t, err) - - l1, l2 := pef.Child("foo"), pef.Child("foo") - - l1.Add(2) - - test.AssertCurrentCost(t, 2.0, l1) - test.AssertCurrentCost(t, 0.0, l2) - test.AssertCurrentCost(t, 2.0, globalEnforcer) - }) - - t.Run("returns a noop enforcer once out of models", func(t *testing.T) { - globalEnforcer := newTestEnforcer(cost.Limit{Threshold: 10.0, Enabled: true}) - pef, err := NewChainedEnforcer("", []cost.Enforcer{globalEnforcer}) - require.NoError(t, err) - - child := pef.Child("foo") - assert.Equal(t, noopChainedEnforcer, child) - }) -} - -func TestChainedEnforcer_Close(t *testing.T) { - t.Run("removes local total from global", func(t *testing.T) { - parentIface, err := NewChainedEnforcer( - "", - []cost.Enforcer{newTestEnforcer(cost.Limit{Threshold: 10.0, Enabled: true}), - newTestEnforcer(cost.Limit{Threshold: 5.0, Enabled: true})}) - require.NoError(t, err) - parent := parentIface.(*chainedEnforcer) - - pqe1, pqe2 := parent.Child("query"), parent.Child("query") - - pqe1.Add(cost.Cost(5.0)) - pqe1.Add(cost.Cost(6.0)) - - pqe2.Add(cost.Cost(7.0)) - - pqe1.Close() - - test.AssertCurrentCost(t, cost.Cost(7.0), parent.local) - pqe2.Close() - test.AssertCurrentCost(t, cost.Cost(0.0), parent.local) - }) - - t.Run("calls into reporter on release", func(t *testing.T) { - ctrl := gomock.NewController(t) - makeEnforcer := func(cr ChainedReporter) cost.Enforcer { - return cost.NewEnforcer(cost.NewStaticLimitManager(cost.NewLimitManagerOptions()), cost.NewTracker(), - cost.NewEnforcerOptions().SetReporter(cr)) - } - - makeReporter := func() *MockChainedReporter { - r := NewMockChainedReporter(ctrl) - r.EXPECT().ReportCurrent(gomock.Any()).AnyTimes() - r.EXPECT().ReportOverLimit(gomock.Any()).AnyTimes() - r.EXPECT().ReportCost(gomock.Any()).AnyTimes() - return r - } - - globalReporter, localReporter := makeReporter(), makeReporter() - - ce, err := NewChainedEnforcer( - "global", - []cost.Enforcer{makeEnforcer(globalReporter), makeEnforcer(localReporter)}) - - child := ce.Child("foo") - child.Add(1.0) - - require.NoError(t, err) - - globalReporter.EXPECT().OnChildClose(floatMatcher(1.0)) - localReporter.EXPECT().OnClose(floatMatcher(1.0)) - - child.Close() - }) -} - -// floatMatcher does a janky delta comparison between floats, since rounding error makes float equality treacherous -type floatMatcher float64 - -func (f floatMatcher) Matches(x interface{}) bool { - other, ok := x.(cost.Cost) - if !ok { - return false - } - - // janky delta comparison - return math.Abs(float64(f)-float64(other)) < 0.00001 -} - -func (f floatMatcher) String() string { - return fmt.Sprintf("%f", f) -} - -func TestChainedEnforcer_Add(t *testing.T) { - assertGlobalError := func(t *testing.T, err error) { - if assert.Error(t, err) { - assert.Regexp(t, "exceeded global limit", err.Error()) - } - } - - assertLocalError := func(t *testing.T, err error) { - if assert.Error(t, err) { - assert.Regexp(t, "exceeded query limit", err.Error()) - } - } - - t.Run("errors on global error", func(t *testing.T) { - pqe := newTestChainedEnforcer(5.0, 100.0) - r := pqe.Add(cost.Cost(6.0)) - assertGlobalError(t, r.Error) - }) - - t.Run("errors on local error", func(t *testing.T) { - pqe := newTestChainedEnforcer(100.0, 5.0) - r := pqe.Add(cost.Cost(6.0)) - assertLocalError(t, r.Error) - }) - - t.Run("adds to local in case of global error", func(t *testing.T) { - pqe := newTestChainedEnforcer(5.0, 100.0) - r := pqe.Add(cost.Cost(6.0)) - assertGlobalError(t, r.Error) - - r, _ = pqe.State() - assert.Equal(t, cost.Report{ - Error: nil, - Cost: 6.0}, - r) - }) - - t.Run("adds to global in case of local error", func(t *testing.T) { - pqe := newTestChainedEnforcer(100.0, 5.0) - r := pqe.Add(cost.Cost(6.0)) - assertLocalError(t, r.Error) - - r, _ = pqe.parent.State() - assert.Equal(t, cost.Report{ - Error: nil, - Cost: 6.0}, - r) - }) - - t.Run("release after local error", func(t *testing.T) { - pqe := newTestChainedEnforcer(10.0, 5.0) - - // exceeds local - r := pqe.Add(6.0) - assertLocalError(t, r.Error) - - pqe.Close() - test.AssertCurrentCost(t, 0.0, pqe.local) - }) - - t.Run("release after global error", func(t *testing.T) { - pqe := newTestChainedEnforcer(5.0, 10.0) - // exceeds global - r := pqe.Add(6.0) - assertGlobalError(t, r.Error) - pqe.Close() - test.AssertCurrentCost(t, 0.0, pqe.local) - }) -} - -func TestChainedEnforcer_State(t *testing.T) { - pqe := newTestChainedEnforcer(10.0, 5.0) - pqe.Add(15.0) - - r, l := pqe.State() - assert.Equal(t, cost.Cost(15), r.Cost) - test.AssertLimitError(t, r.Error, 15.0, 5.0) - assert.Equal(t, cost.Limit{Threshold: 5.0, Enabled: true}, l) -} - -func TestNoopChainedEnforcer_Close(t *testing.T) { - ce := NoopChainedEnforcer() - ce.Close() - test.AssertCurrentCost(t, 0.0, ce) -} - -// utils - -func newTestEnforcer(limit cost.Limit) cost.Enforcer { - return cost.NewEnforcer( - cost.NewStaticLimitManager(cost.NewLimitManagerOptions().SetDefaultLimit(limit)), - cost.NewTracker(), - nil, - ) -} - -func newTestChainedEnforcer(globalLimit float64, localLimit float64) *chainedEnforcer { - rtn, err := NewChainedEnforcer( - "global", - []cost.Enforcer{newTestEnforcer(cost.Limit{Threshold: cost.Cost(globalLimit), Enabled: true}), - newTestEnforcer(cost.Limit{Threshold: cost.Cost(localLimit), Enabled: true})}) - if err != nil { - panic(err.Error()) - } - - return rtn.Child("query").(*chainedEnforcer) -} diff --git a/src/query/executor/engine.go b/src/query/executor/engine.go index 584d141388..c8bd17937c 100644 --- a/src/query/executor/engine.go +++ b/src/query/executor/engine.go @@ -25,7 +25,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser" "github.com/m3db/m3/src/query/storage" @@ -48,10 +47,6 @@ type QueryOptions struct { func NewEngine( engineOpts EngineOptions, ) Engine { - if engineOpts.GlobalEnforcer() == nil { - engineOpts = engineOpts.SetGlobalEnforcer(qcost.NoopChainedEnforcer()) - } - return &engine{ metrics: newEngineMetrics(engineOpts.InstrumentOptions().MetricsScope()), opts: engineOpts, @@ -120,8 +115,6 @@ func (e *engine) ExecuteExpr( fetchOpts *storage.FetchOptions, params models.RequestParams, ) (block.Block, error) { - perQueryEnforcer := e.opts.GlobalEnforcer().Child(qcost.QueryLevel) - defer perQueryEnforcer.Close() req := newRequest(e, params, fetchOpts, e.opts.InstrumentOptions()) nodes, edges, err := req.compile(ctx, parser) if err != nil { @@ -143,7 +136,7 @@ func (e *engine) ExecuteExpr( defer sp.Finish() scope := e.opts.InstrumentOptions().MetricsScope() - queryCtx := models.NewQueryContext(ctx, scope, perQueryEnforcer, + queryCtx := models.NewQueryContext(ctx, scope, opts.QueryContextOptions) if err := state.Execute(queryCtx); err != nil { diff --git a/src/query/executor/engine_test.go b/src/query/executor/engine_test.go index 8906c613c1..aa7b80f383 100644 --- a/src/query/executor/engine_test.go +++ b/src/query/executor/engine_test.go @@ -28,8 +28,6 @@ import ( "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser/promql" "github.com/m3db/m3/src/query/storage" @@ -45,13 +43,11 @@ import ( func newEngine( s storage.Storage, lookbackDuration time.Duration, - enforcer qcost.ChainedEnforcer, instrumentOpts instrument.Options, ) Engine { engineOpts := NewEngineOptions(). SetStore(s). SetLookbackDuration(lookbackDuration). - SetGlobalEnforcer(enforcer). SetInstrumentOptions(instrumentOpts) return NewEngine(engineOpts) @@ -65,7 +61,7 @@ func TestExecute(t *testing.T) { session.EXPECT().IteratorPools().Return(nil, nil) // Results is closed by execute - engine := newEngine(store, time.Minute, nil, instrument.NewOptions()) + engine := newEngine(store, time.Minute, instrument.NewOptions()) _, err := engine.ExecuteProm(context.TODO(), &storage.FetchQuery{}, &QueryOptions{}, storage.NewFetchOptions()) assert.NotNil(t, err) @@ -75,12 +71,6 @@ func TestExecuteExpr(t *testing.T) { ctrl := xtest.NewController(t) defer ctrl.Finish() - mockEnforcer := cost.NewMockChainedEnforcer(ctrl) - mockEnforcer.EXPECT().Close().Times(1) - - mockParent := cost.NewMockChainedEnforcer(ctrl) - mockParent.EXPECT().Child(gomock.Any()).Return(mockEnforcer) - parser, err := promql.Parse("foo", time.Second, models.NewTagOptions(), promql.NewParseOptions()) require.NoError(t, err) @@ -91,7 +81,7 @@ func TestExecuteExpr(t *testing.T) { Blocks: []block.Block{block.NewMockBlock(ctrl)}, }, nil) engine := newEngine(store, defaultLookbackDuration, - mockParent, instrument.NewOptions()) + instrument.NewOptions()) _, err = engine.ExecuteExpr(context.TODO(), parser, &QueryOptions{}, storage.NewFetchOptions(), models.RequestParams{ Start: time.Now().Add(-2 * time.Second), diff --git a/src/query/executor/options.go b/src/query/executor/options.go index ea4cb88f0c..456ff7f60c 100644 --- a/src/query/executor/options.go +++ b/src/query/executor/options.go @@ -23,7 +23,6 @@ package executor import ( "time" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/parser/promql" "github.com/m3db/m3/src/query/storage" "github.com/m3db/m3/src/x/instrument" @@ -31,7 +30,6 @@ import ( type engineOptions struct { instrumentOpts instrument.Options - globalEnforcer qcost.ChainedEnforcer store storage.Storage parseOptions promql.ParseOptions lookbackDuration time.Duration @@ -54,16 +52,6 @@ func (o *engineOptions) SetInstrumentOptions(v instrument.Options) EngineOptions return &opts } -func (o *engineOptions) GlobalEnforcer() qcost.ChainedEnforcer { - return o.globalEnforcer -} - -func (o *engineOptions) SetGlobalEnforcer(v qcost.ChainedEnforcer) EngineOptions { - opts := *o - opts.globalEnforcer = v - return &opts -} - func (o *engineOptions) Store() storage.Storage { return o.store } diff --git a/src/query/executor/types.go b/src/query/executor/types.go index c5194c2f84..83ebb7efd8 100644 --- a/src/query/executor/types.go +++ b/src/query/executor/types.go @@ -25,7 +25,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - qcost "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser" "github.com/m3db/m3/src/query/parser/promql" @@ -69,11 +68,6 @@ type EngineOptions interface { // for metrics. SetInstrumentOptions(instrument.Options) EngineOptions - // GlobalEnforcer returns the query cost enforcer. - GlobalEnforcer() qcost.ChainedEnforcer - // SetGlobalEnforcer sets the query cost enforcer. - SetGlobalEnforcer(qcost.ChainedEnforcer) EngineOptions - // Store returns the storage. Store() storage.Storage // SetStore sets the storage. diff --git a/src/query/functions/fetch_test.go b/src/query/functions/fetch_test.go index 0e6ae9a1fa..9fa3d4baae 100644 --- a/src/query/functions/fetch_test.go +++ b/src/query/functions/fetch_test.go @@ -27,7 +27,6 @@ import ( "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/executor/transform" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/parser" @@ -133,7 +132,7 @@ func TestFetchWithRestrictFetch(t *testing.T) { transformtest.Options(t, transform.OptionsParams{})) ctx := models.NewQueryContext(context.Background(), - tally.NoopScope, cost.NoopChainedEnforcer(), + tally.NoopScope, models.QueryContextOptions{ RestrictFetchType: &models.RestrictFetchTypeQueryContextOptions{ MetricsType: uint(storagemetadata.AggregatedMetricsType), diff --git a/src/query/functions/temporal/base_test.go b/src/query/functions/temporal/base_test.go index 489bd9211d..b2fa8d1c3c 100644 --- a/src/query/functions/temporal/base_test.go +++ b/src/query/functions/temporal/base_test.go @@ -136,7 +136,7 @@ func testTemporalFunc(t *testing.T, opGen opGenerator, tests []testCase) { test.EqualsWithNansWithDelta(t, tt.expected, sink.Values, 0.0001) metaOne := block.SeriesMeta{ - Name: []byte("t1=v1,"), + Name: []byte("{t1=\"v1\"}"), Tags: models.EmptyTags().AddTags([]models.Tag{{ Name: []byte("t1"), Value: []byte("v1"), @@ -144,7 +144,7 @@ func testTemporalFunc(t *testing.T, opGen opGenerator, tests []testCase) { } metaTwo := block.SeriesMeta{ - Name: []byte("t1=v2,"), + Name: []byte("{t1=\"v2\"}"), Tags: models.EmptyTags().AddTags([]models.Tag{{ Name: []byte("t1"), Value: []byte("v2"), @@ -338,7 +338,7 @@ func testParallelProcess(t *testing.T, warning bool) { for i, m := range sink.Metas { expected := fmt.Sprint(expected[i]) - expectedName := fmt.Sprintf("tag=%s,", expected) + expectedName := fmt.Sprintf("{tag=\"%s\"}", expected) assert.Equal(t, expectedName, string(m.Name)) require.Equal(t, 1, m.Tags.Len()) tag, found := m.Tags.Get([]byte(tagName)) diff --git a/src/query/generated/assets/openapi/assets.go b/src/query/generated/assets/openapi/assets.go index 73f79c953d..3ea4391484 100644 --- a/src/query/generated/assets/openapi/assets.go +++ b/src/query/generated/assets/openapi/assets.go @@ -239,48 +239,45 @@ d8dUBmQZxiF/+uI1I7E8TMF6pCyWxDpY7WPA8pmKZsl6ouawOaOS+Sj+BQAA//8by2IcfAIAAA== "/spec.yml": { local: "openapi/spec.yml", - size: 24401, + size: 22713, modtime: 12345, compressed: ` -H4sIAAAAAAAC/+xcW2/jNhZ+969gNfuwBTZRZjLbBfzmxGnGQCZjJGmBbbFAafJIZiuRGl6S8RT73wtK -sixZsi6242Rc5SWJdHh4Lt+5kJT0Bt1+ergaojvD0W8h/gMQVgr0iQ/85LMBufgNMQ8thEHJTb5AZI65 -DwppgfScKeSxAL4bqCfs+yCHyHl3euYMGPfEcICQZjqAIXI+no8vnAFCFBSRLNJM8CFyRogypSWbGQ0U -aRYCUiAZKESxxjOsABnFuI8+nj/c/4K8QGD9w3tERBhJUIoJfor+KwwimCOPcYqE0SgUEhCe2T/trAhr -9Otc60gNXZcKok7Dczo7ZcL93z8rL3+PhESCo1+vmf5gZisqn+m5mZ0SEbqW1g3Pvz+1Oj2CVIk+b0/P -rPIIEcE1JtpaACGOw8QEF2N0LYQfALqWwkROfNfIYIicbA57Q536MVk8lSekCd033yW/7cR2XMAIcAWF -CUYRJnNAN8kt9C4RpTRDSQt3FoiZG2KlQbo3k8ur2/srZzAXStthQumY/3/enb11BtYnU6znQ+S4OGLu -41tnoLGvhoOTlZ7jC3SLQ1ARJlB2+qXgHvONTPw6vojHxbTKWeMyDTCBELhuwSVa0ha5jHxfgo+1kO25 -5cZs4Dq+QOMUoWVmhdsnT4wC8gwn9q5yBorMIYTYYLFPnEGE9VxZT7oK5CMjoBLPZHZJvOxDiieEEouj -9Oekyub2R5kwxHIxRM416LKtEyIRgcRWtgkdIie7fw16SUEEVyYWOZsPR1HASDzM/V0JviSNpKCGtCKV -oCLBFeQUeXd2tvpn3apO7k5sQ5ynRegfErwhct64FDzGWWxt9zanzl064YrR+7P3e57vGjhIRq6kFHLF -4N9716s8T2TDtQIeLcAxohRhvoaPBniMKH1eeERY4hA0yBxxGn4zQRcrqzFeulQ2Yz04RpTewWcDSr8q -cJ4dDzjNttj8KaJYQ2d4JsOOB6GJPj1In2+eTbXX/TP7czL+f8KYQgAatkT0OB7cGdHJsBdDdM4Ia8C2 -3cvqkoTPhkmgQ6SlgeyyXkSWi+31uX9Q/CZ2i/s7GcYqHx69x9NnrEVJ1h3XdqgnVd18uT/Vc1jr5KtD -Irt9HC3qNKdOOcG+htaxxm9p68i40pgTSPYEoLUHj6GLnOaUeYkCXY+fY+oim8tuDVLTsts5ySTjRkFw -BKmmrhYetnYQISRlHGshuxSRy9Wwsq/XKeqqTJ5RX25eUbnZ0cN9Perr0WupRztCuVCwtslXfeV6jsqF -s7OBLoVr0ylEBUFd2Rr5fpP7Q0vU16xD1qydnJttvlvfdqxbRV/3xasvXnsrXjthulC62uasaV+3DrVb -59oRw133fiZ2Whywr1usqu3Y40lWVps+W70AjiXEf+8M5buET3Yok9VhxjutH1M+x4PsVKEe3C+4LdY2 -W++43iyl823WnEeb1yuN18fBQeOgfbbfMRQK5aCCss/+PeoPtLXUNvnvtF4rpf7Oa7a+n+9Bv0fQt8/0 -O+G+kOfLhHWA73N9D/u9LWP/XK43Ozxh2PyoQ2kx60kRdlrOvvAzhyurfFuPHPYd+9bA3s+Z6Xrf3odA -HwIv1Ml0joB9HLyUzxPb4j7WYtqDvwd/935m+cquSyRg3XJTPv8WZRHLy1cyQSVtzBPTcxQKpZGItVCI -godNoIFWY3kpz2UszjffqY8L6rxEn74uwd8Q2TEm3ZkQWmmJoyhz+PY4Hy84DhnBQbBA4hGkZDTZhinM -gsgyHChiSWOvEBcUTgJ4hCC7XThj3hAPMek96Iv8BMcTH7F6lbodNko2y3FcT5I0vPWzGfl3oCWDxwTt -NBcGOawXg2AZH/9CzEOYL9oA/fqgQP/7Qaz25TYuNPKE4dQ6zP6jVg8kfgPwzt2xYyteGE9Ypr2mmP0O -JNUvkhaUmq2QEOe7+vY07WxWVPWv5H6K0m9K5EQrvCn8zNJVMK5mnkSGBm4ZfFpnU8NqM7sCyylIJug4 -LX/rZBvUiTkYrlkIlSLV2/6uMLLgggK3VsbPstwVx7MAaMnEMyECwFmW8QKj5i1pnyTToB7EpQhDpm+E -3zSA2H9MW1EkRJjJ1sSbMFBp7Ls18qyEcBypudAtZ2Wcwpd2M05ypJnQGzDSFR93lcq3wkcj0CsBPgsE -+eOefYW29MbzQP5otJHdhkyx0l1ksvXr6kvE5KLJdWvkI0+DvBV6RAgo1doYkxIAWlkd2sGri5k3wKOD -UHHoxwox7k9BXk5/uhScGCmBk7I9uQlnkKuYnpAh1rZJEmYWQCFJbOaKtmBb9eGITpD3mdJ5hDTEW0pf -tHKBSes6mHwgqbHOYUpjKXAwrSxRrYt3+SnhDgIn+4m1KK3akOoww9p7Ls37XksTLT+Mti4a4xr8Svgw -rs/fFUTuIOdyv/CZPDdJ2ecqn10C/IiJFrJJR27C+zmWVDURMhXTNacdYrRdDj2wir6t1sA/vF/N9ZHZ -Rrd5shB/icW6Bz2hddMtjdTFbbShZjAlgjgq4q/mNRB/Fbypj30C5s91k9GA00gwrhuYqWqvYilxfltD -Q9iMsNjEBcaN9rY/2Sf66iWNhNRtXNd97fBte/B5zFc4gNynLVuY57CKx6jdTcM1xymNNTQUmyQd6dzO -vhJGEpg02S/Nm7eYC7Vt4rQ8PG9rFivZC2bLywnchMnNE+RMbicPk9HN5JfJ7bWzvDj6eTS5GV3cXGVX -bq5GP6cUFe/R7aWQbpXW1iIjW8QKSaBV25J7Hu/VadG6sFd1Obk2wk7SrpWobZeKj3F1sFYA+JFxf5Kd -uXY3W3W4YU4Zxfo1gmnrHP3syMpvQ3ZZsa7oKx1SeYa4zeLotrlsxBcHDTtwqzSX5rBAEBw4+SskMEpv -0Ug/a7wVNmUqW/ANWzENVe1iSZev0XuC2QeRYOuiKEsrx+tmFeFLBEQDvY8/4G2RFvcgagrygzBymzr5 -Qey3DcWUSlBqx37nBRva/Zu4+kx/m5TQdqOm4qGYzhsMazxqzsM6aPKIAwM7V72/AgAA//8jiOAHUV8A -AA== +H4sIAAAAAAAC/+xcX2/jNhJ/96dgtfdwfUicTfZ6gN+cOM0ayGaNJC1wLQ4oTY5kthKpJYfJZov77gdK +tixZsiXZzp815Jc44nDImflxfkNK8jty8/n+ckBurSR/RPQvINQYwKMA5NEXC/rpDyJ88qQsSRvlE2Ez +KgMwBBXBmTDEFyH80DOPNAhAD4h3enzi9YT01aBHCAoMYUC8T2ejc69HCAfDtIhRKDkg3pBwYVCLqUXg +BEUExIAWYAinSKfUALFGyIB8Oru/+434oaL40wfCVBRrMEYoeUz+oyxhVBJfSE6URRIpDYRO3Vc3KqFI +fp8hxmbQ73PFzHF0xqfHQvX/+8/Kyz8SpYmS5PcrgR/tdCkVCJzZ6TFTUd/J9qOzH4+dTQ+gTWrP++MT +ZzwhTEmkDJ0HCJE0Sl1wPiJXSgUhkCutbOwlrVaHA+JlY7gGcxwkYslQvtI26r/7If3rBnb9QsFAGigM +MIwpmwG5TpvIaTqV0gglK/rTUE37ETUIun89vri8ubv0ejNl0HVTBhP9/z49ee/1XEwmFGcD4vVpLPoP +770e0sAMekdLO0fn5IZGYGLKoBz0CyV9EVidxnV0nvRLZI23omUSUgYRSGygJV7IFrUMg0BDQFHp5tpy +fdZoHZ2T0RyhZWWF5qNHwYH4VjLXaryeYTOIIHFYEhOvF1OcGRfJvgH9IBiYNDKZX9IoBzDHEyGpx8n8 +c1Tlc/cxNoqofhoQ7wqw7OtUSMWgqZvbmA+Il7VfAS4kmJLGJlPOxqNxHAqWdOv/aZRciMZaccsaiWow +sZIGcoacnpws/1n1qpdrSXxI87KE/EODPyDeuz4HX0iReLt/kzPndj7gUtGHkw97Hu8KJGjBLrVWeqng +X3u3qzxO7JZrBTwagGPIOaFyBR818Bhy/rzwiKmmESDonPB8+U0Vf1p6TcjSpbIbN4NjyPktfLFg8E2B +8+RwwGm3xeYvMacIreGZdjschKb2dCB9vnHWcW//7+zrePS/VDGHEBC2RPQo6dwa0Wm3V0N0zgkrwHbV +y/KShi9WaOADgtpCdhmfYqfF1foyeFH8pn5L6jsdJSa/PHoPp85YWSVZdbyxQj2qqubL9SnOYKWSr14S +WfNhlKiTnDnlBPsWSscNcZuXjkIapJJBeiYAjSN4CFXkJGfMaxD0ZvwcUhVZT7sbkDqn3dZJJu03DMMD +SDWbuPBluYMppbmQFJVuQyIXy27lWK9KbGKZvKKObt4Q3ewY4Y6POj56K3y0I5QLhLVNvuqY6zmYi2b3 +BtoQ17q7EBUCm2hrGAR14Y+cUMdZL8lZOwU3O3x3sW3JW8VYd+TVkdfeyGsnTBeoq2nOmnS89VKndX3X +Y7Dr2c/YDUtD8W2LXbXrezjJylnTZatXwLGG5PvOUL5N9WQ3ZTIeFrLV/nGu53CQPTeoA/crHos1zdY7 +7jdL6XybPefB5vVK53Xr4EXXQfNsv+NSKNBBhWSX/TvUv9DRUtPkv9N+rZT6W+/Zunq+A/0eQd880++E ++0KeLwtuAnyX6zvY720b+/div9niCcP6Rx1Km1lfq6jVdvaVnzlceuX7euSwq9i3BvZ+7pmu1u3dEuiW +wCtVMq1XwD5uvJTvJzbFfWLFpAN/B/729czild0+00Cx4aF8/i3KIpYXr2SCScuYR4EzEimDRCVWGMLB +pzZE4NVYXsznIpnOd1+pjwrmvEadvjqDQ0V2rsX1rXhNMFU5zzBq+ieweSRi7TCIYhmIBAWbk9Icz0up +zS9ifY7nbxLnplZ4P+yZZ1ehuFp5CkwE6RR8XlWzQdV6dQWVE9BC8ZFN1/yq2BpzEg1WooigckqbfX9b +6FkIQUFbI+dPlUKDmsaXkk5D4CUXT5UKgWZJxg+tmTWUfdQCwdyrCxVFAq9VUNeBuX9s06loiKnQjYXX +YaDS2bcr4hk1SBqbmcKGowrJ4WuzEcc50WzSazDSFh+3lcY3wkct0CsBPg0V++tOfIOm8tb3Qf9s0ep2 +XSbUYJs5Ofq4/BoL/VQXuhXxoY+gbxQOGQNjGjtjXAJAI69DM3i1cfMaeLSYVLL0E4OEDCagLya/XCjJ +rNYgWdmf0kZTyDGmr3RE0dVEyk5DKCSJ9VrJFmqrXhduBflAGMwjpGa9zeWLXi4oacyD6c9i1PIc5TyZ +BQ0nlRTVmLzLz4a1mHC6i9yI0qptSIsRVp5urt/tLFy0+Dmc1akJiRBUwkdIPDstTLnFPBe7xGeK3Hiu +Psd8bgfwM2WodJ2N0kZ3M6q5qRMUJpGrTzvMonoAfS8q6raNDv7pw3KsT8IVuvWDRfRrMq07wDHfNNzC +SW3Cxms4QxgVJqsi+a2kGuFvStbVsY8gghnWOQ0kj5WQWKPMVEeVak3zmz2EqB5hiYsLimv97T7ZDzNt +nmmsNDYJXfu9w/cdwedxX+HYeZ++bOCelzU8Qe1uFq4EziBFqCGbNB1h7jzHKKsZjOv8N8+bN1Qqs23i +dDp8f2sVy7kX3JafJ0gbpY1HxBvfjO/Hw+vxb+ObK29xcfjrcHw9PL++zK5cXw5/nUtUvD2xFyLdKq2t +rIxsE6s0g0ZlS+4pjDdnRWNir6pycmWEG6RZKbGxXCrevG/hrRDog5DBODtpb++26uVGJRec4lsE09Y5 ++tmRlT+GbLNjXcpXBqTy5HibzdFNPW0kF3s1J3DLNDfPYaFiNPTyV1hoDW5RSD/reiscylSW4GuOYmpY +7Xwhl+foPcHso0qxdV6cS6PAY72J8DUGhsDvkp9tdUhLahAzAf1RWb0NT35U+y1DKecajNmx3nnFgnb/ +Lq6+k7NNSmh6UFNxK7T1AUNBx/8DAAD//y+7cdK5WAAA `, }, diff --git a/src/query/generated/assets/openapi/spec.yml b/src/query/generated/assets/openapi/spec.yml index 2efbc9eec9..20730c6a67 100644 --- a/src/query/generated/assets/openapi/spec.yml +++ b/src/query/generated/assets/openapi/spec.yml @@ -617,60 +617,6 @@ paths: description: "" schema: $ref: "#/definitions/GenericError" - /services/m3db/database/config/bootstrappers: - post: - tags: - - "M3DB Database" - summary: "Dynamically override the bootstrappers configured in M3DBs node-level configuration" - operationId: "databaseConfigSetBootstrappers" - consumes: - - "application/json" - produces: - - "application/json" - parameters: - - name: "body" - in: "body" - schema: - $ref: "#/definitions/DatabaseConfigBootstrappers" - responses: - 200: - description: "" - schema: - $ref: "#/definitions/DatabaseConfigBootstrappers" - 400: - description: "" - schema: - $ref: "#/definitions/GenericError" - 500: - description: "" - schema: - $ref: "#/definitions/GenericError" - get: - tags: - - "M3DB Database" - summary: "Retrieve the dynamically configured bootstrappers override, if any" - operationId: "databaseConfigGetBootstrappers" - consumes: - - "application/json" - produces: - - "application/json" - responses: - 200: - description: "" - schema: - $ref: "#/definitions/DatabaseConfigBootstrappers" - 400: - description: "" - schema: - $ref: "#/definitions/GenericError" - 404: - description: "not found if not set" - schema: - $ref: "#/definitions/GenericError" - 500: - description: "" - schema: - $ref: "#/definitions/GenericError" definitions: NamespaceAddRequest: type: "object" @@ -969,10 +915,3 @@ definitions: $ref: "#/definitions/NamespaceGetResponse" placement: $ref: "#/definitions/PlacementGetResponse" - DatabaseConfigBootstrappers: - type: "object" - properties: - values: - type: "array" - items: - type: "string" diff --git a/src/query/generated/mocks/generate.go b/src/query/generated/mocks/generate.go index 95393fd6e6..199b0eac0b 100644 --- a/src/query/generated/mocks/generate.go +++ b/src/query/generated/mocks/generate.go @@ -27,7 +27,6 @@ //go:generate sh -c "mockgen -package=ingest -destination=$GOPATH/src/$PACKAGE/src/cmd/services/m3coordinator/ingest/write_mock.go $PACKAGE/src/cmd/services/m3coordinator/ingest DownsamplerAndWriter" //go:generate sh -c "mockgen -package=transform -destination=$GOPATH/src/$PACKAGE/src/query/executor/transform/types_mock.go $PACKAGE/src/query/executor/transform OpNode" //go:generate sh -c "mockgen -package=executor -destination=$GOPATH/src/$PACKAGE/src/query/executor/types_mock.go $PACKAGE/src/query/executor Engine" -//go:generate sh -c "mockgen -package=cost -destination=$GOPATH/src/github.com/m3db/m3/src/query/cost/cost_mock.go $PACKAGE/src/query/cost ChainedEnforcer,ChainedReporter" //go:generate sh -c "mockgen -package=storage -destination=$GOPATH/src/$PACKAGE/src/query/graphite/storage/storage_mock.go $PACKAGE/src/query/graphite/storage Storage" // mockgen rules for generating mocks for unexported interfaces (file mode). diff --git a/src/query/graphite/storage/m3_wrapper.go b/src/query/graphite/storage/m3_wrapper.go index d5c474ceaa..0ba21ab747 100644 --- a/src/query/graphite/storage/m3_wrapper.go +++ b/src/query/graphite/storage/m3_wrapper.go @@ -29,7 +29,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" xctx "github.com/m3db/m3/src/query/graphite/context" "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/query/graphite/ts" @@ -48,7 +47,6 @@ var ( type m3WrappedStore struct { m3 storage.Storage - enforcer cost.ChainedEnforcer m3dbOpts m3db.Options instrumentOpts instrument.Options opts M3WrappedStorageOptions @@ -78,18 +76,12 @@ type seriesMetadata struct { // storage instance. func NewM3WrappedStorage( m3storage storage.Storage, - enforcer cost.ChainedEnforcer, m3dbOpts m3db.Options, instrumentOpts instrument.Options, opts M3WrappedStorageOptions, ) Storage { - if enforcer == nil { - enforcer = cost.NoopChainedEnforcer() - } - return &m3WrappedStore{ m3: m3storage, - enforcer: enforcer, m3dbOpts: m3dbOpts, instrumentOpts: instrumentOpts, opts: opts, @@ -396,12 +388,9 @@ func (s *m3WrappedStore) FetchByQuery( defer cancel() fetchOptions := storage.NewFetchOptions() fetchOptions.SeriesLimit = fetchOpts.Limit - perQueryEnforcer := s.enforcer.Child(cost.QueryLevel) - defer perQueryEnforcer.Close() // NB: ensure single block return. fetchOptions.BlockType = models.TypeSingleBlock - fetchOptions.Enforcer = perQueryEnforcer fetchOptions.FanoutOptions = &storage.FanoutOptions{ FanoutUnaggregated: storage.FanoutForceDisable, FanoutAggregated: storage.FanoutDefault, diff --git a/src/query/graphite/storage/m3_wrapper_test.go b/src/query/graphite/storage/m3_wrapper_test.go index cb52d52fab..bb57ce9a1b 100644 --- a/src/query/graphite/storage/m3_wrapper_test.go +++ b/src/query/graphite/storage/m3_wrapper_test.go @@ -27,7 +27,6 @@ import ( "time" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" xctx "github.com/m3db/m3/src/query/graphite/context" "github.com/m3db/m3/src/query/graphite/graphite" "github.com/m3db/m3/src/query/models" @@ -195,13 +194,7 @@ func TestFetchByQuery(t *testing.T) { store.EXPECT().FetchBlocks(gomock.Any(), gomock.Any(), gomock.Any()). Return(res, nil) - childEnforcer := cost.NewMockChainedEnforcer(ctrl) - childEnforcer.EXPECT().Close() - - enforcer := cost.NewMockChainedEnforcer(ctrl) - enforcer.EXPECT().Child(cost.QueryLevel).Return(childEnforcer).MinTimes(1) - - wrapper := NewM3WrappedStorage(store, enforcer, testM3DBOpts, + wrapper := NewM3WrappedStorage(store, testM3DBOpts, instrument.NewOptions(), M3WrappedStorageOptions{}) ctx := xctx.New() ctx.SetRequestContext(context.TODO()) @@ -243,7 +236,7 @@ func TestFetchByInvalidQuery(t *testing.T) { query := "a." ctx := xctx.New() - wrapper := NewM3WrappedStorage(store, nil, testM3DBOpts, + wrapper := NewM3WrappedStorage(store, testM3DBOpts, instrument.NewOptions(), M3WrappedStorageOptions{}) result, err := wrapper.FetchByQuery(ctx, query, opts) assert.NoError(t, err) diff --git a/src/query/models/block_type.go b/src/query/models/block_type.go index 1cae3fee8e..ba17c71307 100644 --- a/src/query/models/block_type.go +++ b/src/query/models/block_type.go @@ -31,8 +31,6 @@ func (t FetchedBlockType) Validate() error { return nil case TypeMultiBlock: return ErrMultiBlockDisabled - case TypeDecodedBlock: - return ErrDecodedBlockDeprecated default: return fmt.Errorf(`invalid fetched block type "%v"`, t) } diff --git a/src/query/models/block_type_test.go b/src/query/models/block_type_test.go index a5083d81cf..04665d5e2b 100644 --- a/src/query/models/block_type_test.go +++ b/src/query/models/block_type_test.go @@ -28,8 +28,6 @@ import ( func TestFetchedBlockType(t *testing.T) { assert.NoError(t, TypeSingleBlock.Validate()) - assert.EqualError(t, TypeDecodedBlock.Validate(), - ErrDecodedBlockDeprecated.Error()) assert.EqualError(t, TypeMultiBlock.Validate(), ErrMultiBlockDisabled.Error()) assert.EqualError(t, FetchedBlockType(3).Validate(), diff --git a/src/query/models/config.go b/src/query/models/config.go index 618afe0359..0d0e2cc882 100644 --- a/src/query/models/config.go +++ b/src/query/models/config.go @@ -26,7 +26,6 @@ import ( ) var validIDSchemes = []IDSchemeType{ - TypeLegacy, TypeQuoted, TypePrependMeta, TypeGraphite, @@ -38,7 +37,7 @@ func (t IDSchemeType) Validate() error { return errors.New("id scheme type not set") } - if t >= TypeLegacy && t <= TypeGraphite { + if t >= TypeQuoted && t <= TypeGraphite { return nil } @@ -50,8 +49,6 @@ func (t IDSchemeType) String() string { switch t { case TypeDefault: return "" - case TypeLegacy: - return "legacy" case TypeQuoted: return "quoted" case TypePrependMeta: diff --git a/src/query/models/config_test.go b/src/query/models/config_test.go index 949680db33..e5a4a09548 100644 --- a/src/query/models/config_test.go +++ b/src/query/models/config_test.go @@ -32,8 +32,6 @@ import ( func TestIDSchemeValidation(t *testing.T) { err := TypeDefault.Validate() assert.EqualError(t, err, "id scheme type not set") - err = TypeLegacy.Validate() - assert.NoError(t, err) err = TypePrependMeta.Validate() assert.NoError(t, err) err = TypeQuoted.Validate() @@ -42,7 +40,7 @@ func TestIDSchemeValidation(t *testing.T) { assert.NoError(t, err) err = IDSchemeType(5).Validate() assert.EqualError(t, err, "invalid config id schema type 'unknown':"+ - " should be one of [legacy quoted prepend_meta graphite]") + " should be one of [quoted prepend_meta graphite]") } func TestMetricsTypeUnmarshalYAML(t *testing.T) { @@ -51,7 +49,6 @@ func TestMetricsTypeUnmarshalYAML(t *testing.T) { } validParseSchemes := []IDSchemeType{ - TypeLegacy, TypeQuoted, TypePrependMeta, } diff --git a/src/query/models/options.go b/src/query/models/options.go index d54bb2623b..794c91d9f5 100644 --- a/src/query/models/options.go +++ b/src/query/models/options.go @@ -53,7 +53,7 @@ func NewTagOptions() TagOptions { version: 0, metricName: defaultMetricName, bucketName: defaultBucketName, - idScheme: TypeLegacy, + idScheme: TypeQuoted, allowTagNameDuplicates: defaultAllowTagNameDuplicates, allowTagValueEmpty: defaultAllowTagValueEmpty, } diff --git a/src/query/models/options_test.go b/src/query/models/options_test.go index 0014ccd93a..ccc65641b3 100644 --- a/src/query/models/options_test.go +++ b/src/query/models/options_test.go @@ -30,7 +30,7 @@ func TestDefaultTagOptions(t *testing.T) { opts := NewTagOptions() assert.NoError(t, opts.Validate()) assert.Equal(t, defaultMetricName, opts.MetricName()) - assert.Equal(t, TypeLegacy, opts.IDSchemeType()) + assert.Equal(t, TypeQuoted, opts.IDSchemeType()) } func TestValidTagOptions(t *testing.T) { @@ -69,7 +69,7 @@ func TestBadBucketTagOptions(t *testing.T) { func TestBadSchemeTagOptions(t *testing.T) { msg := "invalid config id schema type 'unknown': should be one of" + - " [legacy quoted prepend_meta graphite]" + " [quoted prepend_meta graphite]" opts := NewTagOptions(). SetIDSchemeType(IDSchemeType(6)) assert.EqualError(t, opts.Validate(), msg) diff --git a/src/query/models/params.go b/src/query/models/params.go index 2ea97bb1ff..3ddbe9adf7 100644 --- a/src/query/models/params.go +++ b/src/query/models/params.go @@ -31,18 +31,12 @@ type FormatType int const ( // FormatPromQL returns results in Prom format FormatPromQL FormatType = iota - // FormatM3QL returns results in M3QL format - FormatM3QL infoMsg = "if this is causing issues for your use case, please file an " + "issue on https://github.com/m3db/m3" ) var ( - // ErrDecodedBlockDeprecated indicates decoded blocks are deprecated. - ErrDecodedBlockDeprecated = fmt.Errorf("decoded block has been deprecated %s", - infoMsg) - // ErrMultiBlockDisabled indicates multi blocks are temporarily disabled. ErrMultiBlockDisabled = fmt.Errorf("multiblock is temporarily disabled %s", infoMsg) @@ -61,12 +55,6 @@ const ( // // NB: Currently disabled. TypeMultiBlock - // TypeDecodedBlock represents a single block which contains all fetched series - // which get decoded. - // - // NB: this is a legacy block type, will be deprecated once there is - // sufficient confidence that other block types are performing correctly. - TypeDecodedBlock ) // RequestParams represents the params from the request. diff --git a/src/query/models/query_context.go b/src/query/models/query_context.go index abb87f47cf..0da1e3daf5 100644 --- a/src/query/models/query_context.go +++ b/src/query/models/query_context.go @@ -24,19 +24,15 @@ import ( "context" "github.com/m3db/m3/src/metrics/policy" - "github.com/m3db/m3/src/query/cost" "github.com/uber-go/tally" ) // QueryContext provides all external state needed to execute and track a query. -// It acts as a hook back into the execution engine for things like -// cost accounting. type QueryContext struct { - Ctx context.Context - Scope tally.Scope - Enforcer cost.ChainedEnforcer - Options QueryContextOptions + Ctx context.Context + Scope tally.Scope + Options QueryContextOptions } // QueryContextOptions contains optional configuration for the query context. @@ -58,26 +54,23 @@ type RestrictFetchTypeQueryContextOptions struct { StoragePolicy policy.StoragePolicy } -// NewQueryContext constructs a QueryContext using the given Enforcer to -// enforce per query limits. +// NewQueryContext constructs a QueryContext from the provided native context. func NewQueryContext( ctx context.Context, scope tally.Scope, - enforcer cost.ChainedEnforcer, options QueryContextOptions, ) *QueryContext { return &QueryContext{ - Ctx: ctx, - Scope: scope, - Enforcer: enforcer, - Options: options, + Ctx: ctx, + Scope: scope, + Options: options, } } // NoopQueryContext returns a query context with no active components. func NoopQueryContext() *QueryContext { return NewQueryContext(context.Background(), tally.NoopScope, - cost.NoopChainedEnforcer(), QueryContextOptions{}) + QueryContextOptions{}) } // WithContext creates a shallow copy of this QueryContext using the new context. diff --git a/src/query/models/tags_id_schemes.go b/src/query/models/tags_id_schemes.go index 33160247bc..fc2c52334f 100644 --- a/src/query/models/tags_id_schemes.go +++ b/src/query/models/tags_id_schemes.go @@ -36,8 +36,6 @@ func id(t Tags) []byte { } switch schemeType { - case TypeLegacy: - return legacyID(t) case TypeQuoted: return quotedID(t) case TypePrependMeta: @@ -51,20 +49,6 @@ func id(t Tags) []byte { } } -func legacyID(t Tags) []byte { - // TODO: pool these bytes. - id := make([]byte, idLen(t)) - idx := -1 - for _, tag := range t.Tags { - idx += copy(id[idx+1:], tag.Name) + 1 - id[idx] = eq - idx += copy(id[idx+1:], tag.Value) + 1 - id[idx] = sep - } - - return id -} - func idLen(t Tags) int { idLen := 2 * t.Len() // account for separators for _, tag := range t.Tags { diff --git a/src/query/models/tags_test.go b/src/query/models/tags_test.go index e5a0df6cca..859b51bd5e 100644 --- a/src/query/models/tags_test.go +++ b/src/query/models/tags_test.go @@ -32,7 +32,6 @@ import ( xerrors "github.com/m3db/m3/src/x/errors" xtest "github.com/m3db/m3/src/x/test" - "github.com/cespare/xxhash/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -49,13 +48,6 @@ func testLongTagIDOutOfOrder(t *testing.T, scheme IDSchemeType) Tags { return tags } -func TestLongTagNewIDOutOfOrderLegacy(t *testing.T) { - tags := testLongTagIDOutOfOrder(t, TypeLegacy) - actual := tags.ID() - assert.Equal(t, idLen(tags), len(actual)) - assert.Equal(t, []byte("t1=v1,t2=v2,t3=v3,t4=v4,"), actual) -} - func TestLongTagNewIDOutOfOrderQuoted(t *testing.T) { tags := testLongTagIDOutOfOrder(t, TypeQuoted) actual := tags.ID() @@ -84,14 +76,6 @@ func TestLongTagNewIDOutOfOrderGraphite(t *testing.T) { assert.Equal(t, []byte("v0.v1.v2.v3.v4.v5.v6.v7.v8.v9.v10.v11.v12"), actual) } -func TestHashedID(t *testing.T) { - tags := testLongTagIDOutOfOrder(t, TypeLegacy) - actual := tags.HashedID() - - expected := xxhash.Sum64String("t1=v1,t2=v2,t3=v3,t4=v4,") - assert.Equal(t, expected, actual) -} - func TestLongTagNewIDOutOfOrderQuotedWithEscape(t *testing.T) { tags := testLongTagIDOutOfOrder(t, TypeQuoted) tags = tags.AddTag(Tag{Name: []byte(`t5""`), Value: []byte(`v"5`)}) @@ -580,7 +564,6 @@ func TestEmptyTags(t *testing.T) { idType IDSchemeType expected string }{ - {TypeLegacy, ""}, {TypePrependMeta, ""}, {TypeGraphite, ""}, {TypeQuoted, "{}"}, @@ -609,7 +592,6 @@ var tagIDSchemes = []struct { name string scheme IDSchemeType }{ - {"___legacy", TypeLegacy}, {"_graphite", TypeGraphite}, {"__prepend", TypePrependMeta}, // only simple quotable tag values. diff --git a/src/query/models/types.go b/src/query/models/types.go index ece31d6a55..66ce809fd4 100644 --- a/src/query/models/types.go +++ b/src/query/models/types.go @@ -42,16 +42,6 @@ const ( // TypeDefault is an invalid scheme that indicates that the default scheme // for the tag options version option should be used. TypeDefault IDSchemeType = iota - // TypeLegacy describes a scheme where IDs are generated by appending - // tag name/value pairs with = and , separators. Note that an additional , is - // added to the end of the ID. - // - // NB: this should not be used, and exists here as a deprecated legacy - // ID generation scheme, as it may cause collisions in situations where - // incoming tags contain the following characters: << =," >>, for example: - // {t1:v1},{t2:v2} -> t1=v1,t2=v2, - // {t1:v1,t2:v2} -> t1=v1,t2=v2, - TypeLegacy // TypeQuoted describes a scheme where IDs are generated by appending // tag names with explicitly quoted and escaped tag values. Tag names are // also escaped if they contain invalid characters. This is equivalent to diff --git a/src/query/remote/client.go b/src/query/remote/client.go index 4fc3d93e18..e4c6e0ba7d 100644 --- a/src/query/remote/client.go +++ b/src/query/remote/client.go @@ -243,8 +243,9 @@ func (c *grpcClient) FetchProm( } return storage.SeriesIteratorsToPromResult( - result, c.opts.ReadWorkerPool(), - options.Enforcer, c.opts.TagOptions()) + result, + c.opts.ReadWorkerPool(), + c.opts.TagOptions()) } func (c *grpcClient) fetchRaw( diff --git a/src/query/server/cost_reporters.go b/src/query/server/cost_reporters.go deleted file mode 100644 index 1f6e2d0b5a..0000000000 --- a/src/query/server/cost_reporters.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package server - -// This file contains reporters and setup for our query/cost.ChainedEnforcer -// instances. -import ( - "fmt" - "sync" - - "github.com/m3db/m3/src/cmd/services/m3query/config" - qcost "github.com/m3db/m3/src/query/cost" - "github.com/m3db/m3/src/x/close" - "github.com/m3db/m3/src/x/cost" - "github.com/m3db/m3/src/x/instrument" - - "github.com/uber-go/tally" -) - -const ( - costScopeName = "cost" - limitManagerScopeName = "limits" - reporterScopeName = "reporter" - queriesOverLimitMetric = "over_datapoints_limit" - datapointsMetric = "datapoints" - datapointsCounterMetric = "datapoints_counter" - maxDatapointsHistMetric = "max_datapoints_hist" -) - -// newConfiguredChainedEnforcer returns a ChainedEnforcer with 3 configured -// levels: global, per-query, per-block. Global and per-query both have limits -// on them (as configured by cfg.Limits); per-block is purely for accounting -// purposes. -// Our enforcers report at least these stats: -// cost_reporter_datapoints{limit="global"}: gauge; -// > the number of datapoints currently in use by this instance. -// -// cost_reporter_datapoints_counter{limiter="global"}: counter; -// > counter representation of the number of datapoints in use by this instance. -// -// cost_reporter_over_datapoints_limit{limiter=~"(global|per_query)"}: counter; -// > how many queries are over the datapoint limit. -// -// cost_reporter_max_datapoints_hist{limiter=~"(global|per_query)"}: histogram; -// > represents the distribution of the maximum datapoints used at any point in each query. -func newConfiguredChainedEnforcer( - cfg *config.Configuration, - instrumentOptions instrument.Options, -) (qcost.ChainedEnforcer, close.SimpleCloser, error) { - scope := instrumentOptions.MetricsScope().SubScope(costScopeName) - - exceededMessage := func(exceedType, exceedLimit string) string { - return fmt.Sprintf("exceeded limits.%s.%s", exceedType, exceedLimit) - } - - // Create global limit manager and enforcer. - globalScope := scope.Tagged(map[string]string{ - "limiter": "global", - }) - globalLimitManagerScope := globalScope.SubScope(limitManagerScopeName) - globalReporterScope := globalScope.SubScope(reporterScopeName) - - globalLimitMgr := cost.NewStaticLimitManager( - cfg.Limits.Global.AsLimitManagerOptions(). - SetInstrumentOptions(instrumentOptions.SetMetricsScope(globalLimitManagerScope))) - - globalTracker := cost.NewTracker() - - globalEnforcer := cost.NewEnforcer(globalLimitMgr, globalTracker, - cost.NewEnforcerOptions(). - SetReporter(newGlobalReporter(globalReporterScope)). - SetCostExceededMessage(exceededMessage("global", "maxFetchedDatapoints"))) - - // Create per query limit manager and enforcer. - queryScope := scope.Tagged(map[string]string{ - "limiter": "query", - }) - queryLimitManagerScope := queryScope.SubScope(limitManagerScopeName) - queryReporterScope := queryScope.SubScope(reporterScopeName) - - queryLimitMgr := cost.NewStaticLimitManager( - cfg.Limits.PerQuery.AsLimitManagerOptions(). - SetInstrumentOptions(instrumentOptions.SetMetricsScope(queryLimitManagerScope))) - - queryTracker := cost.NewTracker() - - queryEnforcer := cost.NewEnforcer(queryLimitMgr, queryTracker, - cost.NewEnforcerOptions(). - SetReporter(newPerQueryReporter(queryReporterScope)). - SetCostExceededMessage(exceededMessage("perQuery", "maxFetchedDatapoints"))) - - // Create block enforcer. - blockEnforcer := cost.NewEnforcer( - cost.NewStaticLimitManager(cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{Enabled: false})), - cost.NewTracker(), - nil) - - // Create chained enforcer. - enforcer, err := qcost.NewChainedEnforcer(qcost.GlobalLevel, []cost.Enforcer{ - globalEnforcer, - queryEnforcer, - blockEnforcer, - }) - if err != nil { - return nil, nil, err - } - - // Start reporting stats for all limit managers. - go globalLimitMgr.Report() - go queryLimitMgr.Report() - - // Close the stats at the end. - closer := close.SimpleCloserFn(func() { - globalLimitMgr.Close() - queryLimitMgr.Close() - }) - - return enforcer, closer, nil -} - -// globalReporter records ChainedEnforcer statistics for the global enforcer. -type globalReporter struct { - datapoints tally.Gauge - datapointsCounter tally.Counter - overLimit overLimitReporter -} - -// assert we implement the interface -var _ cost.EnforcerReporter = (*globalReporter)(nil) - -func newGlobalReporter(s tally.Scope) *globalReporter { - return &globalReporter{ - datapoints: s.Gauge(datapointsMetric), - datapointsCounter: s.Counter(datapointsCounterMetric), - overLimit: newOverLimitReporter(s), - } -} - -func (gr *globalReporter) ReportCurrent(c cost.Cost) { - gr.datapoints.Update(float64(c)) -} - -// ReportCost for global reporters sends the new incoming cost to a counter. -// Since counters can only be incremented, it ignores negative values. -func (gr *globalReporter) ReportCost(c cost.Cost) { - if c > 0 { - gr.datapointsCounter.Inc(int64(c)) - } -} - -// ReportOverLimit delegates to gr.overLimit -func (gr *globalReporter) ReportOverLimit(enabled bool) { - gr.overLimit.ReportOverLimit(enabled) -} - -// perQueryReporter records ChainedEnforcer statistics on a per query level. -type perQueryReporter struct { - mu *sync.Mutex - maxDatapoints cost.Cost - queryHisto tally.Histogram - overLimit overLimitReporter -} - -// assert we implement the interface -var _ qcost.ChainedReporter = (*perQueryReporter)(nil) - -func newPerQueryReporter(scope tally.Scope) *perQueryReporter { - return &perQueryReporter{ - mu: &sync.Mutex{}, - maxDatapoints: 0, - queryHisto: scope.Histogram(maxDatapointsHistMetric, - tally.MustMakeExponentialValueBuckets(10.0, 10.0, 6)), - overLimit: newOverLimitReporter(scope), - } -} - -// ReportCost is a noop for perQueryReporter because it's noisy to report -// the current cost for every query (hard to meaningfully divide out). -// Instead, we report the max datapoints at the end of the query--see on -// release. -func (perQueryReporter) ReportCost(c cost.Cost) {} - -// ReportCurrent is a noop for perQueryReporter--see ReportCost for -// explanation. -func (perQueryReporter) ReportCurrent(c cost.Cost) {} - -// ReportOverLimit reports when a query is over its per query limit. -func (pr *perQueryReporter) ReportOverLimit(enabled bool) { - pr.overLimit.ReportOverLimit(enabled) -} - -// OnChildClose takes the max of the current cost for this query and the -// previously recorded cost. We do this OnChildRelease instead of on -// ReportCurrent to avoid locking every time we add to the Enforcer. -func (pr *perQueryReporter) OnChildClose(curCost cost.Cost) { - pr.mu.Lock() - if curCost > pr.maxDatapoints { - pr.maxDatapoints = curCost - } - pr.mu.Unlock() -} - -// OnClose records the maximum cost seen by this reporter. -func (pr *perQueryReporter) OnClose(curCost cost.Cost) { - pr.mu.Lock() - pr.queryHisto.RecordValue(float64(pr.maxDatapoints)) - pr.mu.Unlock() -} - -// overLimitReporter factors out reporting over limit cases for both global -// and per query enforcer reporters. -type overLimitReporter struct { - queriesOverLimitDisabled tally.Counter - queriesOverLimitEnabled tally.Counter -} - -func newOverLimitReporter(scope tally.Scope) overLimitReporter { - return overLimitReporter{ - queriesOverLimitDisabled: scope.Tagged(map[string]string{ - "enabled": "false", - }).Counter(queriesOverLimitMetric), - - queriesOverLimitEnabled: scope.Tagged(map[string]string{ - "enabled": "true", - }).Counter(queriesOverLimitMetric), - } -} - -// ReportOverLimit increments .over_limit, tagged by -// "enabled". -func (olr overLimitReporter) ReportOverLimit(enabled bool) { - if enabled { - olr.queriesOverLimitEnabled.Inc(1) - } else { - olr.queriesOverLimitDisabled.Inc(1) - } -} diff --git a/src/query/server/cost_reporters_test.go b/src/query/server/cost_reporters_test.go deleted file mode 100644 index a301123bb8..0000000000 --- a/src/query/server/cost_reporters_test.go +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package server - -import ( - "fmt" - "math" - "testing" - - "github.com/m3db/m3/src/cmd/services/m3query/config" - "github.com/m3db/m3/src/query/cost" - "github.com/m3db/m3/src/x/close" - "github.com/m3db/m3/src/x/cost/test" - "github.com/m3db/m3/src/x/instrument" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally" -) - -type enforcerTestCtx struct { - Scope tally.TestScope - GlobalEnforcer cost.ChainedEnforcer - Closer close.SimpleCloser -} - -func (c enforcerTestCtx) Close() { - c.Closer.Close() -} - -func TestNewConfiguredChainedEnforcer(t *testing.T) { - setup := func(t *testing.T, perQueryLimit, globalLimit int) enforcerTestCtx { - s := tally.NewTestScope("", nil) - iopts := instrument.NewOptions().SetMetricsScope(s) - - globalEnforcer, closer, err := newConfiguredChainedEnforcer(&config.Configuration{ - Limits: config.LimitsConfiguration{ - PerQuery: config.PerQueryLimitsConfiguration{ - MaxFetchedDatapoints: perQueryLimit, - }, - Global: config.GlobalLimitsConfiguration{ - MaxFetchedDatapoints: globalLimit, - }, - }, - }, iopts) - - require.NoError(t, err) - - return enforcerTestCtx{ - Scope: s, - GlobalEnforcer: globalEnforcer, - Closer: closer, - } - } - - t.Run("has 3 valid levels", func(t *testing.T) { - tctx := setup(t, 6, 10) - defer tctx.Close() - - assertValid := func(ce cost.ChainedEnforcer) { - assert.NotEqual(t, ce, cost.NoopChainedEnforcer()) - } - - assertValid(tctx.GlobalEnforcer) - - qe := tctx.GlobalEnforcer.Child(cost.QueryLevel) - assertValid(qe) - - block := qe.Child(cost.BlockLevel) - assertValid(block) - - badLevel := block.Child("nonExistent") - assert.Equal(t, cost.NoopChainedEnforcer(), - badLevel) - }) - - t.Run("configures reporters", func(t *testing.T) { - tctx := setup(t, 6, 10) - defer tctx.Close() - - queryEf := tctx.GlobalEnforcer.Child(cost.QueryLevel) - blockEf := queryEf.Child(cost.BlockLevel) - blockEf.Add(7) - - assertHasGauge(t, - tctx.Scope.Snapshot(), - tally.KeyForPrefixedStringMap( - fmt.Sprintf("cost.reporter.%s", datapointsMetric), map[string]string{"limiter": "global"}), - 7, - ) - - blockEf.Close() - queryEf.Close() - - assertHasHistogram(t, - tctx.Scope.Snapshot(), - tally.KeyForPrefixedStringMap( - fmt.Sprintf("cost.reporter.%s", maxDatapointsHistMetric), map[string]string{"limiter": "query"}), - map[float64]int64{10: 1}, - ) - }) - - t.Run("block level doesn't have a limit", func(t *testing.T) { - tctx := setup(t, -1, -1) - defer tctx.Close() - - block := tctx.GlobalEnforcer.Child(cost.QueryLevel).Child(cost.BlockLevel) - assert.NoError(t, block.Add(math.MaxFloat64-1).Error) - }) - - t.Run("works e2e", func(t *testing.T) { - tctx := setup(t, 6, 10) - defer tctx.Close() - - qe1, qe2 := tctx.GlobalEnforcer.Child(cost.QueryLevel), tctx.GlobalEnforcer.Child(cost.QueryLevel) - r := qe1.Add(6) - test.AssertLimitErrorWithMsg( - t, - r.Error, - "exceeded query limit: exceeded limits.perQuery.maxFetchedDatapoints", - 6, - 6) - - r = qe2.Add(3) - require.NoError(t, r.Error) - - r = qe2.Add(2) - test.AssertLimitErrorWithMsg( - t, - r.Error, - "exceeded global limit: exceeded limits.global.maxFetchedDatapoints", - 11, - 10) - - test.AssertCurrentCost(t, 11, tctx.GlobalEnforcer) - - qe2.Close() - test.AssertCurrentCost(t, 6, tctx.GlobalEnforcer) - - // check the block level - blockEf := qe1.Child(cost.BlockLevel) - blockEf.Add(2) - - test.AssertCurrentCost(t, 2, blockEf) - test.AssertCurrentCost(t, 8, qe1) - test.AssertCurrentCost(t, 8, tctx.GlobalEnforcer) - }) -} - -func setupGlobalReporter() (tally.TestScope, *globalReporter) { - s := tally.NewTestScope("", nil) - gr := newGlobalReporter(s) - return s, gr -} - -func TestGlobalReporter_ReportCurrent(t *testing.T) { - s, gr := setupGlobalReporter() - - gr.ReportCurrent(5.0) - assertHasGauge(t, s.Snapshot(), tally.KeyForPrefixedStringMap(datapointsMetric, nil), 5.0) -} - -func TestGlobalReporter_ReportCost(t *testing.T) { - t.Run("reports positive", func(t *testing.T) { - s, gr := setupGlobalReporter() - gr.ReportCost(5.0) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(datapointsCounterMetric, nil), 5.0) - }) - - t.Run("skips negative", func(t *testing.T) { - s, gr := setupGlobalReporter() - gr.ReportCost(-5.0) - - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(datapointsCounterMetric, nil), 0.0) - }) -} - -func TestGlobalReporter_ReportOverLimit(t *testing.T) { - s, gr := setupGlobalReporter() - gr.ReportOverLimit(true) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) -} - -func setupPerQueryReporter() (tally.TestScope, *perQueryReporter) { - s := tally.NewTestScope("", nil) - gr := newPerQueryReporter(s) - return s, gr -} - -func TestPerQueryReporter_ReportOverLimit(t *testing.T) { - s, pqr := setupPerQueryReporter() - pqr.ReportOverLimit(true) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) -} - -func TestPerQueryReporter_OnClose(t *testing.T) { - s, pqr := setupPerQueryReporter() - pqr.OnChildClose(5.0) - pqr.OnChildClose(110.0) - - // ignores current cost - pqr.OnClose(100.0) - assertHasHistogram(t, s.Snapshot(), - tally.KeyForPrefixedStringMap(maxDatapointsHistMetric, nil), - map[float64]int64{ - 1000.0: 1, - }) -} - -func TestPerQueryReporter_OnChildClose(t *testing.T) { - _, pqr := setupPerQueryReporter() - pqr.OnChildClose(5.0) - pqr.OnChildClose(110.0) - - assert.InDelta(t, 110.0, float64(pqr.maxDatapoints), 0.0001) -} - -func TestOverLimitReporter_ReportOverLimit(t *testing.T) { - s := tally.NewTestScope("", nil) - orl := newOverLimitReporter(s) - - orl.ReportOverLimit(true) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) - - orl.ReportOverLimit(false) - assertHasCounter(t, s.Snapshot(), tally.KeyForPrefixedStringMap(queriesOverLimitMetric, map[string]string{ - "enabled": "true", - }), 1) -} - -func assertHasCounter(t *testing.T, snapshot tally.Snapshot, key string, v int) { - counters := snapshot.Counters() - if !assert.Contains(t, counters, key, "No such metric: %s", key) { - return - } - - counter := counters[key] - - assert.Equal(t, int(counter.Value()), v, "Incorrect value for counter %s", key) -} - -func assertHasGauge(t *testing.T, snapshot tally.Snapshot, key string, v int) { - gauges := snapshot.Gauges() - if !assert.Contains(t, gauges, key, "No such metric: %s", key) { - return - } - - gauge := gauges[key] - - assert.Equal(t, int(gauge.Value()), v, "Incorrect value for gauge %s", key) -} - -func assertHasHistogram(t *testing.T, snapshot tally.Snapshot, key string, values map[float64]int64) { - histograms := snapshot.Histograms() - if !assert.Contains(t, histograms, key, "No such metric: %s", key) { - return - } - - hist := histograms[key] - - actualValues := hist.Values() - - // filter zero values - for k, v := range actualValues { - if v == 0 { - delete(actualValues, k) - } - } - - assert.Equal(t, values, actualValues) -} diff --git a/src/query/server/query.go b/src/query/server/query.go index 2f6da9f799..054e7055c1 100644 --- a/src/query/server/query.go +++ b/src/query/server/query.go @@ -409,13 +409,6 @@ func Run(runOpts RunOptions) { logger.Fatal("unrecognized backend", zap.String("backend", string(cfg.Backend))) } - chainedEnforcer, chainedEnforceCloser, err := newConfiguredChainedEnforcer(&cfg, - instrumentOptions) - if err != nil { - logger.Fatal("unable to setup chained enforcer", zap.Error(err)) - } - - defer chainedEnforceCloser.Close() if fn := runOpts.BackendStorageTransform; fn != nil { backendStorage = fn(backendStorage, tsdbOpts, instrumentOptions) } @@ -423,7 +416,6 @@ func Run(runOpts RunOptions) { engineOpts := executor.NewEngineOptions(). SetStore(backendStorage). SetLookbackDuration(*cfg.LookbackDuration). - SetGlobalEnforcer(chainedEnforcer). SetInstrumentOptions(instrumentOptions. SetMetricsScope(instrumentOptions.MetricsScope().SubScope("engine"))) if fn := runOpts.CustomPromQLParseFunction; fn != nil { @@ -478,7 +470,7 @@ func Run(runOpts RunOptions) { instrumentOptions) handlerOptions, err := options.NewHandlerOptions(downsamplerAndWriter, tagOptions, engine, prometheusEngine, m3dbClusters, clusterClient, cfg, - runOpts.DBConfig, chainedEnforcer, fetchOptsBuilder, queryCtxOpts, + runOpts.DBConfig, fetchOptsBuilder, queryCtxOpts, instrumentOptions, cpuProfileDuration, []string{handleroptions.M3DBServiceName}, serviceOptionDefaults, httpd.NewQueryRouter(), httpd.NewQueryRouter(), graphiteStorageOpts, tsdbOpts) @@ -496,12 +488,7 @@ func Run(runOpts RunOptions) { logger.Fatal("unable to register routes", zap.Error(err)) } - listenAddress, err := cfg.ListenAddress.Resolve() - if err != nil { - logger.Fatal("unable to get listen address", zap.Error(err)) - } - - srv := &http.Server{Addr: listenAddress, Handler: handler.Router()} + srv := &http.Server{Addr: cfg.ListenAddress, Handler: handler.Router()} defer func() { logger.Info("closing server") if err := srv.Shutdown(context.Background()); err != nil { @@ -509,10 +496,10 @@ func Run(runOpts RunOptions) { } }() - listener, err := listenerOpts.Listen("tcp", listenAddress) + listener, err := listenerOpts.Listen("tcp", cfg.ListenAddress) if err != nil { logger.Fatal("unable to listen on listen address", - zap.String("address", listenAddress), + zap.String("address", cfg.ListenAddress), zap.Error(err)) } if runOpts.ListenerCh != nil { @@ -522,7 +509,7 @@ func Run(runOpts RunOptions) { logger.Info("starting API server", zap.Stringer("address", listener.Addr())) if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed { logger.Fatal("server serve error", - zap.String("address", listenAddress), + zap.String("address", cfg.ListenAddress), zap.Error(err)) } }() diff --git a/src/query/server/query_test.go b/src/query/server/query_test.go index 9563f98f58..d1f95e0171 100644 --- a/src/query/server/query_test.go +++ b/src/query/server/query_test.go @@ -42,14 +42,11 @@ import ( m3msgproto "github.com/m3db/m3/src/msg/protocol/proto" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/remote" "github.com/m3db/m3/src/query/api/v1/handler/prometheus/remote/test" - "github.com/m3db/m3/src/query/cost" rpc "github.com/m3db/m3/src/query/generated/proto/rpcpb" "github.com/m3db/m3/src/query/storage/m3" xclock "github.com/m3db/m3/src/x/clock" - "github.com/m3db/m3/src/x/close" xconfig "github.com/m3db/m3/src/x/config" "github.com/m3db/m3/src/x/ident" - "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/serialize" xtest "github.com/m3db/m3/src/x/test" @@ -57,15 +54,12 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/uber-go/tally" "go.uber.org/atomic" "google.golang.org/grpc" ) var configYAML = ` -listenAddress: - type: "config" - value: "127.0.0.1:0" +listenAddress: 127.0.0.1:0 logging: level: info @@ -411,9 +405,7 @@ func TestGRPCBackend(t *testing.T) { require.NoError(t, err) grpcAddr := lis.Addr().String() var grpcConfigYAML = fmt.Sprintf(` -listenAddress: - type: "config" - value: "127.0.0.1:0" +listenAddress: 127.0.0.1:0 logging: level: info @@ -507,131 +499,6 @@ writeWorkerPoolPolicy: <-doneCh } -func TestNewPerQueryEnforcer(t *testing.T) { - type testContext struct { - Global cost.ChainedEnforcer - Query cost.ChainedEnforcer - Block cost.ChainedEnforcer - Closer close.SimpleCloser - } - - scope := tally.NewTestScope("", nil) - instrumentOpts := instrument.NewTestOptions(t). - SetMetricsScope(scope) - - setup := func(t *testing.T, globalLimit, queryLimit int) testContext { - cfg := &config.Configuration{ - Limits: config.LimitsConfiguration{ - Global: config.GlobalLimitsConfiguration{ - MaxFetchedDatapoints: 100, - }, - PerQuery: config.PerQueryLimitsConfiguration{ - MaxFetchedDatapoints: 10, - }, - }, - } - - global, closer, err := newConfiguredChainedEnforcer(cfg, instrumentOpts) - require.NoError(t, err) - - queryLvl := global.Child(cost.QueryLevel) - blockLvl := queryLvl.Child(cost.BlockLevel) - - return testContext{ - Global: global, - Query: queryLvl, - Block: blockLvl, - Closer: closer, - } - } - - tctx := setup(t, 100, 10) - defer tctx.Closer.Close() - - // Spot check that limits are setup properly for each level. - r := tctx.Query.Add(11) - require.Error(t, r.Error) - - floatsEqual := func(f1, f2 float64) { - assert.InDelta(t, f1, f2, 0.0000001) - } - - floatsEqual(float64(r.Cost), 11) - - r, _ = tctx.Query.State() - floatsEqual(float64(r.Cost), 11) - - r, _ = tctx.Global.State() - floatsEqual(float64(r.Cost), 11) - require.NoError(t, r.Error) - - // Wait for stats reporting to start. - start := time.Now() - for time.Since(start) < 15*time.Second { - gauges := scope.Snapshot().Gauges() - globalEnabled, globalOk := gauges["cost.limits.enabled+limiter=global"] - queryEnabled, queryOk := gauges["cost.limits.enabled+limiter=query"] - if globalOk && queryOk && globalEnabled.Value() == 1 && queryEnabled.Value() == 1 { - break - } - - time.Sleep(100 * time.Millisecond) - } - - // Check stats. - expectCounterValues := map[string]int64{ - "cost.reporter.over_datapoints_limit+enabled=false,limiter=global": 0, - "cost.reporter.over_datapoints_limit+enabled=true,limiter=global": 0, - "cost.reporter.datapoints_counter+limiter=global": 11, - "cost.reporter.over_datapoints_limit+enabled=false,limiter=query": 0, - "cost.reporter.over_datapoints_limit+enabled=true,limiter=query": 1, - } - expectGaugeValues := map[string]float64{ - "cost.limits.threshold+limiter=global": 100, - "cost.limits.enabled+limiter=global": 1, - "cost.reporter.datapoints+limiter=global": 11, - "cost.limits.threshold+limiter=query": 10, - "cost.limits.enabled+limiter=query": 1, - } - - snapshot := scope.Snapshot() - actualCounterValues := make(map[string]int64) - for k, v := range snapshot.Counters() { - actualCounterValues[k] = v.Value() - - expected, ok := expectCounterValues[k] - if !ok { - continue - } - - // Check match. - assert.Equal(t, expected, v.Value(), - fmt.Sprintf("stat mismatch: stat=%s", k)) - - delete(expectCounterValues, k) - } - assert.Equal(t, 0, len(expectCounterValues), - fmt.Sprintf("missing stats: %+v", expectCounterValues)) - - actualGaugeValues := make(map[string]float64) - for k, v := range snapshot.Gauges() { - actualGaugeValues[k] = v.Value() - - expected, ok := expectGaugeValues[k] - if !ok { - continue - } - - // Check match. - assert.Equal(t, expected, v.Value(), - fmt.Sprintf("stat mismatch: stat=%s", k)) - - delete(expectGaugeValues, k) - } - assert.Equal(t, 0, len(expectGaugeValues), - fmt.Sprintf("missing stats: %+v", expectGaugeValues)) -} - var _ rpc.QueryServer = &queryServer{} type queryServer struct { diff --git a/src/query/storage/fetch.go b/src/query/storage/fetch.go index a20d729eb8..36d71482a8 100644 --- a/src/query/storage/fetch.go +++ b/src/query/storage/fetch.go @@ -23,7 +23,6 @@ package storage import ( "time" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/storagemetadata" @@ -41,8 +40,7 @@ func NewFetchOptions() *FetchOptions { FanoutAggregated: FanoutDefault, FanoutAggregatedOptimized: FanoutDefault, }, - Enforcer: cost.NoopChainedEnforcer(), - Scope: tally.NoopScope, + Scope: tally.NoopScope, } } diff --git a/src/query/storage/m3/config.go b/src/query/storage/m3/config.go index e210f98cf2..c87630b36c 100644 --- a/src/query/storage/m3/config.go +++ b/src/query/storage/m3/config.go @@ -35,9 +35,8 @@ import ( ) var ( - errNotAggregatedClusterNamespace = goerrors.New("not an aggregated cluster namespace") - errBothNamespaceTypeNewAndDeprecatedFieldsSet = goerrors.New("cannot specify both deprecated and non-deprecated fields for namespace type") - errNoNamespaceInitializerSet = goerrors.New("no namespace initializer set") + errNotAggregatedClusterNamespace = goerrors.New("not an aggregated cluster namespace") + errNoNamespaceInitializerSet = goerrors.New("no namespace initializer set") ) // ClustersStaticConfiguration is a set of static cluster configurations. @@ -87,31 +86,16 @@ type ClusterStaticNamespaceConfiguration struct { // Downsample is the configuration for downsampling options to use with // the namespace. Downsample *DownsampleClusterStaticNamespaceConfiguration `yaml:"downsample"` - - // StorageMetricsType is the namespace type. - // - // Deprecated: Use "Type" field when specifying config instead, it is - // invalid to use both. - StorageMetricsType storagemetadata.MetricsType `yaml:"storageMetricsType"` } func (c ClusterStaticNamespaceConfiguration) metricsType() (storagemetadata.MetricsType, error) { unset := storagemetadata.MetricsType(0) - if c.Type != unset && c.StorageMetricsType != unset { - // Don't allow both to not be default - return unset, errBothNamespaceTypeNewAndDeprecatedFieldsSet - } if c.Type != unset { // New field value set return c.Type, nil } - if c.StorageMetricsType != unset { - // Deprecated field value set - return c.StorageMetricsType, nil - } - // Both are unset return storagemetadata.DefaultMetricsType, nil } diff --git a/src/query/storage/m3/storage.go b/src/query/storage/m3/storage.go index 3d2cd7954c..d91edb3926 100644 --- a/src/query/storage/m3/storage.go +++ b/src/query/storage/m3/storage.go @@ -30,7 +30,6 @@ import ( "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/storage/index" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/errors" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage" @@ -117,7 +116,6 @@ func (s *m3storage) FetchProm( fetchResult, err := storage.SeriesIteratorsToPromResult( result, s.opts.ReadWorkerPool(), - options.Enforcer, s.opts.TagOptions(), ) @@ -149,11 +147,6 @@ func FetchResultToBlockResult( StepSize: query.Interval, } - enforcer := options.Enforcer - if enforcer == nil { - enforcer = cost.NoopChainedEnforcer() - } - blocks, err := m3db.ConvertM3DBSeriesIterators( result, bounds, diff --git a/src/query/storage/m3/storage_test.go b/src/query/storage/m3/storage_test.go index c519da0152..4d2a9938eb 100644 --- a/src/query/storage/m3/storage_test.go +++ b/src/query/storage/m3/storage_test.go @@ -858,11 +858,7 @@ func TestInvalidBlockTypes(t *testing.T) { require.NoError(t, err) query := &storage.FetchQuery{} - fetchOpts := &storage.FetchOptions{BlockType: models.TypeDecodedBlock} - _, err = s.FetchBlocks(context.TODO(), query, fetchOpts) - assert.Error(t, err) - - fetchOpts.BlockType = models.TypeMultiBlock + fetchOpts := &storage.FetchOptions{BlockType: models.TypeMultiBlock} _, err = s.FetchBlocks(context.TODO(), query, fetchOpts) assert.Error(t, err) } diff --git a/src/query/storage/prom_converter.go b/src/query/storage/prom_converter.go index 4089dd7746..f0b3869773 100644 --- a/src/query/storage/prom_converter.go +++ b/src/query/storage/prom_converter.go @@ -24,11 +24,9 @@ import ( "sync" "github.com/m3db/m3/src/dbnode/encoding" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/consolidators" - xcost "github.com/m3db/m3/src/x/cost" xerrors "github.com/m3db/m3/src/x/errors" xsync "github.com/m3db/m3/src/x/sync" ) @@ -38,7 +36,6 @@ const initRawFetchAllocSize = 32 func iteratorToPromResult( iter encoding.SeriesIterator, tags models.Tags, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (*prompb.TimeSeries, error) { samples := make([]prompb.Sample, 0, initRawFetchAllocSize) @@ -54,11 +51,6 @@ func iteratorToPromResult( return nil, err } - r := enforcer.Add(xcost.Cost(len(samples))) - if r.Error != nil { - return nil, r.Error - } - return &prompb.TimeSeries{ Labels: TagsToPromLabels(tags), Samples: samples, @@ -68,7 +60,6 @@ func iteratorToPromResult( // Fall back to sequential decompression if unable to decompress concurrently. func toPromSequentially( fetchResult consolidators.SeriesFetchResult, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { count := fetchResult.Count() @@ -79,7 +70,7 @@ func toPromSequentially( return PromResult{}, err } - series, err := iteratorToPromResult(iter, tags, enforcer, tagOptions) + series, err := iteratorToPromResult(iter, tags, tagOptions) if err != nil { return PromResult{}, err } @@ -99,7 +90,6 @@ func toPromSequentially( func toPromConcurrently( fetchResult consolidators.SeriesFetchResult, readWorkerPool xsync.PooledWorkerPool, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { count := fetchResult.Count() @@ -121,7 +111,7 @@ func toPromConcurrently( wg.Add(1) readWorkerPool.Go(func() { defer wg.Done() - series, err := iteratorToPromResult(iter, tags, enforcer, tagOptions) + series, err := iteratorToPromResult(iter, tags, tagOptions) if err != nil { mu.Lock() multiErr = multiErr.Add(err) @@ -155,14 +145,13 @@ func toPromConcurrently( func seriesIteratorsToPromResult( fetchResult consolidators.SeriesFetchResult, readWorkerPool xsync.PooledWorkerPool, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { if readWorkerPool == nil { - return toPromSequentially(fetchResult, enforcer, tagOptions) + return toPromSequentially(fetchResult, tagOptions) } - return toPromConcurrently(fetchResult, readWorkerPool, enforcer, tagOptions) + return toPromConcurrently(fetchResult, readWorkerPool, tagOptions) } // SeriesIteratorsToPromResult converts raw series iterators directly to a @@ -170,7 +159,6 @@ func seriesIteratorsToPromResult( func SeriesIteratorsToPromResult( fetchResult consolidators.SeriesFetchResult, readWorkerPool xsync.PooledWorkerPool, - enforcer cost.ChainedEnforcer, tagOptions models.TagOptions, ) (PromResult, error) { defer fetchResult.Close() @@ -178,12 +166,8 @@ func SeriesIteratorsToPromResult( return PromResult{}, err } - if enforcer == nil { - enforcer = cost.NoopChainedEnforcer() - } - promResult, err := seriesIteratorsToPromResult(fetchResult, - readWorkerPool, enforcer, tagOptions) + readWorkerPool, tagOptions) promResult.Metadata = fetchResult.Metadata return promResult, err } diff --git a/src/query/storage/prom_converter_test.go b/src/query/storage/prom_converter_test.go index cb290c966d..0f74a23f11 100644 --- a/src/query/storage/prom_converter_test.go +++ b/src/query/storage/prom_converter_test.go @@ -27,14 +27,12 @@ import ( "github.com/m3db/m3/src/dbnode/encoding" dts "github.com/m3db/m3/src/dbnode/ts" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/consolidators" "github.com/m3db/m3/src/query/test/seriesiter" "github.com/m3db/m3/src/query/ts" "github.com/m3db/m3/src/x/checked" - xcost "github.com/m3db/m3/src/x/cost" "github.com/m3db/m3/src/x/ident" "github.com/m3db/m3/src/x/pool" xsync "github.com/m3db/m3/src/x/sync" @@ -76,15 +74,13 @@ func verifyExpandPromSeries( ) { iters := seriesiter.NewMockSeriesIters(ctrl, ident.Tag{}, num, 2) fetchResult := fr(t, iters, makeTag("foo", "bar", num)...) - enforcer := cost.NewMockChainedEnforcer(ctrl) - enforcer.EXPECT().Add(xcost.Cost(2)).Times(num) fetchResult.Metadata = block.ResultMetadata{ Exhaustive: ex, LocalOnly: true, Warnings: []block.Warning{block.Warning{Name: "foo", Message: "bar"}}, } - results, err := SeriesIteratorsToPromResult(fetchResult, pools, enforcer, nil) + results, err := SeriesIteratorsToPromResult(fetchResult, pools, nil) assert.NoError(t, err) require.NotNil(t, results) @@ -306,7 +302,7 @@ func TestDecodeIteratorsWithEmptySeries(t *testing.T) { } opts := models.NewTagOptions() - res, err := SeriesIteratorsToPromResult(buildIters(), nil, nil, opts) + res, err := SeriesIteratorsToPromResult(buildIters(), nil, opts) require.NoError(t, err) verifyResult(t, res) @@ -314,7 +310,7 @@ func TestDecodeIteratorsWithEmptySeries(t *testing.T) { require.NoError(t, err) pool.Init() - res, err = SeriesIteratorsToPromResult(buildIters(), pool, nil, opts) + res, err = SeriesIteratorsToPromResult(buildIters(), pool, opts) require.NoError(t, err) verifyResult(t, res) } diff --git a/src/query/storage/types.go b/src/query/storage/types.go index 9321b65fe5..a5157155e3 100644 --- a/src/query/storage/types.go +++ b/src/query/storage/types.go @@ -28,7 +28,6 @@ import ( "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/query/block" - "github.com/m3db/m3/src/query/cost" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage/m3/consolidators" @@ -130,9 +129,6 @@ type FetchOptions struct { Step time.Duration // LookbackDuration if set overrides the default lookback duration. LookbackDuration *time.Duration - // Enforcer is used to enforce resource limits on the number of datapoints - // used by a given query. Limits are imposed at time of decompression. - Enforcer cost.ChainedEnforcer // Scope is used to report metrics about the fetch. Scope tally.Scope // Timeout is the timeout for the request. diff --git a/src/x/config/config.go b/src/x/config/config.go index bbd2f54b6b..5f938c8384 100644 --- a/src/x/config/config.go +++ b/src/x/config/config.go @@ -118,7 +118,8 @@ func deprecationCheck(cfg interface{}, df []string) []string { } name := reflect.TypeOf(cfg).Field(i).Name if strings.HasPrefix(name, deprecatedPrefix) && !v.IsZero() { - // Only log as deprecated if actually using (not zero value). + // Exclude unset deprecated config values from + // raising warnings via checking IsZero. df = append(df, name) } } diff --git a/src/x/config/config_test.go b/src/x/config/config_test.go index 195ea505de..582aace5f8 100644 --- a/src/x/config/config_test.go +++ b/src/x/config/config_test.go @@ -492,6 +492,43 @@ baz: null fmt.Sprintf("expect %#v should be equal actual %#v", expect, actual)) }) + t.Run("DeprecatedNilValue", func(t *testing.T) { + // OK + var cfg configuration + fname := writeFile(t, goodConfig) + defer func() { + require.NoError(t, os.Remove(fname)) + }() + + err := LoadFile(&cfg, fname, Options{}) + require.NoError(t, err) + + df := []string{} + ss := deprecationCheck(cfg, df) + require.Equal(t, 0, len(ss)) + + // Deprecated nil/unset value should be ok and not printed + validConfig := ` +listen_address: localhost:4385 +buffer_space: 1024 +servers: + - server1:8090 + - server2:8010 +` + var cfg2 configurationDeprecated + fname2 := writeFile(t, validConfig) + defer func() { + require.NoError(t, os.Remove(fname2)) + }() + + err = LoadFile(&cfg2, fname2, Options{}) + require.NoError(t, err) + + actual := deprecationCheck(cfg2, df) + require.Equal(t, 0, len(actual), + fmt.Sprintf("expect %#v should be equal actual %#v", 0, actual)) + }) + t.Run("NestedConfig", func(t *testing.T) { // Single Deprecation var cfg nestedConfigurationDeprecated diff --git a/src/x/config/listenaddress/listenaddress.go b/src/x/config/listenaddress/listenaddress.go deleted file mode 100644 index 1f0d6354f5..0000000000 --- a/src/x/config/listenaddress/listenaddress.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package listenaddress provides a configuration struct for resolving -// a listen address from YAML. -// -// Deprecation notice: -// The environment resolution behavior of this -// class has largely been superseded -// by config environment interpolation in go.uber.org/config (see config/README.md for -// details). Instead of: -// -// listenAddress: -// type: environment -// envVarListenPort: MY_PORT_VAR -// -// one can do: -// -// listenAddress: -// value: 0.0.0.0:${MY_PORT_VAR} -package listenaddress - -import ( - "fmt" - "os" - "strconv" -) - -const ( - defaultHostname = "0.0.0.0" -) - -// Resolver is a type of port resolver -type Resolver string - -const ( - // ConfigResolver resolves port using a value provided in config - ConfigResolver Resolver = "config" - // EnvironmentResolver resolves port using an environment variable - // of which the name is provided in config - EnvironmentResolver Resolver = "environment" -) - -// Configuration is the configuration for resolving a listen address. -type Configuration struct { - // DeprecatedListenAddressType is the port type for the port - // DEPRECATED: use config interpolation with `value` (config/README.md) - DeprecatedListenAddressType Resolver `yaml:"type"` - - // Value is the config specified listen address if using config port type. - Value *string `yaml:"value"` - - // EnvVarListenPort specifies the environment variable name for the listen address port. - // DEPRECATED: use config interpolation with `value` (config/README.md) - DeprecatedEnvVarListenPort *string `yaml:"envVarListenPort"` - - // DeprecatedEnvVarListenHost specifies the environment variable name for the listen address hostname. - // DEPRECATED: use config interpolation with `value` (config/README.md) - DeprecatedEnvVarListenHost *string `yaml:"envVarListenHost"` -} - -// Resolve returns the resolved listen address given the configuration. -func (c Configuration) Resolve() (string, error) { - listenAddrType := c.DeprecatedListenAddressType - - if listenAddrType == "" { - // Default to ConfigResolver - listenAddrType = ConfigResolver - } - - var listenAddress string - switch listenAddrType { - case ConfigResolver: - if c.Value == nil { - err := fmt.Errorf("missing listen address value using: resolver=%s", - string(listenAddrType)) - return "", err - } - listenAddress = *c.Value - - case EnvironmentResolver: - // environment variable for port is required - if c.DeprecatedEnvVarListenPort == nil { - err := fmt.Errorf("missing port env var name using: resolver=%s", - string(listenAddrType)) - return "", err - } - portStr := os.Getenv(*c.DeprecatedEnvVarListenPort) - port, err := strconv.Atoi(portStr) - if err != nil { - err := fmt.Errorf("invalid port env var value using: resolver=%s, name=%s", - string(listenAddrType), *c.DeprecatedEnvVarListenPort) - return "", err - } - // if environment variable for hostname is not set, use the default - if c.DeprecatedEnvVarListenHost == nil { - listenAddress = fmt.Sprintf("%s:%d", defaultHostname, port) - } else { - envHost := os.Getenv(*c.DeprecatedEnvVarListenHost) - listenAddress = fmt.Sprintf("%s:%d", envHost, port) - } - - default: - return "", fmt.Errorf("unknown listen address type: resolver=%s", - string(listenAddrType)) - } - - return listenAddress, nil -} diff --git a/src/x/config/listenaddress/listenaddress_test.go b/src/x/config/listenaddress/listenaddress_test.go deleted file mode 100644 index 7d6ede6b9b..0000000000 --- a/src/x/config/listenaddress/listenaddress_test.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package listenaddress provides a configuration struct for resolving -// a listen address from YAML. -package listenaddress - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var ( - envListenPort = "ENV_LISTEN_PORT" - envListenHost = "ENV_LISTEN_HOST" - defaultListen = "0.0.0.0:9000" -) - -func TestListenAddressResolver(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: ConfigResolver, - Value: &defaultListen, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, defaultListen, value) -} - -func TestListenAddressResolverDefaultsToConfigResolver(t *testing.T) { - cfg := Configuration{ - Value: &defaultListen, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, defaultListen, value) -} - -func TestConfigResolverErrorWhenMissing(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: ConfigResolver, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestEnvironmentVariableResolverWithDefault(t *testing.T) { - envPort := "9000" - - require.NoError(t, os.Setenv(envListenPort, envPort)) - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &envListenPort, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, defaultListen, value) -} - -func TestEnvironmentVariableResolver(t *testing.T) { - envHost := "127.0.0.1" - envPort := "9000" - - require.NoError(t, os.Setenv(envListenPort, envPort)) - require.NoError(t, os.Setenv(envListenHost, envHost)) - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &envListenPort, - DeprecatedEnvVarListenHost: &envListenHost, - } - - value, err := cfg.Resolve() - require.NoError(t, err) - - assert.Equal(t, "127.0.0.1:9000", value) -} - -func TestInvalidEnvironmentVariableResolver(t *testing.T) { - varName := "BAD_LISTEN_ENV_PORT" - expected := "foo" - - require.NoError(t, os.Setenv(varName, expected)) - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &varName, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestEnvironmentResolverErrorWhenNameMissing(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestEnvironmentResolverErrorWhenValueMissing(t *testing.T) { - varName := "OTHER_LISTEN_ENV_PORT" - - cfg := Configuration{ - DeprecatedListenAddressType: EnvironmentResolver, - DeprecatedEnvVarListenPort: &varName, - } - - _, err := cfg.Resolve() - require.Error(t, err) -} - -func TestUnknownResolverError(t *testing.T) { - cfg := Configuration{ - DeprecatedListenAddressType: "some-unknown-resolver", - } - - _, err := cfg.Resolve() - require.Error(t, err) -} diff --git a/src/x/cost/enforcer.go b/src/x/cost/enforcer.go deleted file mode 100644 index 5667735632..0000000000 --- a/src/x/cost/enforcer.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" -) - -const ( - defaultCostExceededErrorFmt = "limit reached (current = %v, limit = %v)" -) - -var ( - noopManager = NewStaticLimitManager( - NewLimitManagerOptions(). - SetDefaultLimit(Limit{ - Threshold: maxCost, - Enabled: false, - }, - ), - ) - noopEnforcer = NewEnforcer(noopManager, NewNoopTracker(), nil) -) - -// Report is a report on the cost limits of an enforcer. -type Report struct { - Cost - Error error -} - -// enforcer enforces cost limits for operations. -type enforcer struct { - LimitManager - tracker Tracker - - costMsg string - metrics EnforcerReporter -} - -// NewEnforcer returns a new enforcer for cost limits. -func NewEnforcer(m LimitManager, t Tracker, opts EnforcerOptions) Enforcer { - if opts == nil { - opts = NewEnforcerOptions() - } - - reporter := opts.Reporter() - if reporter == nil { - reporter = NoopEnforcerReporter() - } - - return &enforcer{ - LimitManager: m, - tracker: t, - costMsg: opts.CostExceededMessage(), - metrics: reporter, - } -} - -func (e *enforcer) Reporter() EnforcerReporter { - return e.metrics -} - -// Add adds the cost of an operation to the enforcer's current total. If the operation exceeds -// the enforcer's limit the enforcer will return a CostLimit error in addition to the new total. -func (e *enforcer) Add(cost Cost) Report { - e.metrics.ReportCost(cost) - current := e.tracker.Add(cost) - e.metrics.ReportCurrent(current) - - limit := e.Limit() - overLimit := e.checkLimit(current, limit) - - if overLimit != nil { - // Emit metrics on number of operations that are over the limit even when not enabled. - e.metrics.ReportOverLimit(limit.Enabled) - } - - return Report{ - Cost: current, - Error: overLimit, - } -} - -// State returns the current state of the enforcer. -func (e *enforcer) State() (Report, Limit) { - cost := e.tracker.Current() - l := e.Limit() - err := e.checkLimit(cost, l) - r := Report{ - Cost: cost, - Error: err, - } - return r, l -} - -// Clone clones the current enforcer. The new enforcer uses the same Estimator and LimitManager -// as e buts its Tracker is independent. -func (e *enforcer) Clone() Enforcer { - return &enforcer{ - LimitManager: e.LimitManager, - tracker: NewTracker(), - costMsg: e.costMsg, - metrics: e.metrics, - } -} - -func (e *enforcer) checkLimit(cost Cost, limit Limit) error { - if cost < limit.Threshold || !limit.Enabled { - return nil - } - - return NewCostExceededError(e.costMsg, cost, limit.Threshold) -} - -type costExceededError struct { - Threshold Cost - Current Cost - CustomMsg string -} - -func (ce costExceededError) Error() string { - baseMsg := fmt.Sprintf( - defaultCostExceededErrorFmt, - float64(ce.Current), - float64(ce.Threshold), - ) - if ce.CustomMsg == "" { - return baseMsg - } - - return fmt.Sprintf("%s: %s", ce.CustomMsg, baseMsg) -} - -// NewCostExceededError returns an error for going over an Enforcer's limit. -func NewCostExceededError(customMessage string, cost Cost, threshold Cost) error { - return costExceededError{ - CustomMsg: customMessage, - Current: cost, - Threshold: threshold, - } -} - -// NoopEnforcer returns a new enforcer that always returns a current cost of 0 and -// is always disabled. -func NoopEnforcer() Enforcer { - return noopEnforcer -} - -type noopEnforcerReporter struct{} - -func (noopEnforcerReporter) ReportCost(c Cost) {} -func (noopEnforcerReporter) ReportCurrent(c Cost) {} -func (noopEnforcerReporter) ReportOverLimit(enabled bool) {} - -// NoopEnforcerReporter returns an EnforcerReporter which does nothing on all events. -func NoopEnforcerReporter() EnforcerReporter { - return noopEnforcerReporter{} -} diff --git a/src/x/cost/enforcer_test.go b/src/x/cost/enforcer_test.go deleted file mode 100644 index 27b9a9aa1f..0000000000 --- a/src/x/cost/enforcer_test.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv/mem" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - testThresholdKey = "threshold" - testEnabledKey = "enabled" -) - -func TestEnforcer(t *testing.T) { - tests := []struct { - input Cost - expected Cost - exceededThreshold bool - }{ - { - input: Cost(0), - expected: 0, - exceededThreshold: false, - }, - { - input: Cost(1), - expected: 1, - exceededThreshold: false, - }, - { - input: Cost(3), - expected: 4, - exceededThreshold: false, - }, - { - input: Cost(9), - expected: 13, - exceededThreshold: true, - }, - } - - limit := Limit{ - Threshold: 10, - Enabled: true, - } - mOpts := NewLimitManagerOptions().SetDefaultLimit(limit) - store := mem.NewStore() - msg := "message which contains context on the cost limit" - - m, err := NewDynamicLimitManager(store, testThresholdKey, testEnabledKey, mOpts) - require.NoError(t, err) - - opts := NewEnforcerOptions().SetCostExceededMessage(msg) - e := NewEnforcer(m, NewTracker(), opts) - - for _, test := range tests { - t.Run(fmt.Sprintf("input %v", test.input), func(t *testing.T) { - report := e.Add(test.input) - require.Equal(t, test.expected, report.Cost) - - if test.exceededThreshold { - require.EqualError(t, report.Error, NewCostExceededError(msg, 13, 10).Error()) - } else { - require.NoError(t, report.Error) - } - }) - } - - // State should return the updated cost total. - report, limit := e.State() - require.Equal(t, Cost(13), report.Cost) - require.Equal(t, Cost(10), limit.Threshold) - require.True(t, limit.Enabled) - require.EqualError(t, report.Error, NewCostExceededError(msg, 13, 10).Error()) - - // The error message should start with the message provided in the options. - require.True(t, strings.HasPrefix(report.Error.Error(), msg)) - - // When the threshold is raised, any new operations that stay below it should be legal again. - store.Set(testThresholdKey, &commonpb.Float64Proto{Value: float64(20)}) - for { - if l := e.Limit(); l.Threshold == 20 { - break - } - time.Sleep(100 * time.Millisecond) - } - - report = e.Add(Cost(3)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(16), report.Cost) - - report = e.Add(Cost(5)) - require.NoError(t, err) - require.NoError(t, err) - require.EqualError(t, report.Error, NewCostExceededError(msg, 21, 20).Error()) - require.Equal(t, Cost(21), report.Cost) - - // When the enforcer is disabled any input above the threshold should become legal. - store.Set(testEnabledKey, &commonpb.BoolProto{Value: false}) - for { - if l := e.Limit(); !l.Enabled { - break - } - time.Sleep(100 * time.Millisecond) - } - - report = e.Add(Cost(2)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(23), report.Cost) - - // State should return the updated state. - report, limit = e.State() - require.Equal(t, Cost(23), report.Cost) - require.Equal(t, Cost(20), limit.Threshold) - require.False(t, limit.Enabled) - require.NoError(t, report.Error) -} - -func TestNewCostExceedError(t *testing.T) { - t.Run("with custom error message", func(t *testing.T) { - assert.EqualError( - t, - NewCostExceededError("custom", 1, 2), - "custom: limit reached (current = 1, limit = 2)") - }) - - t.Run("with default message", func(t *testing.T) { - assert.EqualError( - t, - NewCostExceededError("", 1, 2), - "limit reached (current = 1, limit = 2)") - }) -} - -func TestEnforcerClone(t *testing.T) { - store := mem.NewStore() - threshold := Cost(30) - limit := Limit{ - Threshold: threshold, - Enabled: true, - } - mOpts := NewLimitManagerOptions(). - SetDefaultLimit(limit) - - m, err := NewDynamicLimitManager(store, testThresholdKey, testEnabledKey, mOpts) - require.NoError(t, err) - - e := NewEnforcer(m, NewTracker(), nil) - - report := e.Add(Cost(10)) - require.Equal(t, Cost(10), report.Cost) - require.NoError(t, report.Error) - - clone := e.Clone() - - // The cloned enforcer should have no initial cost. - report, limit = clone.State() - require.Equal(t, Cost(0), report.Cost) - require.NoError(t, report.Error) - require.Equal(t, threshold, limit.Threshold) - require.True(t, limit.Enabled) - - // Subsequent calls to Add on each enforcer should be independent. - report = e.Add(Cost(10)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(20), report.Cost) - - report = clone.Add(Cost(5)) - require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, Cost(5), report.Cost) - - // Each enforcer should see the same updates to their state. - var newThreshold Cost = 40 - store.Set(testThresholdKey, &commonpb.Float64Proto{Value: float64(newThreshold)}) - store.Set(testEnabledKey, &commonpb.BoolProto{Value: false}) - - for { - if l := e.Limit(); !l.Enabled { - break - } - time.Sleep(100 * time.Millisecond) - } - - limit = e.Limit() - require.Equal(t, false, limit.Enabled) - require.Equal(t, newThreshold, limit.Threshold) - - limit = clone.Limit() - require.Equal(t, false, limit.Enabled) - require.Equal(t, newThreshold, limit.Threshold) -} - -func TestNoopEnforcer(t *testing.T) { - tests := []struct { - input Cost - }{ - { - input: Cost(0), - }, - { - input: Cost(10), - }, - } - - e := NoopEnforcer() - limit := e.Limit() - assert.Equal(t, maxCost, limit.Threshold) - assert.False(t, limit.Enabled) - - for _, test := range tests { - t.Run(fmt.Sprintf("input %v", test.input), func(t *testing.T) { - report := e.Add(test.input) - assert.Equal(t, Cost(0), report.Cost) - assert.NoError(t, report.Error) - }) - } -} diff --git a/src/x/cost/limit_manager.go b/src/x/cost/limit_manager.go deleted file mode 100644 index 4faa188fdd..0000000000 --- a/src/x/cost/limit_manager.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "sync" - "time" - - "github.com/m3db/m3/src/cluster/kv" - "github.com/m3db/m3/src/cluster/kv/util" - - "github.com/uber-go/tally" - "go.uber.org/atomic" -) - -type limitManager struct { - sync.RWMutex - - threshold *atomic.Float64 - enabled *atomic.Bool - - thresholdWatcher kv.ValueWatch - enabledWatcher kv.ValueWatch - closed bool - closeCh chan struct{} - reportInterval time.Duration - metrics limitManagerMetrics -} - -// NewDynamicLimitManager returns a new LimitWatcher which watches for updates to the cost limit -// of an operation in KV. -func NewDynamicLimitManager( - store kv.Store, - kvLimitKey, kvEnabledKey string, - opts LimitManagerOptions, -) (LimitManager, error) { - if opts == nil { - opts = NewLimitManagerOptions() - } - iOpts := opts.InstrumentOptions() - - var ( - limit = opts.DefaultLimit() - defaultThreshold = float64(limit.Threshold) - defaultEnabled = limit.Enabled - m = &limitManager{ - threshold: atomic.NewFloat64(defaultThreshold), - enabled: atomic.NewBool(defaultEnabled), - closeCh: make(chan struct{}), - reportInterval: iOpts.ReportInterval(), - metrics: newLimitManagerMetrics(iOpts.MetricsScope()), - } - ) - - watchOpts := util.NewOptions().SetLogger(iOpts.Logger()) - thresholdWatcher, err := util.WatchAndUpdateAtomicFloat64( - store, - kvLimitKey, - m.threshold, - defaultThreshold, - watchOpts.SetValidateFn(opts.ValidateLimitFn()), - ) - if err != nil { - return nil, fmt.Errorf("unable to watch key '%s': %v", kvLimitKey, err) - } - m.thresholdWatcher = thresholdWatcher - - enabledWatcher, err := util.WatchAndUpdateAtomicBool( - store, - kvEnabledKey, - m.enabled, - defaultEnabled, - watchOpts, - ) - if err != nil { - return nil, fmt.Errorf("unable to watch key '%s': %v", kvEnabledKey, err) - } - m.enabledWatcher = enabledWatcher - - return m, nil -} - -func (m *limitManager) Limit() Limit { - return Limit{ - Threshold: Cost(m.threshold.Load()), - Enabled: m.enabled.Load(), - } -} - -func (m *limitManager) Close() { - m.Lock() - defer m.Unlock() - if m.closed { - return - } - if m.thresholdWatcher != nil { - m.thresholdWatcher.Close() - } - if m.enabledWatcher != nil { - m.enabledWatcher.Close() - } - close(m.closeCh) - m.closed = true -} - -func (m *limitManager) Report() { - ticker := time.NewTicker(m.reportInterval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - limit := m.Limit() - m.metrics.threshold.Update(float64(limit.Threshold)) - - var v float64 - if limit.Enabled { - v = 1 - } - m.metrics.enabled.Update(v) - - case <-m.closeCh: - return - } - } -} - -// NewStaticLimitManager returns a new LimitManager which always returns the same limit. -func NewStaticLimitManager(opts LimitManagerOptions) LimitManager { - if opts == nil { - opts = NewLimitManagerOptions() - } - iOpts := opts.InstrumentOptions() - - var ( - l = opts.DefaultLimit() - m = &limitManager{ - threshold: atomic.NewFloat64(float64(l.Threshold)), - enabled: atomic.NewBool(l.Enabled), - closeCh: make(chan struct{}), - reportInterval: iOpts.ReportInterval(), - metrics: newLimitManagerMetrics(iOpts.MetricsScope()), - } - ) - return m -} - -type limitManagerMetrics struct { - threshold tally.Gauge - enabled tally.Gauge -} - -func newLimitManagerMetrics(s tally.Scope) limitManagerMetrics { - return limitManagerMetrics{ - threshold: s.Gauge("threshold"), - enabled: s.Gauge("enabled"), - } -} diff --git a/src/x/cost/limit_manager_test.go b/src/x/cost/limit_manager_test.go deleted file mode 100644 index b8c5021cf1..0000000000 --- a/src/x/cost/limit_manager_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "errors" - "testing" - "time" - - "github.com/m3db/m3/src/cluster/generated/proto/commonpb" - "github.com/m3db/m3/src/cluster/kv/mem" - - "github.com/fortytw2/leaktest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDynamicLimitManager(t *testing.T) { - threshold := Cost(100) - enabled := true - store := mem.NewStore() - thresholdKey := "threshold" - enabledKey := "enabled" - - validateFn := func(data interface{}) error { - val := data.(float64) - if val < 50 { - return errors.New("limit cannot be below 50") - } - return nil - } - - limit := Limit{ - Threshold: threshold, - Enabled: enabled, - } - opts := NewLimitManagerOptions(). - SetDefaultLimit(limit). - SetValidateLimitFn(validateFn) - - m, err := NewDynamicLimitManager( - store, thresholdKey, enabledKey, opts, - ) - require.NoError(t, err) - - testLimitManager(t, m, threshold, enabled) - - // Test updates to the threshold. - threshold = 200 - store.Set(thresholdKey, &commonpb.Float64Proto{Value: float64(threshold)}) - - for { - if m.Limit().Threshold == threshold { - break - } - time.Sleep(100 * time.Millisecond) - } - - testLimitManager(t, m, threshold, enabled) - - // Test updates to enabled. - enabled = false - store.Set(enabledKey, &commonpb.BoolProto{Value: enabled}) - - for { - if !m.Limit().Enabled { - break - } - time.Sleep(100 * time.Millisecond) - } - - testLimitManager(t, m, threshold, enabled) - - // Test that invalid updates are ignored. - store.Set(thresholdKey, &commonpb.Float64Proto{Value: 25}) - time.Sleep(100 * time.Millisecond) - - testLimitManager(t, m, threshold, enabled) - - // Ensure we do not leak any goroutines. - m.Close() - leaktest.Check(t) -} - -func TestStaticLimitManager(t *testing.T) { - threshold := Cost(100) - enabled := true - limit := Limit{ - Threshold: threshold, - Enabled: enabled, - } - manager := NewStaticLimitManager(NewLimitManagerOptions().SetDefaultLimit(limit)) - - limit = manager.Limit() - assert.Equal(t, threshold, limit.Threshold) - assert.Equal(t, enabled, limit.Enabled) - - // Ensure we do not leak any goroutines. - manager.Close() - leaktest.Check(t) -} - -func testLimitManager( - t *testing.T, m LimitManager, expectedThreshold Cost, expectedEnabled bool, -) { - l := m.Limit() - assert.Equal(t, expectedThreshold, l.Threshold) - assert.Equal(t, expectedEnabled, l.Enabled) -} diff --git a/src/x/cost/options.go b/src/x/cost/options.go deleted file mode 100644 index 9378cf8ab7..0000000000 --- a/src/x/cost/options.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "github.com/m3db/m3/src/cluster/kv/util" - "github.com/m3db/m3/src/x/instrument" - - "github.com/uber-go/tally" -) - -// LimitManagerOptions provides a set of options for a LimitManager. -type LimitManagerOptions interface { - // SetDefaultLimit sets the default limit for the manager. - SetDefaultLimit(val Limit) LimitManagerOptions - - // DefaultLimit returns the default limit for the manager. - DefaultLimit() Limit - - // SetValidateLimitFn sets the validation function applied to updates to the cost limit. - SetValidateLimitFn(val util.ValidateFn) LimitManagerOptions - - // ValidateLimitFn returns the validation function applied to updates to the cost limit. - ValidateLimitFn() util.ValidateFn - - // SetInstrumentOptions sets the instrument options. - SetInstrumentOptions(val instrument.Options) LimitManagerOptions - - // InstrumentOptions returns the instrument options. - InstrumentOptions() instrument.Options -} - -type limitManagerOptions struct { - defaultLimit Limit - validateLimitFn util.ValidateFn - iOpts instrument.Options -} - -// NewLimitManagerOptions returns a new set of LimitManager options. -func NewLimitManagerOptions() LimitManagerOptions { - return &limitManagerOptions{ - defaultLimit: Limit{ - Threshold: maxCost, - Enabled: false, - }, - iOpts: instrument.NewOptions(), - } -} - -func (o *limitManagerOptions) SetDefaultLimit(val Limit) LimitManagerOptions { - opts := *o - opts.defaultLimit = val - return &opts -} - -func (o *limitManagerOptions) DefaultLimit() Limit { - return o.defaultLimit -} - -func (o *limitManagerOptions) SetValidateLimitFn(val util.ValidateFn) LimitManagerOptions { - opts := *o - opts.validateLimitFn = val - return &opts -} - -func (o *limitManagerOptions) ValidateLimitFn() util.ValidateFn { - return o.validateLimitFn -} - -func (o *limitManagerOptions) SetInstrumentOptions(val instrument.Options) LimitManagerOptions { - opts := *o - opts.iOpts = val - return &opts -} - -func (o *limitManagerOptions) InstrumentOptions() instrument.Options { - return o.iOpts -} - -// EnforcerOptions provides a set of options for an enforcer. -type EnforcerOptions interface { - // Reporter is the reporter which will be used on Enforcer events. - Reporter() EnforcerReporter - - // SetReporter sets Reporter() - SetReporter(r EnforcerReporter) EnforcerOptions - - // SetCostExceededMessage sets CostExceededMessage - SetCostExceededMessage(val string) EnforcerOptions - - // CostExceededMessage returns the message appended to cost limit errors to provide - // more context on the cost limit that was exceeded. - CostExceededMessage() string -} - -type enforcerOptions struct { - msg string - buckets tally.ValueBuckets - reporter EnforcerReporter - iOpts instrument.Options -} - -// NewEnforcerOptions returns a new set of enforcer options. -func NewEnforcerOptions() EnforcerOptions { - return &enforcerOptions{ - buckets: tally.MustMakeExponentialValueBuckets(1, 2, 20), - iOpts: instrument.NewOptions(), - } -} - -func (o *enforcerOptions) Reporter() EnforcerReporter { - return o.reporter -} - -func (o *enforcerOptions) SetReporter(r EnforcerReporter) EnforcerOptions { - opts := *o - opts.reporter = r - return &opts -} - -func (o *enforcerOptions) SetCostExceededMessage(val string) EnforcerOptions { - opts := *o - opts.msg = val - return &opts -} - -func (o *enforcerOptions) CostExceededMessage() string { - return o.msg -} - -func (o *enforcerOptions) SetValueBuckets(val tally.ValueBuckets) EnforcerOptions { - opts := *o - opts.buckets = val - return &opts -} - -func (o *enforcerOptions) ValueBuckets() tally.ValueBuckets { - return o.buckets -} - -func (o *enforcerOptions) SetInstrumentOptions(val instrument.Options) EnforcerOptions { - opts := *o - opts.iOpts = val - return &opts -} - -func (o *enforcerOptions) InstrumentOptions() instrument.Options { - return o.iOpts -} diff --git a/src/x/cost/test/assert.go b/src/x/cost/test/assert.go deleted file mode 100644 index b7c4ac40e3..0000000000 --- a/src/x/cost/test/assert.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package test contains testing utilities for the cost package. -package test - -import ( - "github.com/m3db/m3/src/x/cost" - - "github.com/stretchr/testify/assert" -) - -// AssertCurrentCost is a helper assertion to check that an enforcer has the -// given cost. -func AssertCurrentCost(t assert.TestingT, expectedCost cost.Cost, ef cost.Enforcer) { - actual, _ := ef.State() - assert.Equal(t, expectedCost, actual.Cost) -} - -// AssertLimitError checks that err is a limit error with the given parameters -func AssertLimitError(t assert.TestingT, err error, current, threshold cost.Cost) { - AssertLimitErrorWithMsg(t, err, "", current, threshold) -} - -// AssertLimitErrorWithMsg checks that err is a limit error with the given parameters -func AssertLimitErrorWithMsg(t assert.TestingT, err error, msg string, current, threshold cost.Cost) { - assert.EqualError(t, err, cost.NewCostExceededError(msg, current, threshold).Error()) -} diff --git a/src/x/cost/tracker.go b/src/x/cost/tracker.go deleted file mode 100644 index 9599cdc53f..0000000000 --- a/src/x/cost/tracker.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import "go.uber.org/atomic" - -type tracker struct { - total *atomic.Float64 -} - -// NewTracker returns a new Tracker which maintains a simple running total of all the -// costs it has seen so far. -func NewTracker() Tracker { return tracker{total: atomic.NewFloat64(0)} } -func (t tracker) Add(c Cost) Cost { return Cost(t.total.Add(float64(c))) } -func (t tracker) Current() Cost { return Cost(t.total.Load()) } - -type noopTracker struct{} - -// NewNoopTracker returns a tracker which always always returns a cost of 0. -func NewNoopTracker() Tracker { return noopTracker{} } -func (t noopTracker) Add(c Cost) Cost { return 0 } -func (t noopTracker) Current() Cost { return 0 } diff --git a/src/x/cost/tracker_test.go b/src/x/cost/tracker_test.go deleted file mode 100644 index 37c0c85216..0000000000 --- a/src/x/cost/tracker_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cost - -import ( - "fmt" - "sync" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestTracker(t *testing.T) { - tests := []struct { - input Cost - expected Cost - }{ - { - input: 10, - expected: 10, - }, - { - input: 5, - expected: 15, - }, - { - input: 0, - expected: 15, - }, - { - input: 20, - expected: 35, - }, - } - - tracker := NewTracker() - for _, test := range tests { - t.Run(fmt.Sprintf("testing input: %v", test.input), func(t *testing.T) { - require.Equal(t, test.expected, tracker.Add(test.input)) - require.Equal(t, test.expected, tracker.Current()) - }) - } -} - -func TestTrackerConcurrentUpdates(t *testing.T) { - wg := sync.WaitGroup{} - tracker := NewTracker() - numGoroutines := 10 - numIterations := 1000 - - wg.Add(10) - - for i := 0; i < numGoroutines; i++ { - go func() { - for j := 0; j < numIterations; j++ { - tracker.Add(1) - } - wg.Done() - }() - } - - wg.Wait() - - actual := tracker.Current() - require.Equal(t, Cost(numGoroutines*numIterations), actual) -} - -func TestNoopTracker(t *testing.T) { - tests := []struct { - input Cost - }{ - { - input: 0, - }, - { - input: 1, - }, - { - input: 5, - }, - { - input: 20, - }, - } - - tracker := NewNoopTracker() - for _, test := range tests { - t.Run(fmt.Sprintf("input %v", test.input), func(t *testing.T) { - require.Equal(t, Cost(0), tracker.Add(test.input)) - require.Equal(t, Cost(0), tracker.Current()) - }) - } -} diff --git a/src/x/cost/types.go b/src/x/cost/types.go deleted file mode 100644 index 0e103d4210..0000000000 --- a/src/x/cost/types.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package cost is a library providing utilities for estimating the cost of operations -// and enforcing limits on them. The primary type of interest is Enforcer, which tracks the cost of operations, -// and errors when that cost goes over a particular limit. -package cost - -import ( - "math" -) - -// Cost represents the cost of an operation. -type Cost float64 - -const ( - // maxCost is the maximum cost of an operation. - maxCost = Cost(math.MaxFloat64) -) - -// Limit encapulates the configuration of a cost limit for an operation. -type Limit struct { - Threshold Cost - Enabled bool -} - -// LimitManager manages configuration of a cost limit for an operation. -type LimitManager interface { - // Limit returns the current cost limit for an operation. - Limit() Limit - - // Report reports metrics on the state of the manager. - Report() - - // Close closes the manager. - Close() -} - -// Tracker tracks the cost of operations seen so far. -type Tracker interface { - // Add adds c to the tracker's current cost total and returns the new total. - Add(c Cost) Cost - - // Current returns the tracker's current cost total. - Current() Cost -} - -// Enforcer instances enforce cost limits for operations. -type Enforcer interface { - Add(op Cost) Report - State() (Report, Limit) - Limit() Limit - Clone() Enforcer - Reporter() EnforcerReporter -} - -// An EnforcerReporter is a listener for Enforcer events. -type EnforcerReporter interface { - // ReportCost is called on every call to Enforcer#Add with the added cost - ReportCost(c Cost) - - // ReportCurrent reports the current total on every call to Enforcer#Add - ReportCurrent(c Cost) - - // ReportOverLimit is called every time an enforcer goes over its limit. enabled is true if the limit manager - // says the limit is currently enabled. - ReportOverLimit(enabled bool) -}