WIP [dbnode] Refactor posting list operations to be lazily evaluated and zero copy

.buildkite/pipeline.yml

@@ -42,7 +42,7 @@ steps:
         run: app
         workdir: /go/src/github.com/m3db/m3
     <<: *common
-  - name: "Integration (:docker:)"
+  - name: "Integration (:docker:, postings=default) %n"
     command: make clean install-vendor-m3 docker-integration-test
     parallelism: 2
     env:
@@ -52,6 +52,16 @@ steps:
         gopath-checkout#v1.0.1:
           import: github.com/m3db/m3
     <<: *common
+  - name: "Integration (:docker:, postings=read_only)  %n"
+    command: M3DB_READ_ONLY_POSTINGS=true make clean install-vendor-m3 docker-integration-test
+    parallelism: 2
+    env:
+      CGO_ENABLED: 0
+      GIMME_GO_VERSION: 1.13.x
+    plugins:
+        gopath-checkout#v1.0.1:
+          import: github.com/m3db/m3
+    <<: *common
   - name: "Prometheus compatibility (:docker:)"
     command: make clean install-vendor-m3 docker-compatibility-test
     parallelism: 1
@@ -62,17 +72,17 @@ steps:
         gopath-checkout#v1.0.1:
           import: github.com/m3db/m3
     <<: *common
-  - name: "Integration (dbnode Recently Read) %n"
+  - name: "Integration (dbnode cache=LRU postings=default) %n"
     parallelism: 2
-    command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read
+    command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
     plugins:
       docker-compose#v2.5.1:
         run: app
         workdir: /go/src/github.com/m3db/m3
     <<: *common
-  - name: "Integration (dbnode LRU) %n"
+  - name: "Integration (dbnode cache=LRU postings=read_only) %n"
     parallelism: 2
-    command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
+    command: M3DB_READ_ONLY_POSTINGS=true make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
     plugins:
       docker-compose#v2.5.1:
         run: app

scripts/docker-integration-tests/aggregator/docker-compose.yml

@@ -12,6 +12,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   m3coordinator01:
     expose:
       - "7202"

scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml

@@ -12,6 +12,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   m3coordinator01:
     expose:
       - "7202"

scripts/docker-integration-tests/carbon/docker-compose.yml

@@ -10,6 +10,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator01:
     expose:
       - "7201"

scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml

@@ -10,6 +10,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator01:
     expose:
       - "7201"

scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml

@@ -10,6 +10,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator01:
     expose:
       - "7201"

scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml

@@ -12,6 +12,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=dbnode01
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
   etcd01:

scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml

@@ -12,6 +12,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_a_m3db_local_1
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_a_dbnode02:
@@ -26,6 +27,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_a_m3db_local_2
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_a_coordinator01:
@@ -54,6 +56,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_b_m3db_local_1
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_b_dbnode02:
@@ -68,6 +71,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_b_m3db_local_2
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_b_coordinator01:

scripts/docker-integration-tests/prometheus/docker-compose.yml

@@ -10,6 +10,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator01:
     expose:
       - "7201"

scripts/docker-integration-tests/prometheus_replication/docker-compose.yml

@@ -10,6 +10,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator01:
     expose:
       - "7201"
@@ -32,6 +34,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator02:
     expose:
       - "17201"

scripts/docker-integration-tests/query_fanout/docker-compose.yml

@@ -10,6 +10,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator-cluster-a:
     expose:
       - "7201"
@@ -34,6 +36,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator-cluster-b:
     expose:
       - "7201"
@@ -58,6 +62,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator-cluster-c:
     expose:
       - "7201"

scripts/docker-integration-tests/repair/docker-compose.yml

@@ -12,6 +12,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=m3db_local_1
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
   dbnode02:
@@ -26,6 +27,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=m3db_local_2
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml"
   coordinator01:

scripts/docker-integration-tests/repair_and_replication/docker-compose.yml

@@ -12,6 +12,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_a_m3db_local_1
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_a_dbnode02:
@@ -26,6 +27,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_a_m3db_local_2
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_a_coordinator01:
@@ -54,6 +56,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_b_m3db_local_1
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_b_dbnode02:
@@ -68,6 +71,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_b_m3db_local_2
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_b_coordinator01:

scripts/docker-integration-tests/replication/docker-compose.yml

@@ -12,6 +12,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_a_m3db_local_1
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_a_dbnode02:
@@ -26,6 +27,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_a_m3db_local_2
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_a_coordinator01:
@@ -54,6 +56,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_b_m3db_local_1
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_b_dbnode02:
@@ -68,6 +71,7 @@ services:
     image: "m3dbnode_integration:${REVISION}"
     environment:
       - M3DB_HOST_ID=cluster_b_m3db_local_2
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
     volumes:
       - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
   cluster_b_coordinator01:

scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml

@@ -10,6 +10,8 @@ services:
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_READ_ONLY_POSTINGS # Remove after MigrationReadOnlyPostings done
   coordinator01:
     expose:
       - "7201"

src/cmd/services/m3coordinator/ingest/carbon/ingest.go

@@ -126,15 +126,21 @@ func NewIngester(
 		}
 	})
 
+	scope := opts.InstrumentOptions.MetricsScope()
+	metrics, err := newCarbonIngesterMetrics(scope)
+	if err != nil {
+		return nil, err
+	}
+
 	ingester := &ingester{
 		downsamplerAndWriter: downsamplerAndWriter,
 		opts:                 opts,
 		logger:               opts.InstrumentOptions.Logger(),
 		tagOpts:              tagOpts,
-		metrics: newCarbonIngesterMetrics(
-			opts.InstrumentOptions.MetricsScope()),
-		lineResourcesPool: resourcePool,
+		metrics:              metrics,
+		lineResourcesPool:    resourcePool,
 	}
+
 	// No need to retain watch as NamespaceWatcher.Close() will handle closing any watches
 	// generated by creating listeners.
 	clusterNamespacesWatcher.RegisterListener(ingester)
@@ -170,7 +176,7 @@ func (i *ingester) OnUpdate(clusterNamespaces m3.ClusterNamespaces) {
 		return
 	}
 
-	compiledRules, err := i.compileRulesWithLock(*rules)
+	compiledRules, err := compileRules(*rules)
 	if err != nil {
 		i.logger.Error("failed to compile latest rules. continuing to use existing carbon ingestion "+
 			"rules", zap.Error(err))
@@ -401,10 +407,8 @@ func (i *ingester) writeWithOptions(
 		return err
 	}
 
-	err = i.downsamplerAndWriter.Write(
-		ctx, tags, resources.datapoints, xtime.Second, nil, opts,
-	)
-
+	err = i.downsamplerAndWriter.Write(ctx, tags, resources.datapoints,
+		xtime.Second, nil, opts)
 	if err != nil {
 		i.logger.Error("err writing carbon metric",
 			zap.String("name", string(resources.name)), zap.Error(err))
@@ -419,18 +423,26 @@ func (i *ingester) Close() {
 	// We don't maintain any state in-between connections so there is nothing to do here.
 }
 
-func newCarbonIngesterMetrics(m tally.Scope) carbonIngesterMetrics {
-	return carbonIngesterMetrics{
-		success:   m.Counter("success"),
-		err:       m.Counter("error"),
-		malformed: m.Counter("malformed"),
-	}
+type carbonIngesterMetrics struct {
+	success       tally.Counter
+	err           tally.Counter
+	malformed     tally.Counter
+	ingestLatency tally.Histogram
+	writeLatency  tally.Histogram
 }
 
-type carbonIngesterMetrics struct {
-	success   tally.Counter
-	err       tally.Counter
-	malformed tally.Counter
+func newCarbonIngesterMetrics(scope tally.Scope) (carbonIngesterMetrics, error) {
+	buckets, err := ingest.NewLatencyBuckets()
+	if err != nil {
+		return carbonIngesterMetrics{}, err
+	}
+	return carbonIngesterMetrics{
+		success:       scope.Counter("success"),
+		err:           scope.Counter("error"),
+		malformed:     scope.Counter("malformed"),
+		writeLatency:  scope.SubScope("write").Histogram("latency", buckets.WriteLatencyBuckets),
+		ingestLatency: scope.SubScope("ingest").Histogram("latency", buckets.IngestLatencyBuckets),
+	}, nil
 }
 
 // GenerateTagsFromName accepts a carbon metric name and blows it up into a list of
@@ -519,7 +531,7 @@ func generateTagsFromName(
 // Note that only one rule will be applied per metric and rules are applied
 // such that the first one that matches takes precedence. As a result we need
 // to make sure to maintain the order of the rules when we generate the compiled ones.
-func (i *ingester) compileRulesWithLock(rules CarbonIngesterRules) ([]ruleAndRegex, error) {
+func compileRules(rules CarbonIngesterRules) ([]ruleAndRegex, error) {
 	var compiledRules []ruleAndRegex
 	for _, rule := range rules.Rules {
 		compiled, err := regexp.Compile(rule.Pattern)