Merge #66268 #69496 #69572 #69593 #69594 #69662

66268: storage,kv: faster intent resolution over a key range r=sumeerbhola a=sumeerbhola

Closes #66900

When all the intents on a range are known to be separated,
as indicated by MCCStats, and a ResolveIntentRangeRequest
needs to be processed (for a transaction that did enough
writes that we did not track individual intent keys), we
avoid iterating over MVCC key-value pairs to find those
intents, and instead iterate over the separated lock table
key space.

A new iterForKeyVersions interface is introduced which
provides a narrow iterator interface to be used by
mvccResolveWriteIntent that resolves individual intents.
This interface is only for iterating over a single key,
and is implemented by the usual MVCCIterator and by the
new separatedIntentAndVersionIter. The latter serves
both as a way to find intents over a range and for
individual intent resolution, in an optimized manner.

There are new benchmarks which vary the sparseness
(sparseness of N means that 1 in every N keys has an
intent that is for the txn for which intent resolution
is being performed) and whether the remaining N-1 keys
have an intent from a different transaction
(other-txn-intents=true) or not (the former results
in the cost of at least parsing the MVCCMetadata to
compare txn IDs).

Full results are below. Two things to note are:
- The percent-flushed={0,50} sometimes show a regression.
  These are atypical cases where the SingleDelete to
  remove the intent has not been flushed so the intent
  Set and SingleDelete are both alive. This would be
  a pathalogical case when the LSM is unhealthy. Even
  then the regression is < 170%.
- The percent-flushed=100 cases are the typical one.
  Once sparseness increases to 100 or 1000 we see speed
  gains close to 100x, as indicated by the following:

```
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=100-16        235µs ± 2%      58µs ± 3%   -75.52%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=100-16      154µs ± 2%       2µs ± 4%   -98.72%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=100-16     137µs ± 1%       1µs ± 2%   -99.07%  (p=0.008 n=5+5)
```

Full results:
```
name                                                                                                             old time/op    new time/op    delta
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=0-16          450µs ± 5%     234µs ± 2%   -48.07%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=50-16         294µs ± 3%     101µs ± 4%   -65.49%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=100-16        235µs ± 2%      58µs ± 3%   -75.52%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=0-16        494µs ± 7%     218µs ± 6%   -55.84%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=50-16       251µs ± 2%      57µs ± 3%   -77.29%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=100-16      154µs ± 2%       2µs ± 4%   -98.72%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=0-16         343µs ± 5%     236µs ± 2%   -31.28%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=50-16        240µs ± 1%      74µs ± 2%   -69.02%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=100-16       203µs ± 1%      30µs ± 1%   -84.97%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=0-16       516µs ± 4%     274µs ±13%   -46.91%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=50-16      256µs ± 2%      57µs ± 1%   -77.67%  (p=0.016 n=4+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=100-16     137µs ± 1%       1µs ± 2%   -99.07%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=0-16        325µs ± 2%     237µs ± 1%   -26.98%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=50-16       232µs ± 4%      72µs ± 1%   -68.89%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=100-16      199µs ± 1%      30µs ± 0%   -84.87%  (p=0.016 n=5+4)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=0-16        2.08ms ± 1%    2.68ms ± 8%   +28.57%  (p=0.016 n=4+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=50-16        592µs ± 4%     596µs ± 7%      ~     (p=1.000 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=100-16       242µs ± 2%      59µs ± 3%   -75.73%  (p=0.016 n=4+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=0-16      3.04ms ± 5%    2.60ms ± 3%   -14.59%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=50-16      840µs ± 4%     539µs ± 3%   -35.87%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=100-16     163µs ± 6%       2µs ± 6%   -98.75%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=0-16        936µs ±38%    2497µs ±12%  +166.83%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=50-16       281µs ±12%     551µs ± 2%   +95.64%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=100-16      209µs ±13%      31µs ± 9%   -84.95%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1000/other-txn-intents=false/percent-flushed=0-16     3.04ms ± 6%    2.55ms ± 0%      ~     (p=0.333 n=5+1)

name                                                                                                             old alloc/op   new alloc/op   delta
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=0-16         12.8kB ± 0%    13.0kB ± 0%    +0.92%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=50-16        12.8kB ± 0%    13.0kB ± 0%    +0.94%  (p=0.016 n=5+4)
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=100-16       12.9kB ± 0%    13.0kB ± 0%    +0.59%  (p=0.016 n=5+4)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=0-16       1.75kB ± 0%    0.29kB ± 0%      ~     (p=0.079 n=4+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=50-16      1.75kB ± 0%    0.29kB ± 0%   -83.18%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=100-16     1.75kB ± 0%    0.29kB ± 0%   -83.22%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=0-16        12.8kB ± 0%    11.4kB ± 0%   -11.37%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=50-16       12.8kB ± 0%    11.4kB ± 0%      ~     (p=0.079 n=4+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=100-16      12.8kB ± 0%    11.4kB ± 0%   -11.41%  (p=0.000 n=5+4)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=0-16      1.66kB ± 0%    0.17kB ± 0%   -89.61%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=50-16     1.66kB ± 0%    0.17kB ± 0%   -89.61%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=100-16    1.66kB ± 0%    0.17kB ± 0%   -89.63%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=0-16       12.8kB ± 0%    11.4kB ± 0%   -11.50%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=50-16      12.8kB ± 0%    11.4kB ± 0%      ~     (p=0.079 n=4+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=100-16     12.8kB ± 0%    11.4kB ± 0%   -11.53%  (p=0.029 n=4+4)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=0-16        12.8kB ± 0%    13.0kB ± 0%    +0.95%  (p=0.016 n=5+4)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=50-16       12.8kB ± 0%    13.0kB ± 0%    +0.95%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=100-16      12.9kB ± 0%    13.0kB ± 0%    +0.54%  (p=0.016 n=5+4)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=0-16      1.75kB ± 0%    0.29kB ± 0%   -83.14%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=50-16     1.75kB ± 0%    0.29kB ± 0%   -83.19%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=100-16    1.75kB ± 0%    0.29kB ± 0%   -83.22%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=0-16       12.8kB ± 0%    11.4kB ± 0%   -11.37%  (p=0.000 n=5+4)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=50-16      12.8kB ± 0%    11.4kB ± 0%   -11.37%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=100-16     12.8kB ± 0%    11.4kB ± 0%   -11.41%  (p=0.000 n=5+4)
IntentRangeResolution/separated=true/versions=100/sparseness=1000/other-txn-intents=false/percent-flushed=0-16     1.66kB ± 0%    0.17kB ± 0%   -89.56%  (p=0.000 n=5+1)

name                                                                                                             old allocs/op  new allocs/op  delta
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=0-16            401 ± 0%       404 ± 0%    +0.75%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=50-16           401 ± 0%       404 ± 0%    +0.75%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1/other-txn-intents=false/percent-flushed=100-16          401 ± 0%       404 ± 0%    +0.75%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=0-16          104 ± 0%         8 ± 0%   -92.31%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=50-16         104 ± 0%         8 ± 0%   -92.31%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=false/percent-flushed=100-16        104 ± 0%         8 ± 0%   -92.31%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=0-16           401 ± 0%       305 ± 0%   -23.94%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=50-16          401 ± 0%       305 ± 0%   -23.94%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=100/other-txn-intents=true/percent-flushed=100-16         401 ± 0%       305 ± 0%   -23.94%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=0-16         102 ± 0%         3 ± 0%   -97.06%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=50-16        102 ± 0%         3 ± 0%   -97.06%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=false/percent-flushed=100-16       102 ± 0%         3 ± 0%   -97.06%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=0-16          401 ± 0%       303 ± 0%   -24.44%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=50-16         401 ± 0%       303 ± 0%   -24.44%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=10/sparseness=1000/other-txn-intents=true/percent-flushed=100-16        401 ± 0%       303 ± 0%   -24.44%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=0-16           401 ± 0%       404 ± 0%    +0.75%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=50-16          401 ± 0%       404 ± 0%    +0.75%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1/other-txn-intents=false/percent-flushed=100-16         401 ± 0%       404 ± 0%    +0.75%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=0-16         104 ± 0%         8 ± 0%   -92.31%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=50-16        104 ± 0%         8 ± 0%   -92.31%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=false/percent-flushed=100-16       104 ± 0%         8 ± 0%   -92.31%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=0-16          401 ± 0%       305 ± 0%   -23.94%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=50-16         401 ± 0%       305 ± 0%   -23.94%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=100/other-txn-intents=true/percent-flushed=100-16        401 ± 0%       305 ± 0%   -23.94%  (p=0.008 n=5+5)
IntentRangeResolution/separated=true/versions=100/sparseness=1000/other-txn-intents=false/percent-flushed=0-16        102 ± 0%         3 ± 0%   -97.06%  (p=0.000 n=5+1)
```

Release note (performance improvement): Intent resolution for transactions
that write many intents such that we track intent ranges, for the purpose
of intent resolution, is much faster (potentially 100x) when using the
separated lock table.

Release justification: Fix for high-severity poor performance in existing
functionality, and is a low risk, high benefit change, that potentially
improves ranged intent resolution speed by 100x.

69496: engineccl: copy registry metadata before rename r=jbowens a=jbowens

During a rename operation on an `encryptedFS`, copy the source's file
metadata to the destination first, perform the rename, then remove the
source's metadata.

This ensures that if the destination path does not exist, the rename is
atomic.

Release justification: partial fix for a high-severity bug in existing
functionality
Release note: None

69572: cloud: bump orchestrator version to 21.1.8 r=celiala a=celiala

Release justification: Non-production code changes
Release note: None

69593: roachtest: update 21.2 version map to 21.1.8 r=celiala a=celiala

Release justification: non-production code change
Release note: None

69594: sqlproxyccl: Add pprof debug endpoints r=alyshanjahani-crl a=alyshanjahani-crl

Add pprof debug endpoints to sqlproxy HTTP server.

Note: There is a debug server used by the KV and (multi-tenant) SQL servers (https://github.com/cockroachdb/cockroach/blob/master/pkg/server/debug/server.go#L70). However, this uses a `cluster.Settings` struct, it isn't clear to me whether the sqlproxy server can reuse this.

Also, there is some precedence of adding pprof endpoints directly as seen here:https://github.com/cockroachdb/cockroach/blob/master/pkg/workload/cli/csv_server.go

69662: roachtest: bump tpcc load warehouses and estimates r=ajwerner a=ajwerner

We're hitting the limits on both AWS and GCE sometimes. Pretty much all the
time on AWS.

<img width="903" alt="Screen Shot 2021-08-31 at 3 11 17 PM" src="https://user-images.githubusercontent.com/1839234/131561909-1cacbe18-9a15-488b-b8cf-21120ae7d8ad.png">

Release justification: non-production code changes.

Release note: None

Co-authored-by: sumeerbhola <sumeer@cockroachlabs.com>
Co-authored-by: Jackson Owens <jackson@cockroachlabs.com>
Co-authored-by: Celia La <celiala456@gmail.com>
Co-authored-by: Alyshan Jahani <alyshan@cockroachlabs.com>
Co-authored-by: Andrew Werner <awerner32@gmail.com>

cloud/kubernetes/bring-your-own-certs/client.yaml

@@ -19,7 +19,7 @@ spec:
   serviceAccountName: cockroachdb
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v21.1.7
+    image: cockroachdb/cockroach:v21.1.8
     # Keep a pod open indefinitely so kubectl exec can be used to get a shell to it
     # and run cockroach client commands, such as cockroach sql, cockroach node status, etc.
     command:

cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml

@@ -152,7 +152,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/client-secure.yaml

@@ -31,7 +31,7 @@ spec:
       mountPath: /cockroach-certs
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v21.1.7
+    image: cockroachdb/cockroach:v21.1.8
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/cluster-init-secure.yaml

@@ -33,7 +33,7 @@ spec:
           mountPath: /cockroach-certs
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/cluster-init.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         command:
           - "/cockroach/cockroach"

cloud/kubernetes/cockroachdb-statefulset-secure.yaml

@@ -194,7 +194,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/cockroachdb-statefulset.yaml

@@ -97,7 +97,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/multiregion/client-secure.yaml

@@ -8,7 +8,7 @@ spec:
   serviceAccountName: cockroachdb
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v21.1.7
+    image: cockroachdb/cockroach:v21.1.8
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/multiregion/cluster-init-secure.yaml

@@ -10,7 +10,7 @@ spec:
       serviceAccountName: cockroachdb
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/multiregion/cockroachdb-statefulset-secure.yaml

@@ -166,7 +166,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/multiregion/eks/cockroachdb-statefulset-secure-eks.yaml

@@ -184,7 +184,7 @@ spec:
           name: cockroach-env
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/performance/cockroachdb-daemonset-insecure.yaml

@@ -81,7 +81,7 @@ spec:
       hostNetwork: true
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: If you configured taints to give CockroachDB exclusive access to nodes, feel free
         # to remove the requests and limits sections. If you didn't, you'll need to change these to

cloud/kubernetes/performance/cockroachdb-daemonset-secure.yaml

@@ -197,7 +197,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: If you configured taints to give CockroachDB exclusive access to nodes, feel free
         # to remove the requests and limits sections. If you didn't, you'll need to change these to

cloud/kubernetes/performance/cockroachdb-statefulset-insecure.yaml

@@ -140,7 +140,7 @@ spec:
       - name: cockroachdb
         # NOTE: Always use the most recent version of CockroachDB for the best
         # performance and reliability.
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/performance/cockroachdb-statefulset-secure.yaml

@@ -231,7 +231,7 @@ spec:
       - name: cockroachdb
         # NOTE: Always use the most recent version of CockroachDB for the best
         # performance and reliability.
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/v1.6/client-secure.yaml

@@ -31,7 +31,7 @@ spec:
       mountPath: /cockroach-certs
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v21.1.7
+    image: cockroachdb/cockroach:v21.1.8
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/v1.6/cluster-init-secure.yaml

@@ -33,7 +33,7 @@ spec:
           mountPath: /cockroach-certs
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/v1.6/cluster-init.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         command:
           - "/cockroach/cockroach"

cloud/kubernetes/v1.6/cockroachdb-statefulset-secure.yaml

@@ -177,7 +177,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/v1.6/cockroachdb-statefulset.yaml

@@ -80,7 +80,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/v1.7/client-secure.yaml

@@ -31,7 +31,7 @@ spec:
       mountPath: /cockroach-certs
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v21.1.7
+    image: cockroachdb/cockroach:v21.1.8
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/v1.7/cluster-init-secure.yaml

@@ -33,7 +33,7 @@ spec:
           mountPath: /cockroach-certs
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/v1.7/cluster-init.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         command:
           - "/cockroach/cockroach"

cloud/kubernetes/v1.7/cockroachdb-statefulset-secure.yaml

@@ -189,7 +189,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/v1.7/cockroachdb-statefulset.yaml

@@ -92,7 +92,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v21.1.7
+        image: cockroachdb/cockroach:v21.1.8
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

pkg/ccl/sqlproxyccl/server.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"net"
 	"net/http"
+	"net/http/pprof"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/util/contextutil"
@@ -72,6 +73,14 @@ func NewServer(ctx context.Context, stopper *stop.Stopper, options ProxyOptions)
 	mux.HandleFunc("/_status/vars/", s.handleVars)
 	mux.HandleFunc("/_status/healthz/", s.handleHealth)
 
+	// Taken from pprof's `init()` method. See:
+	// https://golang.org/src/net/http/pprof/pprof.go
+	mux.HandleFunc("/debug/pprof/", pprof.Index)
+	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
 	return s, nil
 }
 
@@ -108,8 +117,9 @@ func (s *Server) handleVars(w http.ResponseWriter, r *http.Request) {
 }
 
 // ServeHTTP starts the proxy's HTTP server on the given listener.
-// The server provides Prometheus metrics at /_status/vars
-// and a health check endpoint at /_status/healthz.
+// The server provides Prometheus metrics at /_status/vars,
+// a health check endpoint at /_status/healthz, and pprof debug
+// endpoints at /debug/pprof.
 func (s *Server) ServeHTTP(ctx context.Context, ln net.Listener) error {
 	srv := http.Server{
 		Handler: s.mux,

pkg/ccl/storageccl/engineccl/encrypted_fs.go

@@ -192,12 +192,38 @@ func (fs *encryptedFS) Remove(name string) error {
 	return fs.fileRegistry.MaybeDeleteEntry(name)
 }
 
-// Rename implements vfs.FS.Rename.
+// Rename implements vfs.FS.Rename. A rename operation needs to both
+// move the file's file registry entry and move the files on the
+// physical filesystem. These operations cannot be done atomically. The
+// encryptedFS's Rename operation provides atomicity only if the
+// destination path does not exist.
+//
+// Rename will first copy the old path's file registry entry to the
+// new path. If the destination exists, a crash after this copy will
+// leave the file at the new path unreadable.
+//
+// Rename then performs a filesystem rename of the actual file. If a
+// crash occurs after the rename, the file at the new path will be
+// readable. The file at the old path won't exist, but the file registry
+// will contain a dangling entry for the old path. The dangling entry
+// will be elided when the file registry is loaded again.
 func (fs *encryptedFS) Rename(oldname, newname string) error {
+	// First copy the metadata from the old name to the new name. If a
+	// file exists at newname, this copy action will make the file at
+	// newname unlegible, because the encryption-at-rest metadata will
+	// not match the file's encryption. This is what makes Rename
+	// non-atomic. If no file exists at newname, the dangling copied
+	// entry has no effect.
+	if err := fs.fileRegistry.MaybeCopyEntry(oldname, newname); err != nil {
+		return err
+	}
+	// Perform the filesystem rename. After the filesystem rename, the
+	// new path is guaranteed to be readable.
 	if err := fs.FS.Rename(oldname, newname); err != nil {
 		return err
 	}
-	return fs.fileRegistry.MaybeRenameEntry(oldname, newname)
+	// Remove the old name's metadata.
+	return fs.fileRegistry.MaybeDeleteEntry(oldname)
 }
 
 // ReuseForWrite implements vfs.FS.ReuseForWrite.

pkg/cmd/roachtest/tests/predecessor_version.go

@@ -36,7 +36,7 @@ func PredecessorVersion(buildVersion version.Version) (string, error) {
 	// checkpoint option enabled to create the missing store directory
 	// fixture (see runVersionUpgrade).
 	verMap := map[string]string{
-		"21.2": "21.1.7",
+		"21.2": "21.1.8",
 		"21.1": "20.2.12",
 		"20.2": "20.1.16",
 		"20.1": "19.2.11",

pkg/cmd/roachtest/tests/tpcc.go

@@ -739,16 +739,16 @@ func registerTPCC(r registry.Registry) {
 		Nodes: 3,
 		CPUs:  16,
 
-		LoadWarehouses: gceOrAws(cloud, 2500, 3000),
-		EstimatedMax:   gceOrAws(cloud, 2100, 2500),
+		LoadWarehouses: gceOrAws(cloud, 3000, 3500),
+		EstimatedMax:   gceOrAws(cloud, 2400, 3000),
 	})
 	registerTPCCBenchSpec(r, tpccBenchSpec{
 		Nodes:                   3,
 		CPUs:                    16,
 		AdmissionControlEnabled: true,
 
-		LoadWarehouses: gceOrAws(cloud, 2500, 3000),
-		EstimatedMax:   gceOrAws(cloud, 2100, 2500),
+		LoadWarehouses: gceOrAws(cloud, 3000, 3500),
+		EstimatedMax:   gceOrAws(cloud, 2400, 3000),
 	})
 	registerTPCCBenchSpec(r, tpccBenchSpec{
 		Nodes: 12,

pkg/kv/kvserver/batch_spanset_test.go

@@ -596,7 +596,7 @@ func TestSpanSetMVCCResolveWriteIntentRange(t *testing.T) {
 		Status: roachpb.PENDING,
 	}
 	if _, _, err := storage.MVCCResolveWriteIntentRange(
-		ctx, batch, nil /* ms */, intent, 0,
+		ctx, batch, nil /* ms */, intent, 0, eng.IsSeparatedIntentsEnabledForTesting(ctx),
 	); err != nil {
 		t.Fatal(err)
 	}

pkg/kv/kvserver/batcheval/BUILD.bazel

@@ -114,6 +114,7 @@ go_test(
     embed = [":batcheval"],
     deps = [
         "//pkg/base",
+        "//pkg/clusterversion",
         "//pkg/keys",
         "//pkg/kv",
         "//pkg/kv/kvserver",

pkg/kv/kvserver/batcheval/cmd_end_transaction.go

@@ -456,6 +456,13 @@ func resolveLocalLocks(
 		// These transactions rely on having their locks resolved synchronously.
 		resolveAllowance = math.MaxInt64
 	}
+	onlySeparatedIntents := false
+	st := evalCtx.ClusterSettings()
+	// Some tests have st == nil.
+	if st != nil {
+		onlySeparatedIntents = st.Version.ActiveVersionOrEmpty(ctx).IsActive(
+			clusterversion.PostSeparatedIntentsMigration)
+	}
 	for _, span := range args.LockSpans {
 		if err := func() error {
 			if resolveAllowance == 0 {
@@ -496,7 +503,7 @@ func resolveLocalLocks(
 			if inSpan != nil {
 				update.Span = *inSpan
 				num, resumeSpan, err := storage.MVCCResolveWriteIntentRange(
-					ctx, readWriter, ms, update, resolveAllowance)
+					ctx, readWriter, ms, update, resolveAllowance, onlySeparatedIntents)
 				if err != nil {
 					return err
 				}

pkg/kv/kvserver/batcheval/cmd_resolve_intent_range.go

@@ -13,6 +13,7 @@ package batcheval
 import (
 	"context"
 
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
@@ -44,8 +45,11 @@ func ResolveIntentRange(
 
 	update := args.AsLockUpdate()
 
+	onlySeparatedIntents :=
+		cArgs.EvalCtx.ClusterSettings().Version.ActiveVersionOrEmpty(ctx).IsActive(
+			clusterversion.PostSeparatedIntentsMigration)
 	numKeys, resumeSpan, err := storage.MVCCResolveWriteIntentRange(
-		ctx, readWriter, ms, update, h.MaxSpanRequestKeys)
+		ctx, readWriter, ms, update, h.MaxSpanRequestKeys, onlySeparatedIntents)
 	if err != nil {
 		return result.Result{}, err
 	}