roachtest: new test simulating system crash and sync failures

c-deps/libroach/CMakeLists.txt

@@ -54,6 +54,7 @@ add_library(roach
   protos/util/log/log.pb.cc
   protos/util/unresolved_addr.pb.cc
   rocksdbutils/env_encryption.cc
+  rocksdbutils/env_sync_fault_injection.cc
 )
 target_include_directories(roach
   PUBLIC  ./include

c-deps/libroach/db.cc

@@ -19,6 +19,7 @@
 #include <rocksdb/sst_file_writer.h>
 #include <rocksdb/table.h>
 #include <rocksdb/utilities/checkpoint.h>
+#include <rocksdb/utilities/object_registry.h>
 #include <stdarg.h>
 #include "batch.h"
 #include "cache.h"
@@ -34,6 +35,7 @@
 #include "iterator.h"
 #include "merge.h"
 #include "options.h"
+#include "rocksdbutils/env_sync_fault_injection.h"
 #include "snapshot.h"
 #include "status.h"
 #include "table_props.h"
@@ -151,7 +153,41 @@ static DBOpenHook* db_open_hook = DBOpenHookOSS;
 
 void DBSetOpenHook(void* hook) { db_open_hook = (DBOpenHook*)hook; }
 
+void DBRegisterTestingEnvs() {
+  // Have a couple `SyncFaultInjectionEnv`s registered with RocksDB for our test
+  // cases to use, which they can select using any of RocksDB's text-based options
+  // mechanisms. Currently the only one Cockroach exposes is options string which
+  // is passed the value from `DBOptions::rocksdb_options`.
+  static rocksdb::Registrar<rocksdb::Env> sync_failure_env_reg(
+      "sync-failure-injection-wrapping-default-env",
+      [](const std::string& /* name */,
+         std::unique_ptr<rocksdb::Env>* /* guard */) {
+        static rocksdb_utils::SyncFaultInjectionEnv env(
+            rocksdb::Env::Default(),
+            0 /* crash_failure_one_in */,
+            500000 /* sync_failure_one_in */,
+            true /* crash_after_sync_failure */);
+        return &env;
+      }
+  );
+
+  static rocksdb::Registrar<rocksdb::Env> crash_failure_env_reg(
+      "crash-failure-injection-wrapping-default-env",
+      [](const std::string& /* name */,
+         std::unique_ptr<rocksdb::Env>* /* guard */) {
+        static rocksdb_utils::SyncFaultInjectionEnv env(
+            rocksdb::Env::Default(),
+            500000 /* crash_failure_one_in */,
+            0 /* sync_failure_one_in */,
+            false /* crash_after_sync_failure */);
+        return &env;
+      }
+  );
+}
+
 DBStatus DBOpen(DBEngine** db, DBSlice dir, DBOptions db_opts) {
+  DBRegisterTestingEnvs();
+
   rocksdb::Options options = DBMakeOptions(db_opts);
 
   const std::string additional_options = ToString(db_opts.rocksdb_options);

c-deps/libroach/rocksdbutils/env_sync_fault_injection.cc

@@ -0,0 +1,132 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied.  See the License for the specific language governing
+// permissions and limitations under the License.
+
+#include "env_sync_fault_injection.h"
+#include "rocksdb/utilities/object_registry.h"
+
+namespace rocksdb_utils {
+
+// See comment above `SyncFaultInjectionEnv` class definition.
+class SyncFaultInjectionWritableFile : public rocksdb::WritableFileWrapper {
+ public:
+  SyncFaultInjectionWritableFile(std::unique_ptr<rocksdb::WritableFile> target,
+                                 int crash_failure_one_in,
+                                 int sync_failure_one_in,
+                                 bool crash_after_sync_failure);
+
+  rocksdb::Status Append(const rocksdb::Slice& data) override;
+  rocksdb::Status Sync() override;
+
+ private:
+  std::unique_ptr<rocksdb::WritableFile> target_;
+  const int crash_failure_one_in_;
+  const int sync_failure_one_in_;
+  const bool crash_after_sync_failure_;
+  // Countdown until crash if a sync failure already happened.
+  int num_syncs_until_crash_;
+  // Lock needed to handle concurrent writes and syncs.
+  std::mutex mu_;
+  // A buffer of written but unsynced data.
+  std::string buffer_;
+
+  // Some constants for use with `num_syncs_until_crash_`.
+  const static int kNoCountdown = -1;
+  const static int kStartCountdown = 10;
+};
+
+SyncFaultInjectionWritableFile::SyncFaultInjectionWritableFile(
+    std::unique_ptr<rocksdb::WritableFile> target,
+    int crash_failure_one_in,
+    int sync_failure_one_in,
+    bool crash_after_sync_failure) :
+  rocksdb::WritableFileWrapper(target.get()),
+  target_(std::move(target)),
+  crash_failure_one_in_(crash_failure_one_in),
+  sync_failure_one_in_(sync_failure_one_in),
+  crash_after_sync_failure_(crash_after_sync_failure),
+  num_syncs_until_crash_(kNoCountdown) {}
+
+rocksdb::Status SyncFaultInjectionWritableFile::Append(
+    const rocksdb::Slice& data) {
+  std::unique_lock<std::mutex> lock(mu_);
+  buffer_.append(data.data(), data.size());
+  return rocksdb::Status::OK();
+}
+
+// We are using process crash to simulate system crash for tests and don't
+// expect these tests to face actual system crashes. So for "syncing" it is
+// sufficient to push data into page cache via the underlying `WritableFile`'s
+// `Append()`. That should be enough for the file data to survive a process
+// crash.
+rocksdb::Status SyncFaultInjectionWritableFile::Sync() {
+  std::unique_lock<std::mutex> lock(mu_);
+  if (num_syncs_until_crash_ > kNoCountdown) {
+    --num_syncs_until_crash_;
+    if (num_syncs_until_crash_ == 0) {
+      exit(0);
+    }
+    // On Linux the behavior after a sync failure occurred is to clear the error
+    // state and continue accepting writes/syncs. To simulate that behavior, we
+    // do not return early here, even though the file is known to have lost writes.
+  }
+
+  if (crash_failure_one_in_ > 0 && random() % crash_failure_one_in_ == 0) {
+    exit(0);
+  } else if (sync_failure_one_in_ > 0 && random() % sync_failure_one_in_ == 0) {
+    if (num_syncs_until_crash_ == kNoCountdown && crash_after_sync_failure_) {
+      // This was the first failure. Start the countdown.
+      num_syncs_until_crash_ = kStartCountdown;
+    }
+    // As mentioned above, after a sync failure we allow continued writes and syncs
+    // to the same file. To make sure those new writes are written at the proper offset,
+    // we cannot drop unsynced writes simply by clearing the buffer. Instead we drop
+    // unsynced writes by overwriting the buffer with all zeros (well, this assumes
+    // the buffer didn't have all zeros to begin with).
+    buffer_.replace(0, buffer_.size(), buffer_.size(), '\0');
+    return rocksdb::Status::IOError();
+  }
+  std::string old_buffer;
+  buffer_.swap(old_buffer);
+  // It should be fine to buffer new writes while we're syncing old ones, so unlock.
+  lock.unlock();
+  return target_->Append(old_buffer);
+}
+
+SyncFaultInjectionEnv::SyncFaultInjectionEnv(
+    Env* target,
+    int crash_failure_one_in,
+    int sync_failure_one_in,
+    bool crash_after_sync_failure) :
+  rocksdb::EnvWrapper(target),
+  crash_failure_one_in_(crash_failure_one_in),
+  sync_failure_one_in_(sync_failure_one_in),
+  crash_after_sync_failure_(crash_after_sync_failure) {}
+
+rocksdb::Status SyncFaultInjectionEnv::NewWritableFile(
+    const std::string& filename,
+    std::unique_ptr<rocksdb::WritableFile>* result,
+    const rocksdb::EnvOptions& env_options) {
+  std::unique_ptr<rocksdb::WritableFile> underlying_file;
+  rocksdb::Status s = EnvWrapper::NewWritableFile(filename, &underlying_file, env_options);
+  if (s.ok()) {
+    result->reset(new SyncFaultInjectionWritableFile(
+          std::move(underlying_file),
+          crash_failure_one_in_,
+          sync_failure_one_in_,
+          crash_after_sync_failure_));
+  }
+  return s;
+}
+
+}  // rocksdb_utils

c-deps/libroach/rocksdbutils/env_sync_fault_injection.h

@@ -0,0 +1,61 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied.  See the License for the specific language governing
+// permissions and limitations under the License.
+
+#pragma once
+
+#include <mutex>
+#include <string>
+
+#include "rocksdb/env.h"
+
+namespace rocksdb_utils {
+
+// `SyncFaultInjectionEnv` creates files that buffer `Append()`s in process memory
+// until `Sync()` is called. Such files enable us to simulate machine crashes by only
+// crashing the process. This works since, unlike normal files whose writes survive
+// process crash in page cache, these files' unsynced writes are dropped on the floor.
+//
+// Such files also enable us to simulate sync failure by dropping unsynced writes at
+// the same time we inject a sync error. This is more comprehensive than the available
+// fault injection tools I looked at (like libfiu and charybdefs), as those ones only
+// inject errors without dropping unsynced writes.
+class SyncFaultInjectionEnv : public rocksdb::EnvWrapper {
+ public:
+  // - `target`: A pointer to the underlying `Env`.
+  // - `crash_failure_one_in`: During a sync operation, crash the process immediately
+  //   with a probability of 1/n. All unsynced writes are lost since they are buffered
+  //   in process memory.
+  // - `sync_failure_one_in`: A sync operation will return failure with a probability
+  //   of 1/n. All unsynced writes for the file are dropped to simulate the failure.
+  // - `crash_after_sync_failure`: If set to true, the program will crash itself some
+  //   time after the first simulated sync failure. It does not happen immediately to
+  //   allow the system to get itself into a weird state in case it doesn't handle sync
+  //   failures properly.
+  SyncFaultInjectionEnv(
+      Env* target,
+      int crash_failure_one_in,
+      int sync_failure_one_in,
+      bool crash_after_sync_failure);
+
+  rocksdb::Status NewWritableFile(const std::string& filename,
+                                  std::unique_ptr<rocksdb::WritableFile>* result,
+                                  const rocksdb::EnvOptions& env_options) override;
+
+ private:
+  const int crash_failure_one_in_;
+  const int sync_failure_one_in_;
+  const bool crash_after_sync_failure_;
+};
+
+}  // rocksdb_utils

pkg/cmd/roachprod/install/cockroach.go

@@ -386,6 +386,10 @@ tar cvf certs.tar certs
 		cmd += `echo ">>> roachprod start: $(date)" >> ` + logDir + "/roachprod.log; " +
 			`ps axeww -o pid -o command >> ` + logDir + "/roachprod.log; " +
 			`[ -x /usr/bin/lslocks ] && /usr/bin/lslocks >> ` + logDir + "/roachprod.log; "
+		if c.IsLocal() {
+			// This is consistent with the working directory used by `roachprod run`.
+			cmd += fmt.Sprintf("cd ${HOME}/local/%d ; ", nodes[i])
+		}
 		cmd += keyCmd +
 			fmt.Sprintf(" export ROACHPROD=%d%s && ", nodes[i], c.Tag) +
 			"GOTRACEBACK=crash " +

pkg/cmd/roachprod/vm/gce/gcloud.go

@@ -51,7 +51,7 @@ var projectsWithGC = []string{defaultProject, "andrei-jepsen"}
 
 // init will inject the GCE provider into vm.Providers, but only if the gcloud tool is available on the local path.
 func init() {
-	var p vm.Provider
+	var p vm.Provider = &Provider{}
 	if _, err := exec.LookPath("gcloud"); err != nil {
 		p = flagstub.New(p, "please install the gcloud CLI utilities "+
 			"(https://cloud.google.com/sdk/downloads)")

pkg/cmd/roachtest/cluster.go

@@ -925,7 +925,7 @@ func (c *cluster) All() nodeListOption {
 	return c.Range(1, c.nodes)
 }
 
-// All returns a node list containing the nodes [begin,end].
+// Range returns a node list containing the nodes [begin,end].
 func (c *cluster) Range(begin, end int) nodeListOption {
 	if begin < 1 || end > c.nodes {
 		c.t.Fatalf("invalid node range: %d-%d (1-%d)", begin, end, c.nodes)
@@ -937,7 +937,7 @@ func (c *cluster) Range(begin, end int) nodeListOption {
 	return r
 }
 
-// All returns a node list containing only the node i.
+// Node returns a node list containing only the node i.
 func (c *cluster) Node(i int) nodeListOption {
 	return c.Range(i, i)
 }

pkg/cmd/roachtest/registry.go

@@ -69,6 +69,7 @@ func registerTests(r *registry) {
 	registerSQLsmith(r)
 	registerSyncTest(r)
 	registerSysbench(r)
+	registerSystemCrashTest(r)
 	registerTPCC(r)
 	registerTypeORM(r)
 	registerLoadSplits(r)